@zhivex-ai/harness 1.0.0 → 1.1.0-rc.3

package/benchmarks/baselines/rc14-holdout-2026-09-10.json

@@ -1,854 +0,0 @@
-{
-  "schemaVersion": 1,
-  "kind": "rc14-holdout-validation",
-  "sourceCommit": "0609f11681cf44d0fc68b7671ed4e37ceaf48309",
-  "sourceHashes": {
-    "bun.lock": "b0440362d043879ece281bcee4c7adfa775a3724d2a9477079e187babd3b2d26",
-    "evaluations/external/requirements.lock": "42e90a08f10f821ebb958af79256f707a9fb5cf365277c4f9e2715d612028179",
-    "package.json": "0ed0eea4c944350f65a7e1e7304d8c0ec7b2ae3d07945490b51c95ae49b54777",
-    "scripts/benchmark-swebench.ts": "ed17063e1847017ba2e72f14732b6ff8f5b1af7cd5253b91302af75f2a9e8476",
-    "scripts/swebench/candidate.py": "f5e3410b91b950acfbe50fc7511891816a8fb1c223082397e2c6300cf1e866cf",
-    "scripts/swebench/candidate.ts": "bf102becfcce0c9a3f0df0a8f1ce0e5aaea7db2d252f6d9454a8370c24e53982",
-    "scripts/swebench/common.py": "12f7851e8106b22f2124751ce56e3742348ebb3c40ef0bc89ee9d2fefa0aff77",
-    "scripts/swebench/context-metrics.ts": "7b84da6789413aa634d997c5963ee6796584e06c3eafe5f68f424fb7e27815e9",
-    "scripts/swebench/mini_driver.py": "bd0a77221878370ec44760122a51f07fb56d969612afa9a7f171248b2958baa6",
-    "scripts/swebench/model-budget.ts": "e46dd406ee5d58a051a15c7d7133513666292303d8c6534f861a6db0e711f47c",
-    "scripts/swebench/preflight.py": "aed092a75dd52f504fba6d14814b69bd627ff17d0e311a1f1279f8b613b69675",
-    "scripts/swebench/prepare.py": "bb7edd3fba5dbbe26597c97d744c60b94dbb7bcdfd7740a4a058c8583bd27e4c",
-    "scripts/swebench/report.py": "d0b8d45f32c48e0ba2cb977f50838065a497a0ecaf3b4243ed94139393ce4d6b",
-    "scripts/swebench/run.py": "24ffa302e673bbf240d6396474631bb4ce80447797584c3ed27f2d1e55e452cb",
-    "scripts/swebench/telemetry.ts": "743c9baf273e9240d704d098b6bd5eea9e76ef3e49ae636aaafa9bfd7a85b45b",
-    "scripts/swebench/test_candidate.py": "e77d16379b748baa2daf184a45ea62587d938b6a7d48dcf9f6061f8af21fafd9",
-    "scripts/swebench/test_contracts.py": "a8c1348f1dff2e563c139e6d3e83eb141b4f2a71f91a58e2c5c051ed2a431024",
-    "scripts/swebench/zhivex-driver.ts": "e33ed929256133fb29be67e3feb282f0d0fd52086576a668021852c151ff35d1",
-    "scripts/time-to-safe-fix-efficiency.ts": "dd20fd8839280332ae4ca80d6846e98e07ff15d79e200b206fa2ba6be2724d2f",
-    "src/bounded-reads.ts": "9807bb460894be3279c5425af19863d1c6e419a1582ba6910ec7e7fc35703ed6",
-    "src/capabilities.ts": "56fdb60a2f6e1d33b2dc26d38b6d2709807e491ba1268d47b97876c2c0591d65",
-    "src/change-envelope.ts": "9047ec978c9e5117965b8093ec961d855f97bbc2c3accdc79b74cfe6a0e4f01e",
-    "src/cli-options.ts": "f48d9be132fed38f2ddbcfecbd8222c664a0df2f20edc28ce8d1b9b1a1900863",
-    "src/cli-profiles.ts": "4fe451f1645e95f4957597a8724fa989b3c661ae1693252bfd6e74a51d20fc74",
-    "src/cli-stream.ts": "5ab8a4b1b8cf0a3dfa7b5b1f9e4e0facf8f6e7f6c341bdf700a41dc8927c825e",
-    "src/cli.ts": "8f44daed380575481e3aa22ec559317f58d88aca30c7fc80fe860163ef5ea336",
-    "src/compaction.ts": "9e06a0abeeef155377f40c160b21c56256bca95ec18c5ee897681a5fc73db169",
-    "src/config.ts": "a31e9f91c0eb59b4d77496431639aa4b47b149e1968c6e0a002a508de4919c35",
-    "src/console-context.ts": "0ad7e4559504b9ed09ab500d615dea09af1075d357f02ec3b77f003d55d951db",
-    "src/console-input.ts": "dc8ea0cdbba1364db288d5156c5ff9819362c46b37ef35a98b0475b6c445b1f8",
-    "src/context-engineering.ts": "1995e564f6b91974e75bf66dd421ddcbea29b1f8eee59b76eb4e34c9d6d8f47e",
-    "src/context-metrics.ts": "3eb30f9db808f918ed8d73d4f8ac19a13acb09ed5c1ee70fdae05201714b9198",
-    "src/edit-contracts.ts": "47cc3132e27f8749d54a32bbcc33bb174cf4448f0d0171b2578cdb559cf2176b",
-    "src/errors.ts": "e02f4eb80562a83762f908bda56c950ec6a2b78725c0c8a49f41fee0057ab903",
-    "src/execution-environment.ts": "980ec7cb0aa74f5dbfead7dedc4ca221a27b997de2bc2d463d7073813fe83bbc",
-    "src/file-security.ts": "b8c9f4988c4339c73a60ab4fa8d394b3eecc19117a0c61a8d19ab6483ca253df",
-    "src/harness.ts": "62ab0f6f2a428ce23f3f3f8847402c75ffe7180c939a0236af87612c04fae1ba",
-    "src/index.ts": "4bbfcd11376d48c4f1c1ec17d6b5c9f51baa9331fa999432be178bae94530a5e",
-    "src/json-contracts.ts": "0780145636b47829a1aa6087c574d5654eb31b7534ea7d9c38c188d648c90272",
-    "src/mcp.ts": "7722de031921c620ec03b9e42a6596ef8415aa8e4bb5f93ea84d2427c466e554",
-    "src/model-budget.ts": "e5a5d3dd8c97df7f18f36ce0a82ed250e4a0c7d49b12580a7393db6ff2183a55",
-    "src/operations.ts": "bcccaf4d1d0ec117c2ed35df92185564d33399e5fdc5c12225ac61f14fd0b4b1",
-    "src/orchestration.ts": "332264f91fc9630e8d5d32e4ca2aebdf063fdb87acc457c4854c711c725d6e26",
-    "src/package-manager.ts": "cec4666bbab2f42e0a269709631f79f9f87c2c4dbb3cafeffacfaff761069e2c",
-    "src/process-runtime.ts": "b7286731a0b5c96dcad0b8e3a4bf1155d0539d73303cd7ebe77ffabf858f1729",
-    "src/providers.ts": "70ccfc7f315cd7238e83d69b719ce4908f3b620c564cbb4f00f93e102e025602",
-    "src/repair-controller.ts": "ce12b1c9fb9a5fa5d370b2eda5c1233cc8c33706477ef66302b52bf7e8c40e09",
-    "src/repair-progress.ts": "38270f07c4e1d81d745aa3a01837cea4420b7caf310df59c7447e56b1a3e0440",
-    "src/repair-verifier.ts": "b70fb11feee8cdc7933947ccac0b78f8bb5c0eaccc3b29ad527197d77f40ffc9",
-    "src/replacement-edits.ts": "8a0efdd3a7a9fc3e5a5475916cbd17ef882265881bb61076db58673e80b72957",
-    "src/routing.ts": "8e7c9f5968b4bd638cbbc86b19847ca183f3418b9a298783b96a88cffa568e22",
-    "src/run-interruption.ts": "df99129cf2dd6b938eca629cb897c4350a7d57fe6bf0ce437f5f3f30271617df",
-    "src/runtime-checkpoints.ts": "a9b555d4c6b5f369f86a3d79ec1a3aab45886282cc7d0ef211aa084ee17e3897",
-    "src/runtime-diagnostics.ts": "e1e5a3a558fa2aaeb477335dc336551f022c1234fe5e742eefc6599c8803e364",
-    "src/runtime-policy.ts": "bed4efad0010aa63d7e083b0c8014cb36a915626ede00b8a40e1664310972f3c",
-    "src/sessions.ts": "5167c8cf36bd4b8845ee278883394700321b65397f49c1dda0cf0d0de09dd06e",
-    "src/sqlite-database.ts": "18765d1ca0c2f2ad87bfd82d311312dfe8a42006556f27bc21da5892ed3b044e",
-    "src/state-backup.ts": "b6f8f1aedeb5dce31b65c23c7ffc50030870655b58a5aaaef9989bafb9da5c68",
-    "src/state-directory.ts": "3711550991951e6b738a08deaa0e06da84d01864c03a9602600f592963fa3211",
-    "src/task-memory.ts": "5b62c863be5a23c33608633cd7aac0464f5536d01cba930aca58b11870170731",
-    "src/terminal-markdown.ts": "8cffb6ebaf25108cb2bee04956d9b2973635b010f5d7ce6be21b3ceffe3e8bfd",
-    "src/terminal-ui.ts": "e50025a8649d1f60efdf3af895b8639730ec2e86553fb80a61a776b11076c705",
-    "src/time-to-safe-fix.ts": "4add1d49c0ea209cc496c66fa2b2012dc7bcab87e678a58c8591b3c2cdc8df8a",
-    "src/tool-registry.ts": "7e90fff11940e525883c9ec0d2adbea2e6616f8bd9bc4ec77e1b7277e4217f2c",
-    "src/version.ts": "b2c2f80a40adc08e8258b81e82434273f0bdfa787528b2221f14efe7f06aa747",
-    "src/workspace.ts": "0f472998a55672a326bfb32404a10073a90c9a34c24758471fede0345f582961"
-  },
-  "sourceAndSnapshotsVerified": true,
-  "datasetRevision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a",
-  "taskIds": [
-    "django__django-15128",
-    "django__django-15851"
-  ],
-  "excludedPriorTaskIds": [
-    "astropy__astropy-14995",
-    "django__django-11265",
-    "django__django-12262",
-    "django__django-13410",
-    "django__django-14915",
-    "django__django-15554",
-    "django__django-15569",
-    "pydata__xarray-4695",
-    "pydata__xarray-6721",
-    "pytest-dev__pytest-8399",
-    "scikit-learn__scikit-learn-10908",
-    "scikit-learn__scikit-learn-14141",
-    "sympy__sympy-15809",
-    "sympy__sympy-20438",
-    "sympy__sympy-21930"
-  ],
-  "seed": 20260910,
-  "repetitions": 3,
-  "localArtifactSha512": "sha512-BIBg1hcrQLAvJJboiFxWv6lzWb3tamr5qO83fWEa9MyrhTkvTJ9FYy62iL+o2N+5Jx5Mj4GOICrX1NpGd827Wg==",
-  "artifactBoundary": "Local tarball inspected and installed separately. Holdout executes the hash-frozen source runtime, not the protected release artifact. No publication certification claimed.",
-  "runs": {
-    "openai": {
-      "model": "gpt-5.6-luna",
-      "reasoning": "low",
-      "transport": "openai-responses",
-      "limits": {
-        "steps": 24,
-        "outputPerTurn": 2048,
-        "inputTokens": 100000,
-        "outputTokens": 16000,
-        "timeoutSeconds": 300,
-        "memoryMb": 2048,
-        "cpus": 2
-      },
-      "report": {
-        "schemaVersion": 1,
-        "kind": "swebench-comparison",
-        "manifestSha256": "53d564ab2d4e608b9740c2292c9056f6d19ef789c9de7c1508c838b38ced8519",
-        "complete": true,
-        "candidates": {
-          "zhivex": {
-            "planned": 6,
-            "recorded": 6,
-            "resolved": 1,
-            "candidateCaptured": 6,
-            "candidateResolved": 3,
-            "candidateGraded": 6,
-            "resolutionRate": 0.16666666666666666,
-            "missing": 0,
-            "gradingFailures": 0,
-            "usageComplete": true,
-            "totalCostUsd": null,
-            "costPerResolvedUsd": null,
-            "resolvedP50Ms": 80594.56762496848
-          },
-          "mini-swe-agent": {
-            "planned": 6,
-            "recorded": 6,
-            "resolved": 4,
-            "candidateCaptured": 6,
-            "candidateResolved": 4,
-            "candidateGraded": 6,
-            "resolutionRate": 0.6666666666666666,
-            "missing": 0,
-            "gradingFailures": 0,
-            "usageComplete": true,
-            "totalCostUsd": null,
-            "costPerResolvedUsd": null,
-            "resolvedP50Ms": 50588.63154100254
-          }
-        },
-        "pairedTaskResolutionDifference": -0.5,
-        "pairedTaskBootstrap95": [
-          -0.6666666666666667,
-          -0.3333333333333333
-        ],
-        "evidenceBoundary": "Local matched-model comparison. Missing and failed runs remain in the denominator. Small-sample intervals are exploratory. No safety superiority is inferred from SWE-bench correctness."
-      },
-      "samples": [
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15128",
-          "repetition": 0,
-          "status": "failed",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 57917,
-          "outputTokens": 931,
-          "cachedInputTokens": 19808,
-          "usageComplete": true,
-          "failure": "WORK_TOKEN_BUDGET",
-          "repairStages": {
-            "candidatePresent": false,
-            "importedPatchPresent": false,
-            "verificationAttempted": false
-          },
-          "repairProgress": {
-            "repeatedResults": 1,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": false,
-            "phase": "explore",
-            "blockedPhaseCalls": 0,
-            "planRecorded": false
-          }
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15128",
-          "repetition": 0,
-          "status": "Submitted",
-          "resolved": true,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 76213,
-          "outputTokens": 1852,
-          "cachedInputTokens": 63396,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15128",
-          "repetition": 1,
-          "status": "Submitted",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 84642,
-          "outputTokens": 2508,
-          "cachedInputTokens": 72996,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15128",
-          "repetition": 1,
-          "status": "failed",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 60677,
-          "outputTokens": 776,
-          "cachedInputTokens": 23581,
-          "usageComplete": true,
-          "failure": "WORK_TOKEN_BUDGET",
-          "repairStages": {
-            "candidatePresent": false,
-            "importedPatchPresent": false,
-            "verificationAttempted": false
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": false,
-            "phase": "explore",
-            "blockedPhaseCalls": 0,
-            "planRecorded": false
-          }
-        },
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15128",
-          "repetition": 2,
-          "status": "failed",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 65281,
-          "outputTokens": 766,
-          "cachedInputTokens": 28295,
-          "usageComplete": true,
-          "failure": "WORK_TOKEN_BUDGET",
-          "repairStages": {
-            "candidatePresent": false,
-            "importedPatchPresent": false,
-            "verificationAttempted": false
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": false,
-            "phase": "explore",
-            "blockedPhaseCalls": 0,
-            "planRecorded": false
-          }
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15128",
-          "repetition": 2,
-          "status": "Submitted",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 79373,
-          "outputTokens": 1971,
-          "cachedInputTokens": 68544,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15851",
-          "repetition": 0,
-          "status": "Submitted",
-          "resolved": true,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 26367,
-          "outputTokens": 1195,
-          "cachedInputTokens": 20848,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15851",
-          "repetition": 0,
-          "status": "failed",
-          "resolved": false,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 72876,
-          "outputTokens": 1512,
-          "cachedInputTokens": 27236,
-          "usageComplete": true,
-          "failure": "EXECUTION_FAILED",
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": false,
-            "verificationAttempted": true
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": true,
-            "phase": "verify",
-            "blockedPhaseCalls": 0,
-            "planRecorded": true
-          }
-        },
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15851",
-          "repetition": 1,
-          "status": "completed",
-          "resolved": true,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 42840,
-          "outputTokens": 897,
-          "cachedInputTokens": 17868,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": true,
-            "verificationAttempted": true
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": false,
-            "phase": "verify",
-            "blockedPhaseCalls": 0,
-            "planRecorded": true
-          }
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15851",
-          "repetition": 1,
-          "status": "Submitted",
-          "resolved": true,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 14726,
-          "outputTokens": 989,
-          "cachedInputTokens": 12580,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15851",
-          "repetition": 2,
-          "status": "Submitted",
-          "resolved": true,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 25208,
-          "outputTokens": 1233,
-          "cachedInputTokens": 21065,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15851",
-          "repetition": 2,
-          "status": "failed",
-          "resolved": false,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 73076,
-          "outputTokens": 1801,
-          "cachedInputTokens": 28815,
-          "usageComplete": true,
-          "failure": "EXECUTION_FAILED",
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": false,
-            "verificationAttempted": true
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": true,
-            "phase": "verify",
-            "blockedPhaseCalls": 0,
-            "planRecorded": true
-          }
-        }
-      ]
-    },
-    "qwen": {
-      "model": "qwen3.8-flash",
-      "reasoning": "none",
-      "transport": "qwen-chat-completions",
-      "limits": {
-        "steps": 24,
-        "outputPerTurn": 2048,
-        "inputTokens": 100000,
-        "outputTokens": 16000,
-        "timeoutSeconds": 300,
-        "memoryMb": 2048,
-        "cpus": 2
-      },
-      "report": {
-        "schemaVersion": 1,
-        "kind": "swebench-comparison",
-        "manifestSha256": "70e9d46277c326551ad59827de191eebcc8cc203493e3b05ef8f188ea6523e5a",
-        "complete": true,
-        "candidates": {
-          "zhivex": {
-            "planned": 6,
-            "recorded": 6,
-            "resolved": 0,
-            "candidateCaptured": 6,
-            "candidateResolved": 2,
-            "candidateGraded": 6,
-            "resolutionRate": 0.0,
-            "missing": 0,
-            "gradingFailures": 0,
-            "usageComplete": true,
-            "totalCostUsd": null,
-            "costPerResolvedUsd": null,
-            "resolvedP50Ms": null
-          },
-          "mini-swe-agent": {
-            "planned": 6,
-            "recorded": 6,
-            "resolved": 3,
-            "candidateCaptured": 6,
-            "candidateResolved": 3,
-            "candidateGraded": 6,
-            "resolutionRate": 0.5,
-            "missing": 0,
-            "gradingFailures": 0,
-            "usageComplete": true,
-            "totalCostUsd": null,
-            "costPerResolvedUsd": null,
-            "resolvedP50Ms": 55649.54887499334
-          }
-        },
-        "pairedTaskResolutionDifference": -0.5,
-        "pairedTaskBootstrap95": [
-          -1.0,
-          0.0
-        ],
-        "evidenceBoundary": "Local matched-model comparison. Missing and failed runs remain in the denominator. Small-sample intervals are exploratory. No safety superiority is inferred from SWE-bench correctness."
-      },
-      "samples": [
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15128",
-          "repetition": 0,
-          "status": "completed",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 40710,
-          "outputTokens": 455,
-          "cachedInputTokens": 24576,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": false,
-            "importedPatchPresent": false,
-            "verificationAttempted": false
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": false,
-            "phase": "explore",
-            "blockedPhaseCalls": 0,
-            "planRecorded": false
-          }
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15128",
-          "repetition": 0,
-          "status": "LimitsExceeded",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 105777,
-          "outputTokens": 1616,
-          "cachedInputTokens": 89344,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": false,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15128",
-          "repetition": 1,
-          "status": "LimitsExceeded",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 110108,
-          "outputTokens": 2110,
-          "cachedInputTokens": 94208,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": false,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15128",
-          "repetition": 1,
-          "status": "failed",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 63362,
-          "outputTokens": 647,
-          "cachedInputTokens": 47232,
-          "usageComplete": true,
-          "failure": "WORK_TOKEN_BUDGET",
-          "repairStages": {
-            "candidatePresent": false,
-            "importedPatchPresent": false,
-            "verificationAttempted": false
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": false,
-            "phase": "explore",
-            "blockedPhaseCalls": 0,
-            "planRecorded": false
-          }
-        },
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15128",
-          "repetition": 2,
-          "status": "failed",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 62385,
-          "outputTokens": 635,
-          "cachedInputTokens": 43520,
-          "usageComplete": true,
-          "failure": "WORK_TOKEN_BUDGET",
-          "repairStages": {
-            "candidatePresent": false,
-            "importedPatchPresent": false,
-            "verificationAttempted": false
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": false,
-            "phase": "explore",
-            "blockedPhaseCalls": 0,
-            "planRecorded": false
-          }
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15128",
-          "repetition": 2,
-          "status": "LimitsExceeded",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 102448,
-          "outputTokens": 1358,
-          "cachedInputTokens": 82944,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": false,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15851",
-          "repetition": 0,
-          "status": "Submitted",
-          "resolved": true,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 23132,
-          "outputTokens": 686,
-          "cachedInputTokens": 16384,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15851",
-          "repetition": 0,
-          "status": "failed",
-          "resolved": false,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 65852,
-          "outputTokens": 1137,
-          "cachedInputTokens": 44032,
-          "usageComplete": true,
-          "failure": "AGENT_FAILED",
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": false,
-            "verificationAttempted": true
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": false,
-            "phase": "verify",
-            "blockedPhaseCalls": 0,
-            "planRecorded": true
-          }
-        },
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15851",
-          "repetition": 1,
-          "status": "failed",
-          "resolved": false,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 98158,
-          "outputTokens": 1084,
-          "cachedInputTokens": 75776,
-          "usageComplete": true,
-          "failure": "INPUT_TOKEN_BUDGET",
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": false,
-            "verificationAttempted": true
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": true,
-            "phase": "verify",
-            "blockedPhaseCalls": 1,
-            "planRecorded": true
-          }
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15851",
-          "repetition": 1,
-          "status": "Submitted",
-          "resolved": true,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 23878,
-          "outputTokens": 745,
-          "cachedInputTokens": 17408,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "mini-swe-agent",
-          "instanceId": "django__django-15851",
-          "repetition": 2,
-          "status": "Submitted",
-          "resolved": true,
-          "candidateResolved": true,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 27959,
-          "outputTokens": 738,
-          "cachedInputTokens": 20736,
-          "usageComplete": true,
-          "failure": null,
-          "repairStages": {
-            "candidatePresent": true,
-            "importedPatchPresent": null,
-            "verificationAttempted": null
-          },
-          "repairProgress": null
-        },
-        {
-          "candidate": "zhivex",
-          "instanceId": "django__django-15851",
-          "repetition": 2,
-          "status": "failed",
-          "resolved": false,
-          "candidateResolved": false,
-          "candidateCaptured": true,
-          "gradingStatus": "completed",
-          "candidateGradingStatus": "completed",
-          "inputTokens": 62918,
-          "outputTokens": 759,
-          "cachedInputTokens": 48128,
-          "usageComplete": true,
-          "failure": "WORK_TOKEN_BUDGET",
-          "repairStages": {
-            "candidatePresent": false,
-            "importedPatchPresent": false,
-            "verificationAttempted": false
-          },
-          "repairProgress": {
-            "repeatedResults": 0,
-            "suppressedResults": 0,
-            "blockedBroadCalls": 0,
-            "enteredClosure": false,
-            "phase": "explore",
-            "blockedPhaseCalls": 0,
-            "planRecorded": true
-          }
-        }
-      ]
-    }
-  },
-  "limitations": [
-    "Two new tasks, both Django; not a general performance estimate.",
-    "Providers ran concurrently on one host; timing is not a controlled comparison.",
-    "Model, reasoning and transport vary across providers.",
-    "No before/after controller ablation was run.",
-    "No supplied prices; no monetary cost claim.",
-    "Correct private candidates are diagnostic only; only independently accepted delivered patches count."
-  ]
-}