@zhivex-ai/harness 1.0.0-rc.7 → 1.0.0

package/benchmarks/baselines/external-qwen-flash-2026-09-08.json

@@ -0,0 +1,1570 @@
+{
+  "schemaVersion": 1,
+  "dataset": "princeton-nlp/SWE-bench_Verified",
+  "revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a",
+  "model": "qwen3.8-flash",
+  "provider": "qwen",
+  "reasoning": "none",
+  "transport": "qwen-chat-completions",
+  "limits": {
+    "steps": 24,
+    "outputPerTurn": 2048,
+    "inputTokens": 100000,
+    "outputTokens": 16000,
+    "timeoutSeconds": 300,
+    "memoryMb": 2048,
+    "cpus": 2
+  },
+  "seed": 20260910,
+  "repetitions": 1,
+  "versions": {
+    "mini-swe-agent": "2.4.6",
+    "swebench": "4.1.0"
+  },
+  "implementation": {
+    "gitCommit": "0a7b69dddaa70a54b6b5b4097691c8112d4d76eb",
+    "sources": {
+      "bun.lock": "d5436a51773d94333612c611a162a3e63765fec88d6bbde2a71879662b49b5cd",
+      "evaluations/external/requirements.lock": "42e90a08f10f821ebb958af79256f707a9fb5cf365277c4f9e2715d612028179",
+      "package.json": "ade3c3a4c0f8ee193c560637bf96f1fe2782f5bc06a42dcdbb320bd7a932a173",
+      "scripts/benchmark-swebench.ts": "ed17063e1847017ba2e72f14732b6ff8f5b1af7cd5253b91302af75f2a9e8476",
+      "scripts/swebench/common.py": "c19cfc30b85dc3ad8dbf1ba6a4165e72ce96f624eb1721b613cfa3207636f7be",
+      "scripts/swebench/mini_driver.py": "bd0a77221878370ec44760122a51f07fb56d969612afa9a7f171248b2958baa6",
+      "scripts/swebench/model-budget.ts": "67b74d45d34235586773b7cf0db4d320306b14a6f432d53c084f80563ccdfa44",
+      "scripts/swebench/preflight.py": "aed092a75dd52f504fba6d14814b69bd627ff17d0e311a1f1279f8b613b69675",
+      "scripts/swebench/prepare.py": "bb7edd3fba5dbbe26597c97d744c60b94dbb7bcdfd7740a4a058c8583bd27e4c",
+      "scripts/swebench/report.py": "70afcdf0d1f081014bb75d878f3ddf9195959e96b523ddf1e0ad5d3ec8273549",
+      "scripts/swebench/run.py": "b184258e7c32313c9c801b50d28d1a9419d0bd2b077a422012d8fc6bc449c916",
+      "scripts/swebench/telemetry.ts": "4c6c0d58641b91539d945f341b368a9c360b6ef89ff3abb9d2f33d3cb133c12e",
+      "scripts/swebench/test_contracts.py": "a8c1348f1dff2e563c139e6d3e83eb141b4f2a71f91a58e2c5c051ed2a431024",
+      "scripts/swebench/zhivex-driver.ts": "7f9b9903f7e80075c4b084b5009f9fc3f8915cb9276f821ab9cb883c196e0fcb",
+      "scripts/time-to-safe-fix-efficiency.ts": "dd20fd8839280332ae4ca80d6846e98e07ff15d79e200b206fa2ba6be2724d2f",
+      "src/bounded-reads.ts": "9807bb460894be3279c5425af19863d1c6e419a1582ba6910ec7e7fc35703ed6",
+      "src/capabilities.ts": "56fdb60a2f6e1d33b2dc26d38b6d2709807e491ba1268d47b97876c2c0591d65",
+      "src/change-envelope.ts": "9047ec978c9e5117965b8093ec961d855f97bbc2c3accdc79b74cfe6a0e4f01e",
+      "src/cli-options.ts": "97781386289e3877c1318e4c94698ca7aaf2f256b89534dfcb21e966eb7d939f",
+      "src/cli-profiles.ts": "4fe451f1645e95f4957597a8724fa989b3c661ae1693252bfd6e74a51d20fc74",
+      "src/cli-stream.ts": "5ab8a4b1b8cf0a3dfa7b5b1f9e4e0facf8f6e7f6c341bdf700a41dc8927c825e",
+      "src/cli.ts": "cce6f58dfbc543acdc029b188d4394cd059d940bdd361351fd877ae2d6e94880",
+      "src/compaction.ts": "0c4f95d2f49c5154c9b87b3f79feca18227e274a8aacd2445e387a857a37853f",
+      "src/config.ts": "c160faaf2d9fb637c5a08a7aff65d761a12c707ed23ebb87ab0e8519e9536bf3",
+      "src/console-context.ts": "0ad7e4559504b9ed09ab500d615dea09af1075d357f02ec3b77f003d55d951db",
+      "src/console-input.ts": "dc8ea0cdbba1364db288d5156c5ff9819362c46b37ef35a98b0475b6c445b1f8",
+      "src/context-engineering.ts": "1995e564f6b91974e75bf66dd421ddcbea29b1f8eee59b76eb4e34c9d6d8f47e",
+      "src/edit-contracts.ts": "47cc3132e27f8749d54a32bbcc33bb174cf4448f0d0171b2578cdb559cf2176b",
+      "src/errors.ts": "e02f4eb80562a83762f908bda56c950ec6a2b78725c0c8a49f41fee0057ab903",
+      "src/execution-environment.ts": "8519e532dfafe4ae51bfbc81e0732267bc6aa9eec871ffde8c917aa001e358e5",
+      "src/file-security.ts": "4590b46c8e4b9d3891a75cd0d0ada14e0ae7b58e8e4bdf82f68df688a1fbb750",
+      "src/harness.ts": "00b45dfce74fd954d72e7aa5b53fe93e8983a63813f50201f4482ce3768de306",
+      "src/index.ts": "f791311f14dc27105191d97bb5454748d10beab827da9aba3ea949ffc32fce7c",
+      "src/json-contracts.ts": "0702e5b0ff16ff6304c74b4101bd31a3fa8eabdebe267d312b56d3871a7e302c",
+      "src/mcp.ts": "5f3f7a35b89063264ae18ab9273b84e2e59ac8c8e611a6b64be377ba61cc1ac5",
+      "src/operations.ts": "0775da79208c6f5d18476cc76404e84ddda610979d9d0e5dc15035aa6999613c",
+      "src/orchestration.ts": "7383da1047363819e6560f51892fd4082e5337856a3d889cdc076f021273774e",
+      "src/package-manager.ts": "cec4666bbab2f42e0a269709631f79f9f87c2c4dbb3cafeffacfaff761069e2c",
+      "src/process-runtime.ts": "b7286731a0b5c96dcad0b8e3a4bf1155d0539d73303cd7ebe77ffabf858f1729",
+      "src/providers.ts": "70ccfc7f315cd7238e83d69b719ce4908f3b620c564cbb4f00f93e102e025602",
+      "src/replacement-edits.ts": "8a0efdd3a7a9fc3e5a5475916cbd17ef882265881bb61076db58673e80b72957",
+      "src/routing.ts": "8e7c9f5968b4bd638cbbc86b19847ca183f3418b9a298783b96a88cffa568e22",
+      "src/run-interruption.ts": "df99129cf2dd6b938eca629cb897c4350a7d57fe6bf0ce437f5f3f30271617df",
+      "src/sessions.ts": "5167c8cf36bd4b8845ee278883394700321b65397f49c1dda0cf0d0de09dd06e",
+      "src/sqlite-database.ts": "18765d1ca0c2f2ad87bfd82d311312dfe8a42006556f27bc21da5892ed3b044e",
+      "src/state-backup.ts": "b6f8f1aedeb5dce31b65c23c7ffc50030870655b58a5aaaef9989bafb9da5c68",
+      "src/state-directory.ts": "3711550991951e6b738a08deaa0e06da84d01864c03a9602600f592963fa3211",
+      "src/terminal-markdown.ts": "8cffb6ebaf25108cb2bee04956d9b2973635b010f5d7ce6be21b3ceffe3e8bfd",
+      "src/terminal-ui.ts": "e50025a8649d1f60efdf3af895b8639730ec2e86553fb80a61a776b11076c705",
+      "src/time-to-safe-fix.ts": "4add1d49c0ea209cc496c66fa2b2012dc7bcab87e678a58c8591b3c2cdc8df8a",
+      "src/version.ts": "b2c2f80a40adc08e8258b81e82434273f0bdfa787528b2221f14efe7f06aa747",
+      "src/workspace.ts": "e9c2ed5c950d751d0f93a5512584d0698449b551d3270b3e115ccca380565222"
+    }
+  },
+  "host": {
+    "system": "Darwin",
+    "architecture": "arm64",
+    "python": "3.12.13",
+    "docker": "29.7.2"
+  },
+  "environmentDifferences": [
+    "Zhivex native OCI: read-only root and writable /workspace snapshot, operator auto-approval; mini: writable /testbed. Both deny container network and use the same derived image and CPU/memory limits.",
+    "Zhivex tool-call limit is steps times eight; mini counts model steps and executes each response's actions. Tool counts are reported, not claimed equivalent.",
+    "Input/output token ceilings are checked between model requests and can overshoot by one response; outputPerTurn is sent to both APIs."
+  ],
+  "approvalPolicy": "Automated operator approves supported isolated Zhivex tools; successful approved verification and patch import is a terminal receipt. Human wait is zero. Not a human-UX comparison.",
+  "excludedTaskIds": [
+    "astropy__astropy-14995",
+    "django__django-12262",
+    "django__django-13410",
+    "django__django-14915",
+    "django__django-15569",
+    "pydata__xarray-4695",
+    "pydata__xarray-6721",
+    "pytest-dev__pytest-8399",
+    "sympy__sympy-15809",
+    "sympy__sympy-20438"
+  ],
+  "excludedManifestSha256": [
+    "4902585186402c9fdd3e10db9c537109b5a108ba46b4e0806e259f6a8c4c15ab",
+    "76739783c68673b6f659d20f757a191a52213add63e46706737535484aec6f41"
+  ],
+  "date": "2026-09-08",
+  "kind": "swebench-qwen-flash-development-comparison",
+  "tasks": [
+    {
+      "instance_id": "sympy__sympy-21930",
+      "repo": "sympy/sympy",
+      "base_commit": "de446c6d85f633271dfec1452f6f28ea783e293f",
+      "evalImage": "swebench/sweb.eval.x86_64.sympy_1776_sympy-21930:zhx-ac9b1c65d632e5a3",
+      "evalImageId": "sha256:ac9b1c65d632e5a3d1af240db144ff27ed9a70b5507705d50480ede6899125d1",
+      "evalTag": "zhx-ac9b1c65d632e5a3",
+      "image": "sha256:81c7bb0b9b74ae262830b7d4b94d042ab60c10cdbe51620c3b387262c10794fc",
+      "checkoutHead": "b7d17e72736ba245d24b87f8f5e8c80565066de4",
+      "baseTree": "a8e44f94a63624d1f4af136f7382125a0794eecf",
+      "upstreamTree": "a8e44f94a63624d1f4af136f7382125a0794eecf",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "scikit-learn__scikit-learn-10908",
+      "repo": "scikit-learn/scikit-learn",
+      "base_commit": "67d06b18c68ee4452768f8a1e868565dd4354abf",
+      "evalImage": "swebench/sweb.eval.x86_64.scikit-learn_1776_scikit-learn-10908:zhx-8727c0f7bfb76c29",
+      "evalImageId": "sha256:8727c0f7bfb76c294cfef16c9fe3d0dc9ddf313e7c0fb3e8e9cbc74c4955404c",
+      "evalTag": "zhx-8727c0f7bfb76c29",
+      "image": "sha256:4da1cd215c9209495a9aa8638026733275c766ab970311d31b1b6a8f834687d8",
+      "checkoutHead": "de60d3cc1d489509f0d7fe1ad141171628d3d4b7",
+      "baseTree": "d7e9da7c13c10167281372f397c7313c52c46077",
+      "upstreamTree": "d7e9da7c13c10167281372f397c7313c52c46077",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "django__django-15554",
+      "repo": "django/django",
+      "base_commit": "59ab3fd0e9e606d7f0f7ca26609c06ee679ece97",
+      "evalImage": "swebench/sweb.eval.x86_64.django_1776_django-15554:zhx-9e1bc1db0e1ec55a",
+      "evalImageId": "sha256:9e1bc1db0e1ec55a32f346c36671df7384a51a9ed84eed9b7e839e1405f04292",
+      "evalTag": "zhx-9e1bc1db0e1ec55a",
+      "image": "sha256:05b0c262568607baf8716c78d6bef059bf4a57a219141c6877034ee1e90a223d",
+      "checkoutHead": "292b4e462f63a7dfa0f9cd742e14cfa0b43727d6",
+      "baseTree": "9014f24e170c210492e65fe54bcbf236e7342c4a",
+      "upstreamTree": "9014f24e170c210492e65fe54bcbf236e7342c4a",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "scikit-learn__scikit-learn-14141",
+      "repo": "scikit-learn/scikit-learn",
+      "base_commit": "3d997697fdd166eff428ea9fd35734b6a8ba113e",
+      "evalImage": "swebench/sweb.eval.x86_64.scikit-learn_1776_scikit-learn-14141:zhx-c0f6a733154dad51",
+      "evalImageId": "sha256:c0f6a733154dad51cb8f617ca2ea09d095a6f56a1576740c2d43f9020294f2e0",
+      "evalTag": "zhx-c0f6a733154dad51",
+      "image": "sha256:a24afa6604e714bb6cf031b045f171e8b883e876534f941bb10ee36854612d6d",
+      "checkoutHead": "f863dd9c48aee1eb395e5ca64e675a765cea44a9",
+      "baseTree": "74477c0718711537b2cb5f9805ffbf9ffd072312",
+      "upstreamTree": "74477c0718711537b2cb5f9805ffbf9ffd072312",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "django__django-11265",
+      "repo": "django/django",
+      "base_commit": "21aa2a5e785eef1f47beb1c3760fdd7d8915ae09",
+      "evalImage": "swebench/sweb.eval.x86_64.django_1776_django-11265:zhx-70a39003fb3ce8f0",
+      "evalImageId": "sha256:70a39003fb3ce8f01ee82dfeb16b09bc3714ab669d3743fb3e1e5689f18ea1ef",
+      "evalTag": "zhx-70a39003fb3ce8f0",
+      "image": "sha256:32bc939face8eeae33506b342f5a260ab00cc49d02b616c58d7ef92b20389532",
+      "checkoutHead": "ad2b3670a78c6db994d93f1b71bccd5d5ee4b9c4",
+      "baseTree": "cbaf76226fd9f842b01da4d2cd52d99d2d256be2",
+      "upstreamTree": "cbaf76226fd9f842b01da4d2cd52d99d2d256be2",
+      "imageModeDifferences": 0
+    }
+  ],
+  "report": {
+    "schemaVersion": 1,
+    "kind": "swebench-comparison",
+    "manifestSha256": "fa943e995a1d4155c372bdaa66f09ffd1e64822ad13b0d4a8430e40522b16f7e",
+    "complete": true,
+    "candidates": {
+      "zhivex": {
+        "planned": 5,
+        "recorded": 5,
+        "resolved": 0,
+        "resolutionRate": 0.0,
+        "missing": 0,
+        "gradingFailures": 0,
+        "usageComplete": true,
+        "totalCostUsd": null,
+        "costPerResolvedUsd": null,
+        "resolvedP50Ms": null
+      },
+      "mini-swe-agent": {
+        "planned": 5,
+        "recorded": 5,
+        "resolved": 2,
+        "resolutionRate": 0.4,
+        "missing": 0,
+        "gradingFailures": 0,
+        "usageComplete": true,
+        "totalCostUsd": null,
+        "costPerResolvedUsd": null,
+        "resolvedP50Ms": 40939.1438330058
+      }
+    },
+    "pairedTaskResolutionDifference": -0.4,
+    "pairedTaskBootstrap95": [
+      -0.8,
+      0.0
+    ],
+    "evidenceBoundary": "Local matched-model comparison. Missing and failed runs remain in the denominator. Small-sample intervals are exploratory. No safety superiority is inferred from SWE-bench correctness."
+  },
+  "samples": [
+    {
+      "candidate": "zhivex",
+      "instanceId": "sympy__sympy-21930",
+      "repetition": 0,
+      "resolved": false,
+      "gradingStatus": "completed",
+      "usageComplete": true,
+      "costUsd": null,
+      "status": "failed",
+      "failure": "INPUT_TOKEN_BUDGET",
+      "durationMs": 38459.725333,
+      "turns": [
+        {
+          "index": 1,
+          "status": "completed",
+          "inputTokens": 3237,
+          "outputTokens": 161,
+          "cachedInputTokens": 0,
+          "requestCharacters": 4987,
+          "tools": [
+            {
+              "name": "list_files",
+              "isError": true,
+              "characters": 4,
+              "diagnostic": {
+                "code": "EXECUTION_FAILED",
+                "category": "execution",
+                "retryable": false,
+                "chain": [
+                  {
+                    "providerReason": null,
+                    "validationIssues": [],
+                    "status": null,
+                    "errorClass": "Object",
+                    "providerHints": [],
+                    "fingerprint": "eb0264922019f3f1e60155c5264772fb2123344f17f7cded6b62a106356a4dca",
+                    "reason": null
+                  }
+                ]
+              }
+            },
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 161
+            }
+          ]
+        },
+        {
+          "index": 2,
+          "status": "completed",
+          "inputTokens": 3472,
+          "outputTokens": 55,
+          "cachedInputTokens": 3072,
+          "requestCharacters": 1105,
+          "tools": [
+            {
+              "name": "list_files",
+              "isError": false,
+              "characters": 3297
+            }
+          ]
+        },
+        {
+          "index": 3,
+          "status": "completed",
+          "inputTokens": 4610,
+          "outputTokens": 84,
+          "cachedInputTokens": 3328,
+          "requestCharacters": 3646,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 7904
+            }
+          ]
+        },
+        {
+          "index": 4,
+          "status": "completed",
+          "inputTokens": 8861,
+          "outputTokens": 255,
+          "cachedInputTokens": 4096,
+          "requestCharacters": 8349,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 8315
+            }
+          ]
+        },
+        {
+          "index": 5,
+          "status": "completed",
+          "inputTokens": 12551,
+          "outputTokens": 114,
+          "cachedInputTokens": 8320,
+          "requestCharacters": 9113,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 740
+            }
+          ]
+        },
+        {
+          "index": 6,
+          "status": "completed",
+          "inputTokens": 13021,
+          "outputTokens": 89,
+          "cachedInputTokens": 12288,
+          "requestCharacters": 1254,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 6612
+            }
+          ]
+        },
+        {
+          "index": 7,
+          "status": "waiting_approval",
+          "inputTokens": 15447,
+          "outputTokens": 141,
+          "cachedInputTokens": 12288,
+          "requestCharacters": 7046,
+          "tools": []
+        },
+        {
+          "index": 8,
+          "status": "waiting_approval",
+          "inputTokens": 3762,
+          "outputTokens": 206,
+          "cachedInputTokens": 2048,
+          "requestCharacters": 7519,
+          "tools": [
+            {
+              "name": "run_environment_shell",
+              "isError": false,
+              "characters": 745,
+              "exitCode": 128,
+              "timedOut": false
+            },
+            {
+              "name": "run_environment_shell",
+              "isError": false,
+              "characters": 1034,
+              "exitCode": 0,
+              "timedOut": false
+            }
+          ]
+        },
+        {
+          "index": 9,
+          "status": "completed",
+          "inputTokens": 4403,
+          "outputTokens": 133,
+          "cachedInputTokens": 3328,
+          "requestCharacters": 332,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 11910
+            }
+          ]
+        },
+        {
+          "index": 10,
+          "status": "completed",
+          "inputTokens": 10675,
+          "outputTokens": 85,
+          "cachedInputTokens": 4096,
+          "requestCharacters": 21880,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 9900
+            }
+          ]
+        },
+        {
+          "index": 11,
+          "status": "completed",
+          "inputTokens": 14094,
+          "outputTokens": 108,
+          "cachedInputTokens": 10240,
+          "requestCharacters": 32213,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 266
+            }
+          ]
+        },
+        {
+          "index": 12,
+          "status": "completed",
+          "inputTokens": 14288,
+          "outputTokens": 98,
+          "cachedInputTokens": 13312,
+          "requestCharacters": 33000,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 5179
+            }
+          ]
+        }
+      ],
+      "inputTokens": 108421,
+      "outputTokens": 1529,
+      "cachedInputTokens": 76416,
+      "modelCalls": 12,
+      "compactions": [
+        {
+          "beforeStep": 8,
+          "estimatedTokensBefore": 9221,
+          "estimatedTokensAfter": 1797
+        }
+      ],
+      "terminalDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "Object",
+            "providerHints": [],
+            "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
+            "reason": "budget"
+          }
+        ]
+      },
+      "thrownDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "HarnessExecutionError",
+            "providerHints": [],
+            "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
+            "reason": "budget"
+          },
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "Error",
+            "providerHints": [],
+            "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
+            "reason": "budget"
+          }
+        ]
+      },
+      "guardrails": [],
+      "toolCalls": 13,
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "protectedFilesChanged": [],
+      "gradingMethod": "empty-patch-no-solution",
+      "officialResolved": false,
+      "gradingDurationMs": 0.004791014362126589,
+      "totalDurationMs": 40163.06795802666
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "sympy__sympy-21930",
+      "repetition": 0,
+      "resolved": false,
+      "gradingStatus": "completed",
+      "usageComplete": true,
+      "costUsd": null,
+      "status": "LimitsExceeded",
+      "failure": null,
+      "durationMs": 60712.348208006006,
+      "toolCalls": 21,
+      "inputTokens": 103566,
+      "outputTokens": 2628,
+      "cachedInputTokens": 87168,
+      "modelCalls": 21,
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "protectedFilesChanged": [],
+      "gradingMethod": "empty-patch-no-solution",
+      "officialResolved": false,
+      "gradingDurationMs": 0.004333036486059427,
+      "totalDurationMs": 62332.5775419944
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "scikit-learn__scikit-learn-10908",
+      "repetition": 0,
+      "resolved": true,
+      "gradingStatus": "completed",
+      "usageComplete": true,
+      "costUsd": null,
+      "status": "Submitted",
+      "failure": null,
+      "durationMs": 25978.861292009242,
+      "toolCalls": 9,
+      "inputTokens": 30440,
+      "outputTokens": 948,
+      "cachedInputTokens": 24704,
+      "modelCalls": 9,
+      "patchSha256": "5bbbc8e6f3953adf89c65636a8d6e75973081d8a28cbf3264485fabeb672de11",
+      "protectedFilesChanged": [],
+      "officialResolved": true,
+      "gradingDurationMs": 20167.721583973616,
+      "totalDurationMs": 47685.66404195735
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "scikit-learn__scikit-learn-10908",
+      "repetition": 0,
+      "resolved": false,
+      "gradingStatus": "completed",
+      "usageComplete": true,
+      "costUsd": null,
+      "status": "failed",
+      "failure": "EXECUTION_FAILED",
+      "durationMs": 14930.337209,
+      "turns": [
+        {
+          "index": 1,
+          "status": "completed",
+          "inputTokens": 3555,
+          "outputTokens": 151,
+          "cachedInputTokens": 2048,
+          "requestCharacters": 2,
+          "tools": [
+            {
+              "name": "list_files",
+              "isError": false,
+              "characters": 2136
+            },
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 305
+            }
+          ]
+        },
+        {
+          "index": 2,
+          "status": "completed",
+          "inputTokens": 4507,
+          "outputTokens": 207,
+          "cachedInputTokens": 3328,
+          "requestCharacters": 2,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 7651
+            },
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 1196
+            }
+          ]
+        },
+        {
+          "index": 3,
+          "status": "completed",
+          "inputTokens": 7812,
+          "outputTokens": 86,
+          "cachedInputTokens": 4096,
+          "requestCharacters": 2,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 6930
+            }
+          ]
+        },
+        {
+          "index": 4,
+          "status": "completed",
+          "inputTokens": 10040,
+          "outputTokens": 113,
+          "cachedInputTokens": 7168,
+          "requestCharacters": 2,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 3123
+            }
+          ]
+        }
+      ],
+      "inputTokens": 37456,
+      "outputTokens": 661,
+      "cachedInputTokens": 26624,
+      "modelCalls": 5,
+      "compactions": [],
+      "terminalDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [
+              {
+                "code": "invalid_type",
+                "path": [
+                  "files",
+                  0,
+                  "endLine"
+                ]
+              }
+            ],
+            "status": null,
+            "errorClass": "Object",
+            "providerHints": [],
+            "fingerprint": "5df6c5850a252365ead3f2df4e2d069515e75359a797e6a0a64c1a1cce54424b",
+            "reason": "invalid input"
+          }
+        ]
+      },
+      "thrownDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [
+              {
+                "code": "invalid_type",
+                "path": [
+                  "files",
+                  0,
+                  "endLine"
+                ]
+              }
+            ],
+            "status": null,
+            "errorClass": "HarnessExecutionError",
+            "providerHints": [],
+            "fingerprint": "5df6c5850a252365ead3f2df4e2d069515e75359a797e6a0a64c1a1cce54424b",
+            "reason": "invalid input"
+          },
+          {
+            "providerReason": null,
+            "validationIssues": [
+              {
+                "code": "invalid_type",
+                "path": [
+                  "files",
+                  0,
+                  "endLine"
+                ]
+              }
+            ],
+            "status": null,
+            "errorClass": "ValidationError",
+            "providerHints": [],
+            "fingerprint": "5df6c5850a252365ead3f2df4e2d069515e75359a797e6a0a64c1a1cce54424b",
+            "reason": "invalid input"
+          }
+        ]
+      },
+      "guardrails": [],
+      "toolCalls": 6,
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "protectedFilesChanged": [],
+      "gradingMethod": "empty-patch-no-solution",
+      "officialResolved": false,
+      "gradingDurationMs": 0.004916044417768717,
+      "totalDurationMs": 16330.313333019149
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "django__django-15554",
+      "repetition": 0,
+      "resolved": false,
+      "gradingStatus": "completed",
+      "usageComplete": true,
+      "costUsd": null,
+      "status": "failed",
+      "failure": "INPUT_TOKEN_BUDGET",
+      "durationMs": 53471.117042000005,
+      "turns": [
+        {
+          "index": 1,
+          "status": "completed",
+          "inputTokens": 3377,
+          "outputTokens": 126,
+          "cachedInputTokens": 2048,
+          "requestCharacters": 2,
+          "tools": [
+            {
+              "name": "list_files",
+              "isError": false,
+              "characters": 2164
+            },
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 982
+            }
+          ]
+        },
+        {
+          "index": 2,
+          "status": "completed",
+          "inputTokens": 4589,
+          "outputTokens": 56,
+          "cachedInputTokens": 3072,
+          "requestCharacters": 2,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 7479
+            }
+          ]
+        },
+        {
+          "index": 3,
+          "status": "completed",
+          "inputTokens": 6829,
+          "outputTokens": 56,
+          "cachedInputTokens": 4096,
+          "requestCharacters": 2,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 8009
+            }
+          ]
+        },
+        {
+          "index": 4,
+          "status": "completed",
+          "inputTokens": 9105,
+          "outputTokens": 56,
+          "cachedInputTokens": 6656,
+          "requestCharacters": 2,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 4044
+            }
+          ]
+        },
+        {
+          "index": 5,
+          "status": "completed",
+          "inputTokens": 10391,
+          "outputTokens": 113,
+          "cachedInputTokens": 8320,
+          "requestCharacters": 2,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 2995
+            }
+          ]
+        },
+        {
+          "index": 6,
+          "status": "completed",
+          "inputTokens": 11885,
+          "outputTokens": 54,
+          "cachedInputTokens": 10240,
+          "requestCharacters": 2,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 7791
+            }
+          ]
+        },
+        {
+          "index": 7,
+          "status": "completed",
+          "inputTokens": 5752,
+          "outputTokens": 83,
+          "cachedInputTokens": 2048,
+          "requestCharacters": 13872,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 8545
+            }
+          ]
+        },
+        {
+          "index": 8,
+          "status": "completed",
+          "inputTokens": 9864,
+          "outputTokens": 87,
+          "cachedInputTokens": 5120,
+          "requestCharacters": 332,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 7764
+            }
+          ]
+        },
+        {
+          "index": 9,
+          "status": "completed",
+          "inputTokens": 12349,
+          "outputTokens": 87,
+          "cachedInputTokens": 9216,
+          "requestCharacters": 31423,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 4380
+            }
+          ]
+        },
+        {
+          "index": 10,
+          "status": "completed",
+          "inputTokens": 13874,
+          "outputTokens": 100,
+          "cachedInputTokens": 11264,
+          "requestCharacters": 36250,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 823
+            }
+          ]
+        },
+        {
+          "index": 11,
+          "status": "completed",
+          "inputTokens": 14350,
+          "outputTokens": 56,
+          "cachedInputTokens": 13312,
+          "requestCharacters": 37583,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 3120
+            }
+          ]
+        }
+      ],
+      "inputTokens": 102365,
+      "outputTokens": 874,
+      "cachedInputTokens": 75392,
+      "modelCalls": 11,
+      "compactions": [
+        {
+          "beforeStep": 7,
+          "estimatedTokensBefore": 10476,
+          "estimatedTokensAfter": 3468
+        },
+        {
+          "beforeStep": 12,
+          "estimatedTokensBefore": 10187,
+          "estimatedTokensAfter": 2309
+        }
+      ],
+      "terminalDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "Object",
+            "providerHints": [],
+            "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
+            "reason": "budget"
+          }
+        ]
+      },
+      "thrownDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "HarnessExecutionError",
+            "providerHints": [],
+            "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
+            "reason": "budget"
+          },
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "Error",
+            "providerHints": [],
+            "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
+            "reason": "budget"
+          }
+        ]
+      },
+      "guardrails": [],
+      "toolCalls": 12,
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "protectedFilesChanged": [],
+      "gradingMethod": "empty-patch-no-solution",
+      "officialResolved": false,
+      "gradingDurationMs": 0.006249989382922649,
+      "totalDurationMs": 58832.29600003688
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "django__django-15554",
+      "repetition": 0,
+      "resolved": false,
+      "gradingStatus": "completed",
+      "usageComplete": true,
+      "costUsd": null,
+      "status": "LimitsExceeded",
+      "failure": null,
+      "durationMs": 42561.118874989916,
+      "toolCalls": 16,
+      "inputTokens": 105486,
+      "outputTokens": 1510,
+      "cachedInputTokens": 86784,
+      "modelCalls": 16,
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "protectedFilesChanged": [],
+      "gradingMethod": "empty-patch-no-solution",
+      "officialResolved": false,
+      "gradingDurationMs": 0.002333021257072687,
+      "totalDurationMs": 44094.71937501803
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "scikit-learn__scikit-learn-14141",
+      "repetition": 0,
+      "resolved": true,
+      "gradingStatus": "completed",
+      "usageComplete": true,
+      "costUsd": null,
+      "status": "Submitted",
+      "failure": null,
+      "durationMs": 19550.653749960475,
+      "toolCalls": 6,
+      "inputTokens": 14157,
+      "outputTokens": 442,
+      "cachedInputTokens": 8192,
+      "modelCalls": 6,
+      "patchSha256": "b1ca3d45bf4e8d854c1921ac8f47ff519d0987ef33adc3c787e2a2c68bb2aba4",
+      "protectedFilesChanged": [],
+      "officialResolved": true,
+      "gradingDurationMs": 19867.57433298044,
+      "totalDurationMs": 40939.1438330058
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "scikit-learn__scikit-learn-14141",
+      "repetition": 0,
+      "resolved": false,
+      "gradingStatus": "completed",
+      "usageComplete": true,
+      "costUsd": null,
+      "status": "failed",
+      "failure": "EXECUTION_FAILED",
+      "durationMs": 38582.481708,
+      "turns": [
+        {
+          "index": 1,
+          "status": "completed",
+          "inputTokens": 3076,
+          "outputTokens": 132,
+          "cachedInputTokens": 2048,
+          "requestCharacters": 4396,
+          "tools": [
+            {
+              "name": "list_files",
+              "isError": false,
+              "characters": 2631
+            },
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 2714
+            }
+          ]
+        },
+        {
+          "index": 2,
+          "status": "completed",
+          "inputTokens": 5320,
+          "outputTokens": 60,
+          "cachedInputTokens": 3072,
+          "requestCharacters": 6143,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 7000
+            }
+          ]
+        },
+        {
+          "index": 3,
+          "status": "waiting_approval",
+          "inputTokens": 7907,
+          "outputTokens": 173,
+          "cachedInputTokens": 5120,
+          "requestCharacters": 7427,
+          "tools": []
+        },
+        {
+          "index": 4,
+          "status": "waiting_approval",
+          "inputTokens": 8858,
+          "outputTokens": 197,
+          "cachedInputTokens": 7168,
+          "requestCharacters": 20648,
+          "tools": [
+            {
+              "name": "run_environment_command",
+              "isError": false,
+              "characters": 394,
+              "exitCode": 0,
+              "timedOut": false
+            }
+          ]
+        },
+        {
+          "index": 5,
+          "status": "waiting_approval",
+          "inputTokens": 9362,
+          "outputTokens": 179,
+          "cachedInputTokens": 8320,
+          "requestCharacters": 21849,
+          "tools": [
+            {
+              "name": "apply_reviewed_replacement",
+              "isError": false,
+              "characters": 475
+            }
+          ]
+        },
+        {
+          "index": 6,
+          "status": "waiting_approval",
+          "inputTokens": 9857,
+          "outputTokens": 117,
+          "cachedInputTokens": 9216,
+          "requestCharacters": 22999,
+          "tools": [
+            {
+              "name": "apply_reviewed_replacement",
+              "isError": false,
+              "characters": 485
+            }
+          ]
+        },
+        {
+          "index": 7,
+          "status": "waiting_approval",
+          "inputTokens": 10513,
+          "outputTokens": 234,
+          "cachedInputTokens": 9216,
+          "requestCharacters": 25583,
+          "tools": [
+            {
+              "name": "run_environment_command",
+              "isError": false,
+              "characters": 1973,
+              "exitCode": 1,
+              "timedOut": false
+            }
+          ]
+        },
+        {
+          "index": 8,
+          "status": "waiting_approval",
+          "inputTokens": 3951,
+          "outputTokens": 177,
+          "cachedInputTokens": 2048,
+          "requestCharacters": 8556,
+          "tools": [
+            {
+              "name": "run_environment_command",
+              "isError": false,
+              "characters": 927,
+              "exitCode": 0,
+              "timedOut": false
+            }
+          ]
+        },
+        {
+          "index": 9,
+          "status": "completed",
+          "inputTokens": 4436,
+          "outputTokens": 15,
+          "cachedInputTokens": 3328,
+          "requestCharacters": 10261,
+          "tools": [
+            {
+              "name": "run_environment_command",
+              "isError": false,
+              "characters": 887,
+              "exitCode": 0,
+              "timedOut": false
+            }
+          ]
+        },
+        {
+          "index": 10,
+          "status": "waiting_approval",
+          "inputTokens": 4932,
+          "outputTokens": 133,
+          "cachedInputTokens": 4096,
+          "requestCharacters": 11378,
+          "tools": [
+            {
+              "name": "inspect_environment_patch",
+              "isError": false,
+              "characters": 795
+            }
+          ]
+        }
+      ],
+      "inputTokens": 68212,
+      "outputTokens": 1417,
+      "cachedInputTokens": 53632,
+      "modelCalls": 10,
+      "compactions": [
+        {
+          "beforeStep": 8,
+          "estimatedTokensBefore": 6988,
+          "estimatedTokensAfter": 2057
+        }
+      ],
+      "terminalDiagnostic": null,
+      "thrownDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": true,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "HarnessExecutionError",
+            "providerHints": [],
+            "fingerprint": "2d43809b162b8543195aa4f8f580e478018783c4e0baff12061c2cbd1496942f",
+            "reason": null
+          },
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "Error",
+            "providerHints": [],
+            "fingerprint": "2d43809b162b8543195aa4f8f580e478018783c4e0baff12061c2cbd1496942f",
+            "reason": null
+          }
+        ]
+      },
+      "guardrails": [],
+      "toolCalls": 13,
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "protectedFilesChanged": [],
+      "gradingMethod": "empty-patch-no-solution",
+      "officialResolved": false,
+      "gradingDurationMs": 0.005082983989268541,
+      "totalDurationMs": 40729.9125000136
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "django__django-11265",
+      "repetition": 0,
+      "resolved": false,
+      "gradingStatus": "completed",
+      "usageComplete": true,
+      "costUsd": null,
+      "status": "failed",
+      "failure": "INPUT_TOKEN_BUDGET",
+      "durationMs": 55047.484833,
+      "turns": [
+        {
+          "index": 1,
+          "status": "completed",
+          "inputTokens": 4137,
+          "outputTokens": 153,
+          "cachedInputTokens": 2048,
+          "requestCharacters": 8177,
+          "tools": [
+            {
+              "name": "list_files",
+              "isError": false,
+              "characters": 2793
+            },
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 400
+            }
+          ]
+        },
+        {
+          "index": 2,
+          "status": "completed",
+          "inputTokens": 5261,
+          "outputTokens": 154,
+          "cachedInputTokens": 4096,
+          "requestCharacters": 4075,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 13628
+            },
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 6613
+            }
+          ]
+        },
+        {
+          "index": 3,
+          "status": "completed",
+          "inputTokens": 13547,
+          "outputTokens": 183,
+          "cachedInputTokens": 5120,
+          "requestCharacters": 21101,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 690
+            },
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 8996
+            }
+          ]
+        },
+        {
+          "index": 4,
+          "status": "completed",
+          "inputTokens": 6392,
+          "outputTokens": 87,
+          "cachedInputTokens": 2048,
+          "requestCharacters": 16095,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 9639
+            }
+          ]
+        },
+        {
+          "index": 5,
+          "status": "completed",
+          "inputTokens": 9533,
+          "outputTokens": 83,
+          "cachedInputTokens": 6144,
+          "requestCharacters": 10087,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 2261
+            }
+          ]
+        },
+        {
+          "index": 6,
+          "status": "completed",
+          "inputTokens": 10623,
+          "outputTokens": 116,
+          "cachedInputTokens": 9216,
+          "requestCharacters": 2729,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 12569
+            }
+          ]
+        },
+        {
+          "index": 7,
+          "status": "completed",
+          "inputTokens": 6987,
+          "outputTokens": 87,
+          "cachedInputTokens": 3072,
+          "requestCharacters": 18880,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 10718
+            }
+          ]
+        },
+        {
+          "index": 8,
+          "status": "waiting_approval",
+          "inputTokens": 10330,
+          "outputTokens": 51,
+          "cachedInputTokens": 6144,
+          "requestCharacters": 11497,
+          "tools": []
+        },
+        {
+          "index": 9,
+          "status": "waiting_approval",
+          "inputTokens": 11580,
+          "outputTokens": 72,
+          "cachedInputTokens": 10240,
+          "requestCharacters": 35660,
+          "tools": [
+            {
+              "name": "run_environment_shell",
+              "isError": false,
+              "characters": 4893,
+              "exitCode": 0,
+              "timedOut": false
+            },
+            {
+              "name": "run_environment_shell",
+              "isError": false,
+              "characters": 5551,
+              "exitCode": 0,
+              "timedOut": false
+            }
+          ]
+        },
+        {
+          "index": 10,
+          "status": "completed",
+          "inputTokens": 4712,
+          "outputTokens": 95,
+          "cachedInputTokens": 3072,
+          "requestCharacters": 332,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 8694
+            }
+          ]
+        },
+        {
+          "index": 11,
+          "status": "completed",
+          "inputTokens": 7337,
+          "outputTokens": 98,
+          "cachedInputTokens": 4096,
+          "requestCharacters": 21403,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 1408
+            }
+          ]
+        },
+        {
+          "index": 12,
+          "status": "completed",
+          "inputTokens": 8031,
+          "outputTokens": 87,
+          "cachedInputTokens": 7168,
+          "requestCharacters": 23331,
+          "tools": [
+            {
+              "name": "read_files",
+              "isError": false,
+              "characters": 21195
+            }
+          ]
+        },
+        {
+          "index": 13,
+          "status": "completed",
+          "inputTokens": 9796,
+          "outputTokens": 121,
+          "cachedInputTokens": 3072,
+          "requestCharacters": 331,
+          "tools": [
+            {
+              "name": "search_many",
+              "isError": false,
+              "characters": 883
+            }
+          ]
+        }
+      ],
+      "inputTokens": 108266,
+      "outputTokens": 1387,
+      "cachedInputTokens": 65536,
+      "modelCalls": 13,
+      "compactions": [
+        {
+          "beforeStep": 4,
+          "estimatedTokensBefore": 10996,
+          "estimatedTokensAfter": 4024
+        },
+        {
+          "beforeStep": 7,
+          "estimatedTokensBefore": 10499,
+          "estimatedTokensAfter": 4720
+        },
+        {
+          "beforeStep": 10,
+          "estimatedTokensBefore": 10335,
+          "estimatedTokensAfter": 2968
+        },
+        {
+          "beforeStep": 13,
+          "estimatedTokensBefore": 11161,
+          "estimatedTokensAfter": 6910
+        }
+      ],
+      "terminalDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "Object",
+            "providerHints": [],
+            "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
+            "reason": "budget"
+          }
+        ]
+      },
+      "thrownDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "HarnessExecutionError",
+            "providerHints": [],
+            "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
+            "reason": "budget"
+          },
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "Error",
+            "providerHints": [],
+            "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
+            "reason": "budget"
+          }
+        ]
+      },
+      "guardrails": [],
+      "toolCalls": 16,
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "protectedFilesChanged": [],
+      "gradingMethod": "empty-patch-no-solution",
+      "officialResolved": false,
+      "gradingDurationMs": 0.00429095234721899,
+      "totalDurationMs": 60055.93599996064
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "django__django-11265",
+      "repetition": 0,
+      "resolved": false,
+      "gradingStatus": "completed",
+      "usageComplete": true,
+      "costUsd": null,
+      "status": "LimitsExceeded",
+      "failure": null,
+      "durationMs": 33320.89087495115,
+      "toolCalls": 14,
+      "inputTokens": 105832,
+      "outputTokens": 876,
+      "cachedInputTokens": 87808,
+      "modelCalls": 14,
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "protectedFilesChanged": [],
+      "gradingMethod": "empty-patch-no-solution",
+      "officialResolved": false,
+      "gradingDurationMs": 0.002125045284628868,
+      "totalDurationMs": 34937.70324997604
+    }
+  ],
+  "rawSamplesSha256": "89a0afc821db1b7cf69d21f16be6c323e6730fab0af68f1e2c60b15f4c36b34d",
+  "validation": {
+    "testsPassed": 446,
+    "preflightPassed": 5,
+    "sourceHashesVerifiedAfterRun": true,
+    "verifiedSourceCount": 52,
+    "qwenStreamSmoke": {
+      "inputTokens": 288,
+      "outputTokens": 13,
+      "toolEvents": 1
+    }
+  },
+  "sdkDependencies": {
+    "@zhivex-ai/agents": "1.3.1",
+    "@zhivex-ai/core": "1.13.0",
+    "@zhivex-ai/gemini": "0.11.0",
+    "@zhivex-ai/meta": "0.2.2",
+    "@zhivex-ai/openai": "0.11.2",
+    "@zhivex-ai/qwen": "0.11.4"
+  },
+  "limitations": [
+    "Five known development cases, one repetition; not a new holdout.",
+    "Model, transport, reasoning setting and harness behavior differ from the earlier GPT run; no causal before/after efficiency or model ranking claim.",
+    "Native requires successful approved verification before import; mini uses its original submission flow.",
+    "One response may exceed a cumulative ceiling. No monetary rates supplied.",
+    "Earlier integration attempt with Qwen 0.11.1 was aborted because stream usage was missing; its interrupted billed usage is unknown and it is not a completed comparison.",
+    "Empty native patches count as unresolved without hidden test execution. Local ARM-to-AMD64 emulation is not general latency evidence."
+  ],
+  "abortedIntegrationAttempt": "results/swebench/qwen-flash-comparison-2026-09-08"
+}