@zhivex-ai/harness 1.0.0-rc.7 → 1.0.0

package/benchmarks/baselines/external-final-2026-09-08.json

@@ -0,0 +1,588 @@
+{
+  "schemaVersion": 1,
+  "date": "2026-09-08",
+  "kind": "final-external-cohort",
+  "dataset": "princeton-nlp/SWE-bench_Verified",
+  "revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a",
+  "model": "gpt-5.6-luna",
+  "seed": 20260910,
+  "repetitions": 1,
+  "limits": {
+    "steps": 24,
+    "outputPerTurn": 2048,
+    "inputTokens": 100000,
+    "outputTokens": 16000,
+    "timeoutSeconds": 300,
+    "memoryMb": 2048,
+    "cpus": 2
+  },
+  "versions": {
+    "mini-swe-agent": "2.4.6",
+    "swebench": "4.1.0"
+  },
+  "implementation": {
+    "gitCommit": "0a7b69dddaa70a54b6b5b4097691c8112d4d76eb",
+    "sources": {
+      "bun.lock": "1293016c4d75c3dba51281c95485ea23c80097541ced9575803fe5dcf3e9f8d9",
+      "evaluations/external/requirements.lock": "42e90a08f10f821ebb958af79256f707a9fb5cf365277c4f9e2715d612028179",
+      "package.json": "c9cb835bf12a8630846d79909950636d10896a1b6b60ed1fd5468697b94aee08",
+      "scripts/benchmark-swebench.ts": "ed17063e1847017ba2e72f14732b6ff8f5b1af7cd5253b91302af75f2a9e8476",
+      "scripts/swebench/common.py": "c19cfc30b85dc3ad8dbf1ba6a4165e72ce96f624eb1721b613cfa3207636f7be",
+      "scripts/swebench/mini_driver.py": "46badf61f041208b3a4428beb4e574f896a459d43ab4b7a03dc616ff04b1baf3",
+      "scripts/swebench/model-budget.ts": "8d2f5e33340bc9d3b44e40ad9fde7fcea32d0df20c4733d7f2e58a255bfd55cd",
+      "scripts/swebench/preflight.py": "f3a9fc241896eb5d63138127fea4396e59d95c277c009a29a2265ee35f4aa6eb",
+      "scripts/swebench/prepare.py": "8dc3695a24b49e4fbc81333f602d484431485800dcb3381130249bfe13391c20",
+      "scripts/swebench/report.py": "70afcdf0d1f081014bb75d878f3ddf9195959e96b523ddf1e0ad5d3ec8273549",
+      "scripts/swebench/run.py": "f39254bdb12ffcc2a3d39c131d02c91ff839535af6e234a0923d09b5443f459e",
+      "scripts/swebench/telemetry.ts": "66bb6393641de2b789e6daa2f1dcc4cb9e0c77cfcf15f351d03f606d4da262b1",
+      "scripts/swebench/test_contracts.py": "a8c1348f1dff2e563c139e6d3e83eb141b4f2a71f91a58e2c5c051ed2a431024",
+      "scripts/swebench/zhivex-driver.ts": "02153ad5812219b71d19c6fea8d01a51bc3793686090b343a53da1aaccb0ecaf",
+      "scripts/time-to-safe-fix-efficiency.ts": "dd20fd8839280332ae4ca80d6846e98e07ff15d79e200b206fa2ba6be2724d2f",
+      "src/bounded-reads.ts": "9807bb460894be3279c5425af19863d1c6e419a1582ba6910ec7e7fc35703ed6",
+      "src/capabilities.ts": "56fdb60a2f6e1d33b2dc26d38b6d2709807e491ba1268d47b97876c2c0591d65",
+      "src/change-envelope.ts": "9047ec978c9e5117965b8093ec961d855f97bbc2c3accdc79b74cfe6a0e4f01e",
+      "src/cli-options.ts": "97781386289e3877c1318e4c94698ca7aaf2f256b89534dfcb21e966eb7d939f",
+      "src/cli-profiles.ts": "4fe451f1645e95f4957597a8724fa989b3c661ae1693252bfd6e74a51d20fc74",
+      "src/cli-stream.ts": "5ab8a4b1b8cf0a3dfa7b5b1f9e4e0facf8f6e7f6c341bdf700a41dc8927c825e",
+      "src/cli.ts": "cce6f58dfbc543acdc029b188d4394cd059d940bdd361351fd877ae2d6e94880",
+      "src/compacted-tool-history.ts": "3bd433b9cf3e6fd3ad82e680e5cdc8512ddff380093cec2dc3d918d6f42b7fe7",
+      "src/compaction.ts": "3ce3933355d021c7ec8d7a9c791d9f1cd94379c44107547d0adb08145662bc2b",
+      "src/config.ts": "c160faaf2d9fb637c5a08a7aff65d761a12c707ed23ebb87ab0e8519e9536bf3",
+      "src/console-context.ts": "0ad7e4559504b9ed09ab500d615dea09af1075d357f02ec3b77f003d55d951db",
+      "src/console-input.ts": "dc8ea0cdbba1364db288d5156c5ff9819362c46b37ef35a98b0475b6c445b1f8",
+      "src/context-engineering.ts": "1995e564f6b91974e75bf66dd421ddcbea29b1f8eee59b76eb4e34c9d6d8f47e",
+      "src/edit-contracts.ts": "47cc3132e27f8749d54a32bbcc33bb174cf4448f0d0171b2578cdb559cf2176b",
+      "src/errors.ts": "e02f4eb80562a83762f908bda56c950ec6a2b78725c0c8a49f41fee0057ab903",
+      "src/execution-environment.ts": "8519e532dfafe4ae51bfbc81e0732267bc6aa9eec871ffde8c917aa001e358e5",
+      "src/file-security.ts": "4590b46c8e4b9d3891a75cd0d0ada14e0ae7b58e8e4bdf82f68df688a1fbb750",
+      "src/harness.ts": "82b75651065b40e27cbc71a49ce2d27eb307c6f7570f4a35201942ac487e92d6",
+      "src/index.ts": "f791311f14dc27105191d97bb5454748d10beab827da9aba3ea949ffc32fce7c",
+      "src/json-contracts.ts": "0702e5b0ff16ff6304c74b4101bd31a3fa8eabdebe267d312b56d3871a7e302c",
+      "src/mcp.ts": "5f3f7a35b89063264ae18ab9273b84e2e59ac8c8e611a6b64be377ba61cc1ac5",
+      "src/operations.ts": "0775da79208c6f5d18476cc76404e84ddda610979d9d0e5dc15035aa6999613c",
+      "src/orchestration.ts": "7383da1047363819e6560f51892fd4082e5337856a3d889cdc076f021273774e",
+      "src/package-manager.ts": "cec4666bbab2f42e0a269709631f79f9f87c2c4dbb3cafeffacfaff761069e2c",
+      "src/process-runtime.ts": "b7286731a0b5c96dcad0b8e3a4bf1155d0539d73303cd7ebe77ffabf858f1729",
+      "src/providers.ts": "70ccfc7f315cd7238e83d69b719ce4908f3b620c564cbb4f00f93e102e025602",
+      "src/replacement-edits.ts": "8a0efdd3a7a9fc3e5a5475916cbd17ef882265881bb61076db58673e80b72957",
+      "src/routing.ts": "8e7c9f5968b4bd638cbbc86b19847ca183f3418b9a298783b96a88cffa568e22",
+      "src/run-interruption.ts": "df99129cf2dd6b938eca629cb897c4350a7d57fe6bf0ce437f5f3f30271617df",
+      "src/sessions.ts": "5167c8cf36bd4b8845ee278883394700321b65397f49c1dda0cf0d0de09dd06e",
+      "src/sqlite-database.ts": "18765d1ca0c2f2ad87bfd82d311312dfe8a42006556f27bc21da5892ed3b044e",
+      "src/state-backup.ts": "b6f8f1aedeb5dce31b65c23c7ffc50030870655b58a5aaaef9989bafb9da5c68",
+      "src/state-directory.ts": "3711550991951e6b738a08deaa0e06da84d01864c03a9602600f592963fa3211",
+      "src/terminal-markdown.ts": "8cffb6ebaf25108cb2bee04956d9b2973635b010f5d7ce6be21b3ceffe3e8bfd",
+      "src/terminal-ui.ts": "e50025a8649d1f60efdf3af895b8639730ec2e86553fb80a61a776b11076c705",
+      "src/time-to-safe-fix.ts": "4add1d49c0ea209cc496c66fa2b2012dc7bcab87e678a58c8591b3c2cdc8df8a",
+      "src/usage-accounting.ts": "b57bf7e48037646b8c40e6c39b5983e28840487a16bb27de632a54b57ff4594b",
+      "src/version.ts": "b2c2f80a40adc08e8258b81e82434273f0bdfa787528b2221f14efe7f06aa747",
+      "src/workspace.ts": "259d0fef0621a8655c232b23dabc3df508343bb73cdb76713acd9601ca4b3cd7"
+    }
+  },
+  "host": {
+    "system": "Darwin",
+    "architecture": "arm64",
+    "python": "3.12.13",
+    "docker": "29.7.2"
+  },
+  "environmentDifferences": [
+    "Zhivex native OCI: read-only root and writable /workspace snapshot, operator auto-approval; mini: writable /testbed. Both deny container network and use the same derived image and CPU/memory limits.",
+    "Zhivex tool-call limit is steps times eight; mini counts model steps and executes each response's actions. Tool counts are reported, not claimed equivalent.",
+    "Input/output token ceilings are checked between model requests and can overshoot by one response; outputPerTurn is sent to both APIs."
+  ],
+  "approvalPolicy": "Automated operator approves supported isolated Zhivex tools; successful approved patch import is a terminal receipt. Human wait is zero. Not a human-UX comparison.",
+  "excludedTaskIds": [
+    "astropy__astropy-14995",
+    "django__django-12262",
+    "django__django-13410",
+    "django__django-14915",
+    "django__django-15569",
+    "pydata__xarray-4695",
+    "pydata__xarray-6721",
+    "pytest-dev__pytest-8399",
+    "sympy__sympy-15809",
+    "sympy__sympy-20438"
+  ],
+  "excludedManifestSha256": [
+    "4902585186402c9fdd3e10db9c537109b5a108ba46b4e0806e259f6a8c4c15ab",
+    "76739783c68673b6f659d20f757a191a52213add63e46706737535484aec6f41"
+  ],
+  "tasks": [
+    {
+      "instance_id": "sympy__sympy-21930",
+      "repo": "sympy/sympy",
+      "base_commit": "de446c6d85f633271dfec1452f6f28ea783e293f",
+      "evalImage": "swebench/sweb.eval.x86_64.sympy_1776_sympy-21930:zhx-ac9b1c65d632e5a3",
+      "evalImageId": "sha256:ac9b1c65d632e5a3d1af240db144ff27ed9a70b5507705d50480ede6899125d1",
+      "evalTag": "zhx-ac9b1c65d632e5a3",
+      "image": "sha256:81c7bb0b9b74ae262830b7d4b94d042ab60c10cdbe51620c3b387262c10794fc",
+      "checkoutHead": "b7d17e72736ba245d24b87f8f5e8c80565066de4",
+      "baseTree": "a8e44f94a63624d1f4af136f7382125a0794eecf",
+      "upstreamTree": "a8e44f94a63624d1f4af136f7382125a0794eecf",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "scikit-learn__scikit-learn-10908",
+      "repo": "scikit-learn/scikit-learn",
+      "base_commit": "67d06b18c68ee4452768f8a1e868565dd4354abf",
+      "evalImage": "swebench/sweb.eval.x86_64.scikit-learn_1776_scikit-learn-10908:zhx-8727c0f7bfb76c29",
+      "evalImageId": "sha256:8727c0f7bfb76c294cfef16c9fe3d0dc9ddf313e7c0fb3e8e9cbc74c4955404c",
+      "evalTag": "zhx-8727c0f7bfb76c29",
+      "image": "sha256:4da1cd215c9209495a9aa8638026733275c766ab970311d31b1b6a8f834687d8",
+      "checkoutHead": "de60d3cc1d489509f0d7fe1ad141171628d3d4b7",
+      "baseTree": "d7e9da7c13c10167281372f397c7313c52c46077",
+      "upstreamTree": "d7e9da7c13c10167281372f397c7313c52c46077",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "django__django-15554",
+      "repo": "django/django",
+      "base_commit": "59ab3fd0e9e606d7f0f7ca26609c06ee679ece97",
+      "evalImage": "swebench/sweb.eval.x86_64.django_1776_django-15554:zhx-9e1bc1db0e1ec55a",
+      "evalImageId": "sha256:9e1bc1db0e1ec55a32f346c36671df7384a51a9ed84eed9b7e839e1405f04292",
+      "evalTag": "zhx-9e1bc1db0e1ec55a",
+      "image": "sha256:05b0c262568607baf8716c78d6bef059bf4a57a219141c6877034ee1e90a223d",
+      "checkoutHead": "292b4e462f63a7dfa0f9cd742e14cfa0b43727d6",
+      "baseTree": "9014f24e170c210492e65fe54bcbf236e7342c4a",
+      "upstreamTree": "9014f24e170c210492e65fe54bcbf236e7342c4a",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "scikit-learn__scikit-learn-14141",
+      "repo": "scikit-learn/scikit-learn",
+      "base_commit": "3d997697fdd166eff428ea9fd35734b6a8ba113e",
+      "evalImage": "swebench/sweb.eval.x86_64.scikit-learn_1776_scikit-learn-14141:zhx-c0f6a733154dad51",
+      "evalImageId": "sha256:c0f6a733154dad51cb8f617ca2ea09d095a6f56a1576740c2d43f9020294f2e0",
+      "evalTag": "zhx-c0f6a733154dad51",
+      "image": "sha256:a24afa6604e714bb6cf031b045f171e8b883e876534f941bb10ee36854612d6d",
+      "checkoutHead": "f863dd9c48aee1eb395e5ca64e675a765cea44a9",
+      "baseTree": "74477c0718711537b2cb5f9805ffbf9ffd072312",
+      "upstreamTree": "74477c0718711537b2cb5f9805ffbf9ffd072312",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "django__django-11265",
+      "repo": "django/django",
+      "base_commit": "21aa2a5e785eef1f47beb1c3760fdd7d8915ae09",
+      "evalImage": "swebench/sweb.eval.x86_64.django_1776_django-11265:zhx-70a39003fb3ce8f0",
+      "evalImageId": "sha256:70a39003fb3ce8f01ee82dfeb16b09bc3714ab669d3743fb3e1e5689f18ea1ef",
+      "evalTag": "zhx-70a39003fb3ce8f0",
+      "image": "sha256:32bc939face8eeae33506b342f5a260ab00cc49d02b616c58d7ef92b20389532",
+      "checkoutHead": "ad2b3670a78c6db994d93f1b71bccd5d5ee4b9c4",
+      "baseTree": "cbaf76226fd9f842b01da4d2cd52d99d2d256be2",
+      "upstreamTree": "cbaf76226fd9f842b01da4d2cd52d99d2d256be2",
+      "imageModeDifferences": 0
+    }
+  ],
+  "report": {
+    "schemaVersion": 1,
+    "kind": "swebench-comparison",
+    "manifestSha256": "41bef4c53a58a99ccbc291b9f45f0e5e0641076913a2291e02407709abeda5e7",
+    "complete": true,
+    "candidates": {
+      "zhivex": {
+        "planned": 5,
+        "recorded": 5,
+        "resolved": 1,
+        "resolutionRate": 0.2,
+        "missing": 0,
+        "gradingFailures": 0,
+        "usageComplete": true,
+        "totalCostUsd": null,
+        "costPerResolvedUsd": null,
+        "resolvedP50Ms": 35303.871916024946
+      },
+      "mini-swe-agent": {
+        "planned": 5,
+        "recorded": 5,
+        "resolved": 2,
+        "resolutionRate": 0.4,
+        "missing": 0,
+        "gradingFailures": 0,
+        "usageComplete": true,
+        "totalCostUsd": null,
+        "costPerResolvedUsd": null,
+        "resolvedP50Ms": 51026.759750035126
+      }
+    },
+    "pairedTaskResolutionDifference": -0.2,
+    "pairedTaskBootstrap95": [
+      -0.6,
+      0.0
+    ],
+    "evidenceBoundary": "Local matched-model comparison. Missing and failed runs remain in the denominator. Small-sample intervals are exploratory. No safety superiority is inferred from SWE-bench correctness."
+  },
+  "samples": [
+    {
+      "candidate": "zhivex",
+      "instanceId": "sympy__sympy-21930",
+      "repetition": 0,
+      "status": "failed",
+      "failure": "EXECUTION_FAILED",
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "gradingMethod": "empty-patch-no-solution",
+      "inputTokens": 109770,
+      "outputTokens": 1405,
+      "cachedInputTokens": 89375,
+      "modelCalls": 14,
+      "toolCalls": 15,
+      "usageComplete": true,
+      "durationMs": 36313.512582999996,
+      "gradingDurationMs": 0.0038750004023313522,
+      "totalDurationMs": 38426.89483403228,
+      "protectedFilesChanged": [],
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "costUsd": null,
+      "compactions": [
+        {
+          "beforeStep": 9,
+          "estimatedTokensBefore": 16191,
+          "estimatedTokensAfter": 2451
+        },
+        {
+          "beforeStep": 15,
+          "estimatedTokensBefore": 13984,
+          "estimatedTokensAfter": 2323
+        }
+      ],
+      "guardrails": [],
+      "thrownDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "HarnessExecutionError",
+            "providerHints": [],
+            "fingerprint": "44307c15efe2937d3c485a25c58f92f1c49edc8682c93acc9d63fc7f55902d46",
+            "reason": "budget"
+          },
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "GuardrailTriggeredError",
+            "providerHints": [],
+            "fingerprint": "44307c15efe2937d3c485a25c58f92f1c49edc8682c93acc9d63fc7f55902d46",
+            "reason": "budget"
+          }
+        ]
+      }
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "sympy__sympy-21930",
+      "repetition": 0,
+      "status": "Submitted",
+      "failure": null,
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "inputTokens": 63382,
+      "outputTokens": 1760,
+      "cachedInputTokens": 54988,
+      "modelCalls": 10,
+      "toolCalls": 12,
+      "usageComplete": true,
+      "durationMs": 43753.464707988314,
+      "gradingDurationMs": 30527.015165949706,
+      "totalDurationMs": 76162.07329201279,
+      "protectedFilesChanged": [],
+      "patchSha256": "ca1e662809a4a64f31e79e420c92d3917640414e13da8f99b3d8dbfe1e255654",
+      "costUsd": null
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "scikit-learn__scikit-learn-10908",
+      "repetition": 0,
+      "status": "Submitted",
+      "failure": null,
+      "resolved": true,
+      "officialResolved": true,
+      "gradingStatus": "completed",
+      "inputTokens": 37875,
+      "outputTokens": 1739,
+      "cachedInputTokens": 31958,
+      "modelCalls": 9,
+      "toolCalls": 9,
+      "usageComplete": true,
+      "durationMs": 39658.867082966026,
+      "gradingDurationMs": 20469.47483404074,
+      "totalDurationMs": 61865.53675000323,
+      "protectedFilesChanged": [],
+      "patchSha256": "e5f2d5a54427ce867ca72aa5fbbd62212057a967c3aea36883c58ff3bd1cd1f5",
+      "costUsd": null
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "scikit-learn__scikit-learn-10908",
+      "repetition": 0,
+      "status": "failed",
+      "failure": "EXECUTION_FAILED",
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "gradingMethod": "empty-patch-no-solution",
+      "inputTokens": 61923,
+      "outputTokens": 1022,
+      "cachedInputTokens": 51405,
+      "modelCalls": 11,
+      "toolCalls": 11,
+      "usageComplete": true,
+      "durationMs": 22486.702500000003,
+      "gradingDurationMs": 0.007875030860304832,
+      "totalDurationMs": 24393.029000028037,
+      "protectedFilesChanged": [],
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "costUsd": null,
+      "compactions": [
+        {
+          "beforeStep": 8,
+          "estimatedTokensBefore": 14068,
+          "estimatedTokensAfter": 2460
+        }
+      ],
+      "guardrails": [],
+      "thrownDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "HarnessExecutionError",
+            "providerHints": [],
+            "fingerprint": "571ffa7ac28d66e19b6bd16f27679452d0593632d00d8215db49cf64d2bc1d34",
+            "reason": "not allowlisted"
+          },
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "GuardrailTriggeredError",
+            "providerHints": [],
+            "fingerprint": "571ffa7ac28d66e19b6bd16f27679452d0593632d00d8215db49cf64d2bc1d34",
+            "reason": "not allowlisted"
+          }
+        ]
+      }
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "django__django-15554",
+      "repetition": 0,
+      "status": "failed",
+      "failure": "EXECUTION_FAILED",
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "gradingMethod": "empty-patch-no-solution",
+      "inputTokens": 108608,
+      "outputTokens": 1050,
+      "cachedInputTokens": 85519,
+      "modelCalls": 13,
+      "toolCalls": 13,
+      "usageComplete": true,
+      "durationMs": 30223.128666999997,
+      "gradingDurationMs": 0.004749977961182594,
+      "totalDurationMs": 36155.90329194674,
+      "protectedFilesChanged": [],
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "costUsd": null,
+      "compactions": [
+        {
+          "beforeStep": 7,
+          "estimatedTokensBefore": 17027,
+          "estimatedTokensAfter": 2392
+        },
+        {
+          "beforeStep": 14,
+          "estimatedTokensBefore": 15818,
+          "estimatedTokensAfter": 2262
+        }
+      ],
+      "guardrails": [],
+      "thrownDiagnostic": {
+        "code": "EXECUTION_FAILED",
+        "category": "execution",
+        "retryable": false,
+        "chain": [
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "HarnessExecutionError",
+            "providerHints": [],
+            "fingerprint": "257a5212bb68ab6203f762391a207d9c3c64d5dc57ab39da72b6941615630aff",
+            "reason": "budget"
+          },
+          {
+            "providerReason": null,
+            "validationIssues": [],
+            "status": null,
+            "errorClass": "GuardrailTriggeredError",
+            "providerHints": [],
+            "fingerprint": "257a5212bb68ab6203f762391a207d9c3c64d5dc57ab39da72b6941615630aff",
+            "reason": "budget"
+          }
+        ]
+      }
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "django__django-15554",
+      "repetition": 0,
+      "status": "LimitsExceeded",
+      "failure": null,
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "inputTokens": 113398,
+      "outputTokens": 2488,
+      "cachedInputTokens": 98935,
+      "modelCalls": 12,
+      "toolCalls": 14,
+      "usageComplete": true,
+      "durationMs": 59101.23329202179,
+      "gradingDurationMs": 21510.963333013933,
+      "totalDurationMs": 82337.00520801358,
+      "protectedFilesChanged": [],
+      "patchSha256": "d899d303cd872b5a7fa9265b44ad58e897e6ca571b56de5b1edd1c3ad8c96469",
+      "costUsd": null
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "scikit-learn__scikit-learn-14141",
+      "repetition": 0,
+      "status": "Submitted",
+      "failure": null,
+      "resolved": true,
+      "officialResolved": true,
+      "gradingStatus": "completed",
+      "inputTokens": 48594,
+      "outputTokens": 992,
+      "cachedInputTokens": 40803,
+      "modelCalls": 8,
+      "toolCalls": 8,
+      "usageComplete": true,
+      "durationMs": 28712.256375001743,
+      "gradingDurationMs": 20437.585500010755,
+      "totalDurationMs": 51026.759750035126,
+      "protectedFilesChanged": [],
+      "patchSha256": "1650dcf9f4fe3e202d584bc014b3a0b054812ba3138f5c710b35c8d10f883172",
+      "costUsd": null
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "scikit-learn__scikit-learn-14141",
+      "repetition": 0,
+      "status": "completed",
+      "failure": null,
+      "resolved": true,
+      "officialResolved": true,
+      "gradingStatus": "completed",
+      "inputTokens": 38637,
+      "outputTokens": 637,
+      "cachedInputTokens": 31743,
+      "modelCalls": 6,
+      "toolCalls": 7,
+      "usageComplete": true,
+      "durationMs": 12561.703917,
+      "gradingDurationMs": 20575.091709033586,
+      "totalDurationMs": 35303.871916024946,
+      "protectedFilesChanged": [],
+      "patchSha256": "c8e5d23fa74e9c51166898ccadad0fb3550252a530cbb9cb1afd74da1ad160d8",
+      "costUsd": null,
+      "compactions": [],
+      "guardrails": [],
+      "thrownDiagnostic": null
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "django__django-11265",
+      "repetition": 0,
+      "status": "failed",
+      "failure": null,
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "gradingMethod": "empty-patch-no-solution",
+      "inputTokens": 103654,
+      "outputTokens": 1349,
+      "cachedInputTokens": 82882,
+      "modelCalls": 13,
+      "toolCalls": 14,
+      "usageComplete": true,
+      "durationMs": 30578.890375,
+      "gradingDurationMs": 0.0037499703466892242,
+      "totalDurationMs": 36611.47362500196,
+      "protectedFilesChanged": [],
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "costUsd": null,
+      "compactions": [
+        {
+          "beforeStep": 7,
+          "estimatedTokensBefore": 18334,
+          "estimatedTokensAfter": 3871
+        }
+      ],
+      "guardrails": [
+        {
+          "stage": "output",
+          "budgetLimit": "maxInputTokens"
+        }
+      ],
+      "thrownDiagnostic": null
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "django__django-11265",
+      "repetition": 0,
+      "status": "Submitted",
+      "failure": null,
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "inputTokens": 68305,
+      "outputTokens": 1767,
+      "cachedInputTokens": 57620,
+      "modelCalls": 9,
+      "toolCalls": 13,
+      "usageComplete": true,
+      "durationMs": 38476.34570900118,
+      "gradingDurationMs": 26749.848500010557,
+      "totalDurationMs": 67099.94679200463,
+      "protectedFilesChanged": [],
+      "patchSha256": "eae54f8d808244d0b0e7c17c38d022f6e3d3d461dbe119bdc2cbb93a5069c6d9",
+      "costUsd": null
+    }
+  ],
+  "rawSamplesSha256": "3127eab6724c58d70b6d50b1681c3c3e711565dec77bcf285d75726d8e18878f",
+  "validation": {
+    "checkPassed": true,
+    "testsPassed": 427,
+    "preflightPassed": 5,
+    "sourceHashesVerifiedAfterRun": true,
+    "offlineAccountingControl": {
+      "modelResponses": 3,
+      "inputTokensPerResponse": 100,
+      "beforeReportedInputTokens": 500,
+      "afterReportedInputTokens": 300
+    }
+  },
+  "limitations": [
+    "Five tasks and one repetition; exploratory local evidence, no general superiority claim.",
+    "This cohort differs from earlier pilot/intermediate selections, so changes in success rate do not measure causal improvement.",
+    "No pricing rates supplied; monetary costs unknown. Earlier interrupted/failed development attempts can have incomplete billed usage.",
+    "Native approvals, tools, filesystem policies and terminal import differ from upstream mini submission.",
+    "Local macOS ARM host emulates Linux AMD64; no general latency claim.",
+    "The intermediate cohort exposed duplicate SDK usage accounting and remains separate; all its tasks were excluded here."
+  ]
+}