@zhivex-ai/harness 1.0.0-rc.7 → 1.0.0

package/benchmarks/baselines/external-pilot-2026-09-08.json

@@ -0,0 +1,439 @@
+{
+  "schemaVersion": 1,
+  "date": "2026-09-08",
+  "kind": "exploratory-external-pilot",
+  "dataset": "princeton-nlp/SWE-bench_Verified",
+  "revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a",
+  "model": "gpt-5.6-luna",
+  "seed": 20260908,
+  "repetitions": 1,
+  "limits": {
+    "steps": 24,
+    "outputPerTurn": 2048,
+    "inputTokens": 100000,
+    "outputTokens": 16000,
+    "timeoutSeconds": 300,
+    "memoryMb": 2048,
+    "cpus": 2
+  },
+  "versions": {
+    "mini-swe-agent": "2.4.6",
+    "swebench": "4.1.0"
+  },
+  "implementation": {
+    "gitCommit": "0a7b69dddaa70a54b6b5b4097691c8112d4d76eb",
+    "sources": {
+      "bun.lock": "1293016c4d75c3dba51281c95485ea23c80097541ced9575803fe5dcf3e9f8d9",
+      "evaluations/external/requirements.lock": "42e90a08f10f821ebb958af79256f707a9fb5cf365277c4f9e2715d612028179",
+      "package.json": "c9cb835bf12a8630846d79909950636d10896a1b6b60ed1fd5468697b94aee08",
+      "scripts/swebench/common.py": "c19cfc30b85dc3ad8dbf1ba6a4165e72ce96f624eb1721b613cfa3207636f7be",
+      "scripts/swebench/mini_driver.py": "46badf61f041208b3a4428beb4e574f896a459d43ab4b7a03dc616ff04b1baf3",
+      "scripts/swebench/preflight.py": "f3a9fc241896eb5d63138127fea4396e59d95c277c009a29a2265ee35f4aa6eb",
+      "scripts/swebench/prepare.py": "94e535c7b8c0f7581eebb06f4ed6d60ad7d566d42d7e8c603bf2964f45d42156",
+      "scripts/swebench/report.py": "70afcdf0d1f081014bb75d878f3ddf9195959e96b523ddf1e0ad5d3ec8273549",
+      "scripts/swebench/run.py": "a3f99566856a97489177722702082c88b861f3b9af5d4bea8d833962b2db491f",
+      "scripts/swebench/test_contracts.py": "a8c1348f1dff2e563c139e6d3e83eb141b4f2a71f91a58e2c5c051ed2a431024",
+      "scripts/swebench/zhivex-driver.ts": "0883f23e050d4fb3ac56a8e751d811bbc8e964085d2bd575c68a86ed10180486",
+      "src/bounded-reads.ts": "9807bb460894be3279c5425af19863d1c6e419a1582ba6910ec7e7fc35703ed6",
+      "src/capabilities.ts": "56fdb60a2f6e1d33b2dc26d38b6d2709807e491ba1268d47b97876c2c0591d65",
+      "src/change-envelope.ts": "9047ec978c9e5117965b8093ec961d855f97bbc2c3accdc79b74cfe6a0e4f01e",
+      "src/cli-options.ts": "97781386289e3877c1318e4c94698ca7aaf2f256b89534dfcb21e966eb7d939f",
+      "src/cli-profiles.ts": "4fe451f1645e95f4957597a8724fa989b3c661ae1693252bfd6e74a51d20fc74",
+      "src/cli-stream.ts": "5ab8a4b1b8cf0a3dfa7b5b1f9e4e0facf8f6e7f6c341bdf700a41dc8927c825e",
+      "src/cli.ts": "cce6f58dfbc543acdc029b188d4394cd059d940bdd361351fd877ae2d6e94880",
+      "src/compaction.ts": "9cc6ef39ebd2a024fac729177e2015db5121cc7168c25096d439061258599d1b",
+      "src/config.ts": "c160faaf2d9fb637c5a08a7aff65d761a12c707ed23ebb87ab0e8519e9536bf3",
+      "src/console-context.ts": "0ad7e4559504b9ed09ab500d615dea09af1075d357f02ec3b77f003d55d951db",
+      "src/console-input.ts": "dc8ea0cdbba1364db288d5156c5ff9819362c46b37ef35a98b0475b6c445b1f8",
+      "src/context-engineering.ts": "1995e564f6b91974e75bf66dd421ddcbea29b1f8eee59b76eb4e34c9d6d8f47e",
+      "src/edit-contracts.ts": "47cc3132e27f8749d54a32bbcc33bb174cf4448f0d0171b2578cdb559cf2176b",
+      "src/errors.ts": "e02f4eb80562a83762f908bda56c950ec6a2b78725c0c8a49f41fee0057ab903",
+      "src/execution-environment.ts": "341c977be61b9c2d50ed0239a52e9ef7ea5d553f3ea2b322b6567c5b645c1ff8",
+      "src/file-security.ts": "4590b46c8e4b9d3891a75cd0d0ada14e0ae7b58e8e4bdf82f68df688a1fbb750",
+      "src/harness.ts": "e2186d84e7c969de8ed2141853aabbd9b28ba48422afcd87af2b0084d1aac5fc",
+      "src/index.ts": "f791311f14dc27105191d97bb5454748d10beab827da9aba3ea949ffc32fce7c",
+      "src/json-contracts.ts": "0702e5b0ff16ff6304c74b4101bd31a3fa8eabdebe267d312b56d3871a7e302c",
+      "src/mcp.ts": "5f3f7a35b89063264ae18ab9273b84e2e59ac8c8e611a6b64be377ba61cc1ac5",
+      "src/operations.ts": "0775da79208c6f5d18476cc76404e84ddda610979d9d0e5dc15035aa6999613c",
+      "src/orchestration.ts": "7383da1047363819e6560f51892fd4082e5337856a3d889cdc076f021273774e",
+      "src/package-manager.ts": "cec4666bbab2f42e0a269709631f79f9f87c2c4dbb3cafeffacfaff761069e2c",
+      "src/process-runtime.ts": "b7286731a0b5c96dcad0b8e3a4bf1155d0539d73303cd7ebe77ffabf858f1729",
+      "src/providers.ts": "70ccfc7f315cd7238e83d69b719ce4908f3b620c564cbb4f00f93e102e025602",
+      "src/routing.ts": "8e7c9f5968b4bd638cbbc86b19847ca183f3418b9a298783b96a88cffa568e22",
+      "src/run-interruption.ts": "df99129cf2dd6b938eca629cb897c4350a7d57fe6bf0ce437f5f3f30271617df",
+      "src/sessions.ts": "5167c8cf36bd4b8845ee278883394700321b65397f49c1dda0cf0d0de09dd06e",
+      "src/sqlite-database.ts": "18765d1ca0c2f2ad87bfd82d311312dfe8a42006556f27bc21da5892ed3b044e",
+      "src/state-backup.ts": "b6f8f1aedeb5dce31b65c23c7ffc50030870655b58a5aaaef9989bafb9da5c68",
+      "src/state-directory.ts": "3711550991951e6b738a08deaa0e06da84d01864c03a9602600f592963fa3211",
+      "src/terminal-markdown.ts": "8cffb6ebaf25108cb2bee04956d9b2973635b010f5d7ce6be21b3ceffe3e8bfd",
+      "src/terminal-ui.ts": "e50025a8649d1f60efdf3af895b8639730ec2e86553fb80a61a776b11076c705",
+      "src/time-to-safe-fix.ts": "4add1d49c0ea209cc496c66fa2b2012dc7bcab87e678a58c8591b3c2cdc8df8a",
+      "src/version.ts": "b2c2f80a40adc08e8258b81e82434273f0bdfa787528b2221f14efe7f06aa747",
+      "src/workspace.ts": "e5cff88e2b1e1ef4b0226d197c5a563b44133dd9f7de6e0c9a9dcf1d575fac3b"
+    }
+  },
+  "environmentDifferences": [
+    "Zhivex native OCI: read-only root and writable /workspace snapshot, operator auto-approval; mini: writable /testbed. Both deny container network and use the same derived image and CPU/memory limits.",
+    "Zhivex tool-call limit is steps times eight; mini counts model steps and executes each response's actions. Tool counts are reported, not claimed equivalent.",
+    "Input/output token ceilings are checked between model requests and can overshoot by one response; outputPerTurn is sent to both APIs."
+  ],
+  "hostObservation": {
+    "system": "Darwin",
+    "architecture": "arm64",
+    "candidateArchitecture": "linux/amd64",
+    "recordedAfterRun": true
+  },
+  "rawSamplesSha256": "3b531a885851dacd5ff3dfe6897661d4ac68b09b6da301acff4962f6dcab0c50",
+  "report": {
+    "schemaVersion": 1,
+    "kind": "swebench-comparison",
+    "manifestSha256": "bcaa8b72db20b0fe312fc18f876408b47df60ce57b4e9ecbcd9497a5186f3b71",
+    "complete": true,
+    "candidates": {
+      "zhivex": {
+        "planned": 5,
+        "recorded": 5,
+        "resolved": 0,
+        "resolutionRate": 0.0,
+        "missing": 0,
+        "gradingFailures": 0,
+        "usageComplete": false,
+        "totalCostUsd": null,
+        "costPerResolvedUsd": null,
+        "resolvedP50Ms": null
+      },
+      "mini-swe-agent": {
+        "planned": 5,
+        "recorded": 5,
+        "resolved": 5,
+        "resolutionRate": 1.0,
+        "missing": 0,
+        "gradingFailures": 0,
+        "usageComplete": true,
+        "totalCostUsd": null,
+        "costPerResolvedUsd": null,
+        "resolvedP50Ms": 56212.63020799961
+      }
+    },
+    "pairedTaskResolutionDifference": -1.0,
+    "pairedTaskBootstrap95": [
+      -1.0,
+      -1.0
+    ],
+    "evidenceBoundary": "Local matched-model comparison. Missing and failed runs remain in the denominator. Small-sample intervals are exploratory. No safety superiority is inferred from SWE-bench correctness."
+  },
+  "samples": [
+    {
+      "candidate": "zhivex",
+      "instanceId": "django__django-12262",
+      "repetition": 0,
+      "status": "failed",
+      "failure": "EXECUTION_FAILED",
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "gradingMethod": "empty-patch-no-solution",
+      "inputTokens": null,
+      "outputTokens": null,
+      "cachedInputTokens": null,
+      "modelCalls": null,
+      "toolCalls": 7,
+      "usageComplete": false,
+      "durationMs": 13677.864375,
+      "gradingDurationMs": 0.008000002708286047,
+      "totalDurationMs": 19484.622207994107,
+      "protectedFilesChanged": [],
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "costUsd": null
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "django__django-12262",
+      "repetition": 0,
+      "status": "Submitted",
+      "failure": null,
+      "resolved": true,
+      "officialResolved": true,
+      "gradingStatus": "completed",
+      "inputTokens": 35265,
+      "outputTokens": 1371,
+      "cachedInputTokens": 30245,
+      "modelCalls": 8,
+      "toolCalls": 8,
+      "usageComplete": true,
+      "durationMs": 30544.840458023828,
+      "gradingDurationMs": 28582.4573750142,
+      "totalDurationMs": 61055.9477499919,
+      "protectedFilesChanged": [],
+      "patchSha256": "4fc36a12a7a8626848e6ebcb116505f092b65468b9e7b901db86f11f3ec3fa0a",
+      "costUsd": null
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "pydata__xarray-4695",
+      "repetition": 0,
+      "status": "Submitted",
+      "failure": null,
+      "resolved": true,
+      "officialResolved": true,
+      "gradingStatus": "completed",
+      "inputTokens": 56514,
+      "outputTokens": 1687,
+      "cachedInputTokens": 49365,
+      "modelCalls": 8,
+      "toolCalls": 10,
+      "usageComplete": true,
+      "durationMs": 42424.86758297309,
+      "gradingDurationMs": 32091.419000003953,
+      "totalDurationMs": 76435.33033295535,
+      "protectedFilesChanged": [],
+      "patchSha256": "e80a2529f817bff7b55bad625289999722349994e461c8bdbc41089986d67929",
+      "costUsd": null
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "pydata__xarray-4695",
+      "repetition": 0,
+      "status": "failed",
+      "failure": null,
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "gradingMethod": "empty-patch-no-solution",
+      "inputTokens": 152643,
+      "outputTokens": 983,
+      "cachedInputTokens": 126124,
+      "modelCalls": 8,
+      "toolCalls": 8,
+      "usageComplete": true,
+      "durationMs": 18746.6455,
+      "gradingDurationMs": 0.00833295052871108,
+      "totalDurationMs": 19573.17333301762,
+      "protectedFilesChanged": [],
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "costUsd": null
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "pytest-dev__pytest-8399",
+      "repetition": 0,
+      "status": "failed",
+      "failure": null,
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "gradingMethod": "empty-patch-no-solution",
+      "inputTokens": 119718,
+      "outputTokens": 2621,
+      "cachedInputTokens": 91384,
+      "modelCalls": 10,
+      "toolCalls": 10,
+      "usageComplete": true,
+      "durationMs": 38416.944709,
+      "gradingDurationMs": 0.00504200579598546,
+      "totalDurationMs": 39259.015540999826,
+      "protectedFilesChanged": [],
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "costUsd": null
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "pytest-dev__pytest-8399",
+      "repetition": 0,
+      "status": "Submitted",
+      "failure": null,
+      "resolved": true,
+      "officialResolved": true,
+      "gradingStatus": "completed",
+      "inputTokens": 31289,
+      "outputTokens": 1390,
+      "cachedInputTokens": 24682,
+      "modelCalls": 7,
+      "toolCalls": 7,
+      "usageComplete": true,
+      "durationMs": 28087.192916020285,
+      "gradingDurationMs": 22515.386040962767,
+      "totalDurationMs": 52476.374458987266,
+      "protectedFilesChanged": [],
+      "patchSha256": "6c49c4e740cea7386407d88688ee488142668d3a908aaeea1ca630296dd52ff7",
+      "costUsd": null
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "django__django-15569",
+      "repetition": 0,
+      "status": "Submitted",
+      "failure": null,
+      "resolved": true,
+      "officialResolved": true,
+      "gradingStatus": "completed",
+      "inputTokens": 24366,
+      "outputTokens": 1240,
+      "cachedInputTokens": 19770,
+      "modelCalls": 8,
+      "toolCalls": 8,
+      "usageComplete": true,
+      "durationMs": 27954.237999976613,
+      "gradingDurationMs": 23797.319792036433,
+      "totalDurationMs": 53833.63074995577,
+      "protectedFilesChanged": [],
+      "patchSha256": "613e2497df50016f646c4772f08c0caef5e393f7281f89cc7902829eabfcf727",
+      "costUsd": null
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "django__django-15569",
+      "repetition": 0,
+      "status": "failed",
+      "failure": null,
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "gradingMethod": "empty-patch-no-solution",
+      "inputTokens": 110074,
+      "outputTokens": 2469,
+      "cachedInputTokens": 96292,
+      "modelCalls": 10,
+      "toolCalls": 11,
+      "usageComplete": true,
+      "durationMs": 58955.677459,
+      "gradingDurationMs": 0.012500036973506212,
+      "totalDurationMs": 66539.94154103566,
+      "protectedFilesChanged": [],
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "costUsd": null
+    },
+    {
+      "candidate": "zhivex",
+      "instanceId": "django__django-14915",
+      "repetition": 0,
+      "status": "completed",
+      "failure": null,
+      "resolved": false,
+      "officialResolved": false,
+      "gradingStatus": "completed",
+      "gradingMethod": "empty-patch-no-solution",
+      "inputTokens": 1607,
+      "outputTokens": 494,
+      "cachedInputTokens": 1348,
+      "modelCalls": 1,
+      "toolCalls": 0,
+      "usageComplete": true,
+      "durationMs": 8851.873416,
+      "gradingDurationMs": 0.01737498678267002,
+      "totalDurationMs": 15627.18308402691,
+      "protectedFilesChanged": [],
+      "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "costUsd": null
+    },
+    {
+      "candidate": "mini-swe-agent",
+      "instanceId": "django__django-14915",
+      "repetition": 0,
+      "status": "Submitted",
+      "failure": null,
+      "resolved": true,
+      "officialResolved": true,
+      "gradingStatus": "completed",
+      "inputTokens": 27973,
+      "outputTokens": 1390,
+      "cachedInputTokens": 23001,
+      "modelCalls": 8,
+      "toolCalls": 11,
+      "usageComplete": true,
+      "durationMs": 31511.68758404674,
+      "gradingDurationMs": 22792.866834031884,
+      "totalDurationMs": 56212.63020799961,
+      "protectedFilesChanged": [],
+      "patchSha256": "de226204ce78325860a067864ba2e168c4a57884a0af11655a2af3ff6416255c",
+      "costUsd": null
+    }
+  ],
+  "graderControls": {
+    "negative": {
+      "gradingStatus": "completed",
+      "resolved": false
+    },
+    "gold": {
+      "gradingStatus": "completed",
+      "resolved": true
+    }
+  },
+  "limitations": [
+    "Five tasks, one repetition: development pilot, not a population estimate or leaderboard claim.",
+    "Failed native run lost model telemetry; unknown values normalized to null here.",
+    "Three native failed runs exceeded cumulative input ceiling; exact terminal reasons were not retained.",
+    "Prices were not supplied; costs unknown. Earlier interrupted integration attempt may include unknown billed usage.",
+    "Native and upstream tools, prompts, filesystem policies and patch-import flows differ.",
+    "Host ARM emulates Linux AMD64; timing is local only.",
+    "Source snapshot preserves executed version; subsequent runner identity checks were added after the pilot.",
+    "Pilot source hash list omitted the shared telemetry helper and CLI launcher; future runner includes both. Raw source snapshot covers the recorded hashes only."
+  ],
+  "tasks": [
+    {
+      "instance_id": "django__django-12262",
+      "repo": "django/django",
+      "base_commit": "69331bb851c34f05bc77e9fc24020fe6908b9cd5",
+      "evalImage": "swebench/sweb.eval.x86_64.django_1776_django-12262:zhx-23c0777e08a37bd0",
+      "evalImageId": "sha256:23c0777e08a37bd0643f7638eaca9c57d1ee16fca79b6312b8d6de7fd1b5e7da",
+      "evalTag": "zhx-23c0777e08a37bd0",
+      "image": "sha256:9d73ee28310b4c46b552ac5af1df9a1a29ca892e7c1197474a53ed8ab33e73b5",
+      "checkoutHead": "6b2b46b41688bca917e590c7420e045d635365e5",
+      "baseTree": "88307fafa4610ef0c958cab7d18d723d63d5f495",
+      "upstreamTree": "88307fafa4610ef0c958cab7d18d723d63d5f495",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "pydata__xarray-4695",
+      "repo": "pydata/xarray",
+      "base_commit": "51ef2a66c4e0896eab7d2b03e3dfb3963e338e3c",
+      "evalImage": "swebench/sweb.eval.x86_64.pydata_1776_xarray-4695:zhx-fe6074de116357f2",
+      "evalImageId": "sha256:fe6074de116357f2b6f1b3b964726cd02b866f68509f3e74d3a58c48a778ac6d",
+      "evalTag": "zhx-fe6074de116357f2",
+      "image": "sha256:b754ea0042638b1b5a021b1b6acefaae4ec97d689587e6c2c0a3cdaf053383c5",
+      "checkoutHead": "a4cf03327c39366aca0ca7dfa7ab72b4521c658a",
+      "baseTree": "2452479f6f4c2de000d4f024a9b815c75c273d2a",
+      "upstreamTree": "2452479f6f4c2de000d4f024a9b815c75c273d2a",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "pytest-dev__pytest-8399",
+      "repo": "pytest-dev/pytest",
+      "base_commit": "6e7dc8bac831cd8cf7a53b08efa366bd84f0c0fe",
+      "evalImage": "swebench/sweb.eval.x86_64.pytest-dev_1776_pytest-8399:zhx-1403b263348f1da0",
+      "evalImageId": "sha256:1403b263348f1da02330634d688ebf8dd61fa69f2c301203c89551889946cb0d",
+      "evalTag": "zhx-1403b263348f1da0",
+      "image": "sha256:3269ccbd3f71a0b1ac4a3f5987b5288ce842c5880e81acd4410913ee514756c7",
+      "checkoutHead": "ac43e94eaf25c8c109af2500d327ef938dcf24d7",
+      "baseTree": "8501ad3839bb098676d9d3354979cef6e218d95c",
+      "upstreamTree": "8501ad3839bb098676d9d3354979cef6e218d95c",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "django__django-15569",
+      "repo": "django/django",
+      "base_commit": "884b4c27f506b3c29d58509fc83a35c30ea10d94",
+      "evalImage": "swebench/sweb.eval.x86_64.django_1776_django-15569:zhx-13d3ee8c6e4aa4d5",
+      "evalImageId": "sha256:13d3ee8c6e4aa4d5c34024f32ada971a11153bfe0023339ddc4bdfb0ca48925d",
+      "evalTag": "zhx-13d3ee8c6e4aa4d5",
+      "image": "sha256:75df77fc887642fcd8876219377a295e5d9aece12fe530bbed442a2bbeb3664f",
+      "checkoutHead": "d838c27da4388d13219dd8b7a858cb43c5e83623",
+      "baseTree": "5a345ced69aa5bba7a0f9d893d984359555160af",
+      "upstreamTree": "5a345ced69aa5bba7a0f9d893d984359555160af",
+      "imageModeDifferences": 0
+    },
+    {
+      "instance_id": "django__django-14915",
+      "repo": "django/django",
+      "base_commit": "903aaa35e5ceaa33bfc9b19b7f6da65ce5a91dd4",
+      "evalImage": "swebench/sweb.eval.x86_64.django_1776_django-14915:zhx-a677be35326049cb",
+      "evalImageId": "sha256:a677be35326049cb42933368d879f3ecfe408c4da1fb8842b70b556e22245b6b",
+      "evalTag": "zhx-a677be35326049cb",
+      "image": "sha256:14eed0dac453442afc6e6970a22d0bfdcca453b1c0d5cdb8fd5b88a8a83b5834",
+      "checkoutHead": "76ca4f33ea42f4e92e485096ca5a06f91aa51c15",
+      "baseTree": "e11d86a000acf1f8b4d18e945abd62be478041e0",
+      "upstreamTree": "e11e4698c812238fda4177459904145cbbb1176a",
+      "imageModeDifferences": 6514
+    }
+  ]
+}