agent-hitch 0.2.15 → 0.2.17

package/docs/evidence/eval-preparation-2026-10-09.json

@@ -0,0 +1,393 @@
+{
+  "recorded_at": "2026-10-09",
+  "scope": "Historical measurements preceding the PR cancellation-cleanup hardening and transplant to dev; no claim that the final PR runtime was used in these measurements.",
+  "environment": {
+    "host": "Linux x86_64; Intel Xeon Platinum KVM guest; 16 logical CPUs; 65,923,194,880 bytes RAM",
+    "node": "22.22.1",
+    "docker": "29.1.3",
+    "harbor": "0.21.0",
+    "model": "GPT-6 Luna via subscription (provider snapshot unresolved)"
+  },
+  "provenance": [
+    {
+      "archive": "profiling-20/dataset-fix-replay.json",
+      "sha256": "208c288701c3ad2aad541fa08caf7a234436263ed50e02b75db8380e47180e06"
+    },
+    {
+      "archive": "profiling-runtime-sharing/setup-comparison.json",
+      "sha256": "076bee8f9a5c49d1fcad24103a4be1ba437f32b51b71aed8fbfca36e650d3bd9"
+    },
+    {
+      "archive": "image-preparation-optimization/results.json",
+      "sha256": "cde8c293e6caba11c385cec78ac53d58bd4731f46b1912e811493aff08246a64"
+    },
+    {
+      "archive": "image-preparation-optimization/verifier-replay/result.json",
+      "sha256": "fa0cfcc9ce93605c24781705c5ad3060b4a4750836e86ebfb34693f576486431"
+    },
+    {
+      "archive": "gear-one-round-optimized/repeat-baseline-100/three-way-comparison.json",
+      "sha256": "5303f428331d32c920fdc8fc809f3a7957eccc749a643a99f72d9eb64feff5a0"
+    }
+  ],
+  "dataset_replay": {
+    "measured_at": "2026-10-08T15:35:35.503Z",
+    "model_calls": 0,
+    "note": "Sequential dataset-check replay, not end-to-end evaluation; bytes are logical reads, not physical disk I/O.",
+    "logical_byte_reduction": 0.9047557673677712,
+    "results": [
+      {
+        "mode": "optimized",
+        "tasks": 20,
+        "dataset_digest": "sha256:d66a830075362fc03124543c2f1ee4649e19b693a582d38feffc1a0a3f3ffd21",
+        "wall_ms": 31458.281092,
+        "collection_ms": 8200.212244000002,
+        "successful_dataset_reads": 93486,
+        "logical_dataset_bytes": 2786968032
+      },
+      {
+        "mode": "baseline",
+        "tasks": 20,
+        "dataset_digest": "sha256:d66a830075362fc03124543c2f1ee4649e19b693a582d38feffc1a0a3f3ffd21",
+        "wall_ms": 403823.85438100004,
+        "collection_ms": 389990.11956300004,
+        "successful_dataset_reads": 981204,
+        "logical_dataset_bytes": 29261278662
+      },
+      {
+        "mode": "optimized",
+        "tasks": 20,
+        "dataset_digest": "sha256:d66a830075362fc03124543c2f1ee4649e19b693a582d38feffc1a0a3f3ffd21",
+        "wall_ms": 48336.64004099998,
+        "collection_ms": 11848.485729999957,
+        "successful_dataset_reads": 93486,
+        "logical_dataset_bytes": 2786968032
+      }
+    ]
+  },
+  "runtime_setup": {
+    "tasks_per_batch": 20,
+    "concurrency": 16,
+    "order": [
+      "upload",
+      "shared",
+      "shared",
+      "upload"
+    ],
+    "successful_setups": 80,
+    "model_calls": 0,
+    "upload": {
+      "setup_seconds_per_task": 30.16,
+      "harness_upload_seconds_per_task": 18.31,
+      "controller_upload_seconds_per_task": 0.51,
+      "host_process_cpu_core_seconds_per_batch": 194.25,
+      "batch_wall_seconds": 111.76
+    },
+    "shared": {
+      "setup_seconds_per_task": 10.85,
+      "harness_upload_seconds_per_task": 0,
+      "controller_upload_seconds_per_task": 0,
+      "host_process_cpu_core_seconds_per_batch": 109.59,
+      "batch_wall_seconds": 40.84
+    },
+    "excluded_snapshot_seconds": [
+      10.259019555,
+      12.346748256
+    ]
+  },
+  "image_and_lifecycle": {
+    "scope": "Same frozen 20 tasks, C16, Luna subscription; harness prepared in advance. Baseline has warm dependency/build-layer caches; optimized full runs also reuse the prepared-image cache. Cold image preparation and harness warmup are measured separately. Whole-host and daemon resource counters include other host activity. Container and process totals can miss sub-second lifetimes.",
+    "baseline": {
+      "evalId": "eval_64cbf7280dfe4f598745928a5d2b5744",
+      "runtimeIds": [
+        "sha256:2658a4f4417eac3807a4d0c8d7fc1e5e901fd060f1ad363d7e478827eed1282b"
+      ],
+      "wallSeconds": 514.1380000114441,
+      "phaseMeans": {
+        "slot_seconds": 285.1684499979019,
+        "environment_setup_seconds": 67.6658903837204,
+        "agent_setup_seconds": 17.371835076808928,
+        "agent_execution_seconds": 106.45874143838883,
+        "verifier_seconds": 65.41652805805207,
+        "native_other_seconds": 25.621624088287355,
+        "after_native_seconds": 1.5526309370994569,
+        "after_publish_seconds": 1.0812000155448913,
+        "verifier_body_seconds": 15.2249,
+        "verifier_lifecycle_seconds": 50.19162805805206
+      },
+      "preparationMean": {
+        "candidate": 39.5421057337,
+        "verifier": 20.7157033891
+      },
+      "stops": {
+        "candidate": {
+          "count": 20,
+          "sum": 213.26049423217773,
+          "mean": 10.663024711608887,
+          "median": 10.501216650009155,
+          "max": 11.307181596755981
+        },
+        "simulator": {
+          "count": 20,
+          "sum": 38.22067189216614,
+          "mean": 1.9110335946083068,
+          "median": 1.8672200441360474,
+          "max": 4.260472536087036
+        },
+        "verifier": {
+          "count": 20,
+          "sum": 210.8357436656952,
+          "mean": 10.54178718328476,
+          "median": 10.36497938632965,
+          "max": 11.635977268218994
+        }
+      },
+      "valid": 20,
+      "passes": 2,
+      "dockerDaemonCpuSeconds": 1316.926889,
+      "diskWriteGiB": 19.91425323486328,
+      "agentAverage": 4.5173376614991865,
+      "modelAverage": 3.092310337033131,
+      "probeQuality": {
+        "unclosedPython": 0,
+        "unclosedHost": 0,
+        "missingPatches": [],
+        "containerCoverage": {
+          "dockerListFailures": 1,
+          "containerApiFailures": 0,
+          "supplementalSamples": 515,
+          "supplementedContainerSamples": 29,
+          "eventErrors": [],
+          "observerRestarts": []
+        }
+      }
+    },
+    "warm_first": {
+      "evalId": "eval_9592b7d83bd64d84a0ef7322ac32617f",
+      "runtimeIds": [
+        "sha256:072b2230200ed16e5e27966342725e101e46023cb23f093b2fafa3c8b48849d9"
+      ],
+      "wallSeconds": 399.9470000267029,
+      "phaseMeans": {
+        "slot_seconds": 207.14064998626708,
+        "environment_setup_seconds": 33.20775940418243,
+        "agent_setup_seconds": 17.37574373483658,
+        "agent_execution_seconds": 117.32188109159469,
+        "verifier_seconds": 22.65836421251297,
+        "native_other_seconds": 13.715027666091919,
+        "after_native_seconds": 1.7154239416122437,
+        "after_publish_seconds": 1.1464499354362487,
+        "verifier_body_seconds": 14.51355,
+        "verifier_lifecycle_seconds": 8.14481421251297
+      },
+      "preparationMean": {
+        "candidate": 3.6222376064999997,
+        "verifier": 1.4092942058
+      },
+      "stops": {
+        "candidate": {
+          "count": 20,
+          "sum": 5.032052278518677,
+          "mean": 0.25160261392593386,
+          "median": 0.21620488166809082,
+          "max": 0.6075403690338135
+        },
+        "simulator": {
+          "count": 20,
+          "sum": 29.80813455581665,
+          "mean": 1.4904067277908326,
+          "median": 1.4453762769699097,
+          "max": 2.123774766921997
+        },
+        "verifier": {
+          "count": 20,
+          "sum": 5.935137748718262,
+          "mean": 0.2967568874359131,
+          "median": 0.24156415462493896,
+          "max": 0.7315433025360107
+        }
+      },
+      "valid": 20,
+      "passes": 3,
+      "dockerDaemonCpuSeconds": 342.916441,
+      "diskWriteGiB": 4.655399322509766,
+      "agentAverage": 6.599403245108653,
+      "modelAverage": 4.016821683707523,
+      "probeQuality": {
+        "unclosedPython": 0,
+        "unclosedHost": 0,
+        "missingPatches": [],
+        "containerCoverage": {
+          "dockerListFailures": 0,
+          "containerApiFailures": 0,
+          "supplementalSamples": 401,
+          "supplementedContainerSamples": 0,
+          "eventErrors": [],
+          "observerRestarts": []
+        }
+      }
+    },
+    "warm_final": {
+      "evalId": "eval_23be5bcafce94109aae85d32c7094f88",
+      "runtimeIds": [
+        "sha256:e6ecc646f98c5906aa44d863779941f6bf82f236d2ebfa7bc399c34bd087df2b"
+      ],
+      "wallSeconds": 372.13700008392334,
+      "phaseMeans": {
+        "slot_seconds": 201.82434998750688,
+        "environment_setup_seconds": 29.518565595149994,
+        "agent_setup_seconds": 16.651894664764406,
+        "agent_execution_seconds": 120.01988308429718,
+        "verifier_seconds": 20.959102857112885,
+        "native_other_seconds": 12.259714829921723,
+        "after_native_seconds": 1.205488955974579,
+        "after_publish_seconds": 1.2097000002861023,
+        "verifier_body_seconds": 13.78495,
+        "verifier_lifecycle_seconds": 7.174152857112884
+      },
+      "preparationMean": {
+        "candidate": 3.1690642686,
+        "verifier": 1.23830467335
+      },
+      "stops": {
+        "candidate": {
+          "count": 20,
+          "sum": 4.894258499145508,
+          "mean": 0.2447129249572754,
+          "median": 0.22374379634857178,
+          "max": 0.6866343021392822
+        },
+        "simulator": {
+          "count": 20,
+          "sum": 27.246562242507935,
+          "mean": 1.3623281121253967,
+          "median": 1.336445689201355,
+          "max": 1.9595155715942383
+        },
+        "verifier": {
+          "count": 20,
+          "sum": 6.640327215194702,
+          "mean": 0.3320163607597351,
+          "median": 0.22688746452331543,
+          "max": 1.4987709522247314
+        }
+      },
+      "valid": 20,
+      "passes": 4,
+      "dockerDaemonCpuSeconds": 334.41200299999997,
+      "diskWriteGiB": 4.676868438720703,
+      "agentAverage": 7.2866287076979495,
+      "modelAverage": 4.82437277143521,
+      "probeQuality": {
+        "unclosedPython": 0,
+        "unclosedHost": 0,
+        "missingPatches": [],
+        "containerCoverage": {
+          "dockerListFailures": 0,
+          "containerApiFailures": 0,
+          "supplementalSamples": 373,
+          "supplementedContainerSamples": 0,
+          "eventErrors": [],
+          "observerRestarts": []
+        }
+      }
+    },
+    "preparation": {
+      "prepare-cold-c16": {
+        "label": "prepare-cold-c16",
+        "wallSeconds": 140.38053471798776,
+        "tasks": 20,
+        "roles": 40,
+        "imagePreparations": 60,
+        "cacheHits": 0,
+        "uniqueKeys": 60,
+        "hostCpuSeconds": 18.738639999999997,
+        "childCliCpuSeconds": 189.417582,
+        "roleMeans": {
+          "candidate": 62.5674219975539,
+          "verifier": 27.406673145008973
+        }
+      },
+      "prepare-warm-c16-a": {
+        "label": "prepare-warm-c16-a",
+        "wallSeconds": 17.670493709039874,
+        "tasks": 20,
+        "roles": 40,
+        "imagePreparations": 60,
+        "cacheHits": 60,
+        "uniqueKeys": 60,
+        "hostCpuSeconds": 16.190249,
+        "childCliCpuSeconds": 15.900727,
+        "roleMeans": {
+          "candidate": 7.530257558551966,
+          "verifier": 3.9297443004033994
+        }
+      },
+      "prepare-warm-c16-b": {
+        "label": "prepare-warm-c16-b",
+        "wallSeconds": 17.613185116962995,
+        "tasks": 20,
+        "roles": 40,
+        "imagePreparations": 60,
+        "cacheHits": 60,
+        "uniqueKeys": 60,
+        "hostCpuSeconds": 16.228609000000002,
+        "childCliCpuSeconds": 16.086499,
+        "roleMeans": {
+          "candidate": 7.4858669871580785,
+          "verifier": 4.021057845893665
+        }
+      }
+    }
+  },
+  "fixed_snapshot_verifier_replay": {
+    "tasks": 20,
+    "all_match": true,
+    "artifacts": [
+      "reward.json",
+      "assertions.json",
+      "process.json"
+    ]
+  },
+  "full_100_task_comparison": {
+    "method": "Three matched baseline evaluations, one logical trial per task. Binary pass is reward == total_score == 1. Invalid slots are never scored as zero.",
+    "observedFullRuns": {
+      "old": {
+        "tasks": 100,
+        "passes": 13,
+        "passRate": 0.13,
+        "meanProcessScore": 0.5384416037179195,
+        "invalid": 0
+      },
+      "new1": {
+        "tasks": 100,
+        "passes": 18,
+        "passRate": 0.18,
+        "meanProcessScore": 0.5258498004546767,
+        "invalid": 0
+      },
+      "new2": {
+        "tasks": 100,
+        "passes": 15,
+        "passRate": 0.15,
+        "meanProcessScore": 0.49785881969119955,
+        "invalid": 0
+      }
+    },
+    "firstPassNew2": {
+      "tasks": 99,
+      "passes": 15,
+      "passRate": 0.15151515151515152,
+      "meanProcessScore": 0.4942296879997687,
+      "invalid": 1
+    },
+    "limitations": [
+      "Only one old and two new model runs; repeat variation cannot prove absence of a version effect.",
+      "Old concurrency was 8; both new runs use 16. Source harness and task/verifier identities match, but old packaged runtime differs.",
+      "The configured model is the same; identity_resolved=false, so a precise provider model snapshot is not pinned.",
+      "Same-version variability includes model sampling and execution/service timing; it is not an estimate of pure sampling noise.",
+      "The primary common-valid cohort excludes infrastructure-invalid first-pass tasks and may have survivorship bias; repaired results are separately labeled.",
+      "Pairwise exact McNemar p-values are exploratory, assume independent task pairs, and are not equivalence tests or evidence of causal attribution."
+    ]
+  }
+}

package/docs/evidence/eval-preparation-2026-10-09.md

@@ -0,0 +1,178 @@
+# Evaluation preparation measurements — 2026-10-09
+
+Repeated local Harbor tasks spend substantial time preparing identical inputs.
+The changes reduce dataset payload reads, share immutable runtime installations,
+reuse prepared images, and shorten shutdown of Harbor's default keepalive.
+They do not share mutable task state or change scoring formulas.
+
+The [aggregate measurements](eval-preparation-2026-10-09.json) include exact
+values, runtime identities, evaluation IDs, measurement limitations, and SHA-256
+digests of the local source archives. Raw trajectories, task payloads, and
+credentials are not included. These are measurements of the development
+versions, before transplanting the changes onto `dev` and hardening cancellation
+cleanup during PR review. They are not fresh performance certification of the
+final PR runtime.
+
+## Environment and comparison boundaries
+
+Measurements used a Linux x86_64 KVM host with 16 logical Intel Xeon Platinum
+CPUs and approximately 61.4 GiB RAM, Node 22.22.1, Docker 29.1.3, and Harbor
+0.21.0. Live evaluations used the same frozen 20 AutomationBench Marketing tasks,
+DSH harness, and GPT-6 Luna subscription configuration, with concurrency 16.
+The precise provider model snapshot was unresolved. The implementation contains
+no benchmark-specific rules; synthetic regressions cover independent layouts,
+datasets, build contexts, and harness artifacts.
+
+Each comparison isolates a different development stage. Their percentages must
+not be added or multiplied into a claimed overall speedup. Model workload,
+filesystem cache state, and other activity on the shared host can affect elapsed
+times. Per-task phases overlap across tasks and cannot be summed into batch wall
+time.
+
+## Dataset verification
+
+A model-free replay brackets the original implementation with two optimized
+runs. It exercises admission checks, native-descriptor lookup, score collection
+checks, and the new final dataset sweep. All three runs have the same dataset
+digest and descriptor results.
+
+| Metric, 20 tasks | Original | Optimized, first | Optimized, second |
+| --- | ---: | ---: | ---: |
+| Logical dataset bytes read | 29,261,278,662 | 2,786,968,032 | 2,786,968,032 |
+| Successful dataset file reads | 981,204 | 93,486 | 93,486 |
+| Full verification replay, seconds | 403.82 | 31.46 | 48.34 |
+| Collection portion, seconds | 389.99 | 8.20 | 11.85 |
+
+Logical bytes decrease by **90.48%**. This is not a physical disk I/O counter or
+an end-to-end evaluation speedup. Shared-host load affected timings; the stable
+read counts are the stronger evidence. Payload reads grow linearly for this v1
+collection path, while manifest parsing and task-set metadata checks still occur
+per collection. There is no pathname/mtime cache. Resource v2 closure validation
+and standalone full verification remain in place.
+
+Replay the current implementation with any standardized v1 dataset and a built
+pre-optimization checkout. After integrating #46, scoped verification uses
+identity-bound execution and trial receipts. The current script counts task
+content reads only; the historical measurements above also include root
+manifest metadata reads and are retained unchanged.
+
+```sh
+npm run build
+node dist/scripts/benchmark-dataset-verification.js \
+  --dataset /absolute/v1-dataset \
+  --baseline /absolute/original-hitch \
+  --output /absolute/comparison.json
+```
+
+## Shared immutable runtime and dependencies
+
+Four controlled batches ran `upload → shared → shared → upload`, each with 20
+real bridge setups at concurrency 16. All 80 setups succeeded. Controller and
+harness content identities and the base image were fixed. This experiment
+contains no model, simulator, or verifier execution.
+
+| Metric | Upload | Read-only snapshots |
+| --- | ---: | ---: |
+| Mean setup, seconds/task | 30.16 | 10.85 |
+| Harness upload, seconds/task | 18.31 | 0 |
+| Controller upload, seconds/task | 0.51 | 0 |
+| Host worker/CLI CPU, core-seconds/batch | 194.25 | 109.59 |
+| Batch wall time including create/cleanup, seconds | 111.76 | 40.84 |
+
+Mean setup decreases by **64.0%** and measured host process CPU by **43.6%**.
+The batch and per-task times exclude **10.26–12.35 seconds** to create and verify
+the one-time snapshots. Host process CPU excludes dockerd/containerd. Existing
+artifact validation and per-task Node extraction remain; they were approximately
+unchanged. An earlier full live evaluation at this stage was slower overall
+(755.01 versus 723.52 seconds) because other phases dominated. Sharing alone did
+not establish an end-to-end gain.
+
+## Image preparation and verifier lifecycle
+
+This baseline already includes dataset and shared-runtime optimizations. Both
+sides have warm dependency and Docker build-layer caches; the optimized side
+additionally reuses the new prepared-image cache and manages Harbor's default
+keepalive. Harness preparation occurred in advance.
+
+| Metric | Baseline | Optimized warm run 1 | Optimized warm run 2 |
+| --- | ---: | ---: | ---: |
+| Full evaluation wall time, seconds | 514.14 | 399.95 | 372.14 |
+| Candidate + sidecar image preparation, seconds/task | 39.54 | 3.62 | 3.17 |
+| Environment setup including readiness, seconds/task | 67.67 | 33.21 | 29.52 |
+| Verifier image preparation, seconds/task | 20.72 | 1.41 | 1.24 |
+| Complete verifier phase, seconds/task | 65.42 | 22.66 | 20.96 |
+| Verifier lifecycle excluding scoring body, seconds/task | 50.19 | 8.14 | 7.17 |
+| Scoring body, seconds/task | 15.22 | 14.51 | 13.78 |
+| Candidate stop signal to exit, seconds/container | 10.66 | 0.25 | 0.24 |
+| Verifier stop signal to exit, seconds/container | 10.54 | 0.30 | 0.33 |
+| Dockerd + containerd CPU, core-seconds | 1,316.93 | 342.92 | 334.41 |
+| Whole-host disk writes, GiB | 19.91 | 4.66 | 4.68 |
+
+Both optimized runs produced 20/20 valid executions. Wall time decreased by
+**22.2–27.6%** in these runs; the final run's candidate and verifier image
+preparation decreased by **92.0%** and **94.0%**. Scoring execution remains in a
+separate verifier container. Whole-host writes and daemon CPU include unrelated
+activity and cannot be attributed exclusively to this evaluation. Process and
+container sampling can miss very short lifetimes.
+
+The cold preparation-only run took **140.38 seconds** for 60 image preparations.
+Two warm repeats took **17.67** and **17.61 seconds**, with 60/60 hits. There were
+**60 distinct complete build contexts**: observed gains here are primarily reuse
+across rounds. This implementation does not deduplicate shared subsets of
+different contexts. Full-context hashing intentionally invalidates on changes to
+unused or ignored files as well.
+
+The final live run's 60 executed image IDs matched their preparation receipts.
+Candidate and verifier mounts were checked. Average active agents increased
+from 4.52 to 7.29, and average simultaneous model requests from 3.09 to 4.82;
+these averages do not guarantee throughput at higher configured concurrency.
+
+## Correctness evidence and limits
+
+Regrading the same 20 saved pre-optimization snapshots produced identical
+`reward.json`, `assertions.json`, and `process.json` outputs for all 20 tasks.
+This isolates verifier behavior from new model trajectories. Unit and real Docker
+fixtures additionally cover immutable input identities, cross-process build
+deduplication, invalidation, eviction, failures, cancellation, isolated writable
+state, role separation, prebuilt-image precedence, and preserved custom commands.
+
+A complete Gear round also finished with 140/140 valid executions: 100 baseline,
+12 local, and 28 physical bridge trials. The candidate was rejected by the bridge
+objective; successful execution does not imply promotion or task success.
+
+The old 100-task baseline scored 13/100; two optimized baselines scored 18/100 and
+15/100. The second optimized baseline initially had one model transport failure;
+that invalid slot was repaired, retaining the other 99 trials. The old run used
+concurrency 8 and the optimized runs used 16. Seven tasks passed only in the old
+run and failed in both optimized runs. These limited, differently concurrent
+live samples do **not** establish score equivalence or prove that every
+difference is model randomness. Matched-concurrency repeated comparisons remain
+necessary for that conclusion.
+
+## Opt-in behavior and tradeoffs
+
+See [the storage and preparation guide](../resource-storage.zh-CN.md) for current
+configuration. Dataset verification changes apply to the admitted v1 evaluation
+path; runtime sharing, image caching, and managed keepalive are opt-in. Upload
+remains the default runtime transport. Read-only runtime sharing requires local
+Linux Docker and harnesses that do not modify their installation directory.
+
+Image caching uses private host-local state and conservative supported inputs.
+Unsupported Compose/Dockerfile features and explicit force builds retain the
+original build path. Prepared images and evaluation snapshots are retained;
+there is no automatic cache garbage collector. Mutable workspaces, credentials,
+Node extraction, and verifier instances remain private. Repeated cancellation
+must finish Docker process-group cleanup before releasing build locks.
+
+Review found and fixed a case where the Docker CLI had exited but a descendant
+still held its pipes: cancellation previously skipped cleanup when the leader
+already had a return code. The new real-process regression waits for leader exit,
+cancels a stubborn descendant, and verifies pipe closure before returning.
+Output draining stays active during cleanup to avoid pipe backpressure deadlocks.
+
+Review also replaced role-marker/suffix matching with the exact candidate session
+derived from `TrialPaths`. A truncated verifier session can lose its role marker,
+and a verifier step named `env` can resemble a candidate suffix. Regression tests
+cover both cases, long trial names, and legitimate candidate names containing
+`__verifier__`; real Docker checks confirm that verifier image contents and
+mounts stay independent.