@holmes-lab/holmes-kit 0.39.0 → 0.40.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (325) hide show
  1. package/CHANGELOG.md +578 -6
  2. package/README.md +120 -41
  3. package/dist/.build-id +1 -1
  4. package/dist/holmes/admission/core/admission-policy.d.ts +37 -0
  5. package/dist/holmes/admission/core/admission-policy.js +147 -0
  6. package/dist/holmes/admission/core/bounded-reader.d.ts +56 -0
  7. package/dist/holmes/admission/core/bounded-reader.js +135 -0
  8. package/dist/holmes/admission/core/change-classifier.d.ts +34 -0
  9. package/dist/holmes/admission/core/change-classifier.js +278 -0
  10. package/dist/holmes/admission/core/config-loader.d.ts +30 -0
  11. package/dist/holmes/admission/core/config-loader.js +145 -0
  12. package/dist/holmes/admission/core/event-judgement.d.ts +37 -0
  13. package/dist/holmes/admission/core/event-judgement.js +251 -0
  14. package/dist/holmes/admission/core/path-rules.d.ts +35 -0
  15. package/dist/holmes/admission/core/path-rules.js +88 -0
  16. package/dist/holmes/admission/core/pin.d.ts +7 -0
  17. package/dist/holmes/admission/core/pin.js +29 -0
  18. package/dist/holmes/admission/core/ports.d.ts +48 -0
  19. package/dist/holmes/admission/core/ports.js +11 -0
  20. package/dist/holmes/admission/core/range-resolver.d.ts +32 -0
  21. package/dist/holmes/admission/core/range-resolver.js +51 -0
  22. package/dist/holmes/admission/core/self-containment.d.ts +1 -0
  23. package/dist/holmes/admission/core/self-containment.js +125 -0
  24. package/dist/holmes/admission/core/signature-source.d.ts +9 -0
  25. package/dist/holmes/admission/core/signature-source.js +4 -0
  26. package/dist/holmes/admission/core/transition-planner.d.ts +15 -0
  27. package/dist/holmes/admission/core/transition-planner.js +14 -0
  28. package/dist/holmes/admission/core/verdict.d.ts +38 -0
  29. package/dist/holmes/admission/core/verdict.js +17 -0
  30. package/dist/holmes/admission/core/wiring.d.ts +44 -0
  31. package/dist/holmes/admission/core/wiring.js +618 -0
  32. package/dist/holmes/admission/forge/git-object-port.d.ts +4 -0
  33. package/dist/holmes/admission/forge/git-object-port.js +132 -0
  34. package/dist/holmes/assoc/assoc-arm.d.ts +1 -1
  35. package/dist/holmes/assoc/assoc-arm.js +1 -1
  36. package/dist/holmes/cli/agents.d.ts +8 -0
  37. package/dist/holmes/cli/agents.js +9 -5
  38. package/dist/holmes/cli/approve-cli.d.ts +2 -0
  39. package/dist/holmes/cli/approve-cli.js +392 -0
  40. package/dist/holmes/cli/approve-ref.d.ts +13 -7
  41. package/dist/holmes/cli/approve-ref.js +30 -10
  42. package/dist/holmes/cli/approve-status.d.ts +26 -2
  43. package/dist/holmes/cli/approve-status.js +98 -18
  44. package/dist/holmes/cli/approve-watch.d.ts +5 -0
  45. package/dist/holmes/cli/approve-watch.js +10 -5
  46. package/dist/holmes/cli/approve.d.ts +49 -13
  47. package/dist/holmes/cli/approve.js +249 -67
  48. package/dist/holmes/cli/attest.js +2 -1
  49. package/dist/holmes/cli/autonomy.d.ts +4 -1
  50. package/dist/holmes/cli/autonomy.js +17 -10
  51. package/dist/holmes/cli/codex-oracle.d.ts +11 -0
  52. package/dist/holmes/cli/codex-oracle.js +5 -1
  53. package/dist/holmes/cli/codex-toml.d.ts +10 -0
  54. package/dist/holmes/cli/codex-toml.js +36 -2
  55. package/dist/holmes/cli/doctor-boundary.d.ts +76 -0
  56. package/dist/holmes/cli/doctor-boundary.js +308 -0
  57. package/dist/holmes/cli/doctor-locks.d.ts +15 -0
  58. package/dist/holmes/cli/doctor-locks.js +119 -0
  59. package/dist/holmes/cli/doctor.d.ts +2 -0
  60. package/dist/holmes/cli/doctor.js +34 -5
  61. package/dist/holmes/cli/flags.d.ts +23 -0
  62. package/dist/holmes/cli/flags.js +89 -0
  63. package/dist/holmes/cli/index.d.ts +5 -1
  64. package/dist/holmes/cli/index.js +226 -442
  65. package/dist/holmes/cli/init.js +33 -0
  66. package/dist/holmes/cli/interactive-prompt.d.ts +39 -2
  67. package/dist/holmes/cli/interactive-prompt.js +98 -16
  68. package/dist/holmes/cli/lock-recover.d.ts +5 -0
  69. package/dist/holmes/cli/lock-recover.js +126 -0
  70. package/dist/holmes/cli/mcp-processes.js +1 -1
  71. package/dist/holmes/cli/release-docs.js +1 -1
  72. package/dist/holmes/cli/run-exec.d.ts +141 -0
  73. package/dist/holmes/cli/run-exec.js +1435 -0
  74. package/dist/holmes/cli/run-provision.d.ts +3 -1
  75. package/dist/holmes/cli/run-provision.js +255 -44
  76. package/dist/holmes/cli/run-wiring.d.ts +69 -0
  77. package/dist/holmes/cli/run-wiring.js +270 -0
  78. package/dist/holmes/cli/run.d.ts +21 -0
  79. package/dist/holmes/cli/run.js +341 -92
  80. package/dist/holmes/cli/runtime-binding-approval.js +32 -19
  81. package/dist/holmes/cli/screen-safe.d.ts +3 -92
  82. package/dist/holmes/cli/screen-safe.js +18 -756
  83. package/dist/holmes/config/config.d.ts +19 -0
  84. package/dist/holmes/config/config.js +18 -3
  85. package/dist/holmes/cpg/foundation/language-matrix.js +2 -2
  86. package/dist/holmes/cpg/import-resolver.js +1 -1
  87. package/dist/holmes/governance/approval-decisions.d.ts +47 -0
  88. package/dist/holmes/governance/approval-decisions.js +29 -0
  89. package/dist/holmes/governance/approval-grants.d.ts +220 -1
  90. package/dist/holmes/governance/approval-grants.js +612 -33
  91. package/dist/holmes/governance/approval-queue.d.ts +26 -0
  92. package/dist/holmes/governance/approval-queue.js +109 -11
  93. package/dist/holmes/governance/approval-spend.d.ts +95 -0
  94. package/dist/holmes/governance/approval-spend.js +191 -0
  95. package/dist/holmes/governance/autonomous-shell.d.ts +61 -0
  96. package/dist/holmes/governance/autonomous-shell.js +332 -0
  97. package/dist/holmes/governance/autonomy.d.ts +12 -0
  98. package/dist/holmes/governance/autonomy.js +51 -15
  99. package/dist/holmes/governance/constitution.d.ts +5 -0
  100. package/dist/holmes/governance/constitution.js +32 -12
  101. package/dist/holmes/governance/full-autonomy.d.ts +10 -0
  102. package/dist/holmes/governance/full-autonomy.js +103 -7
  103. package/dist/holmes/governance/harness-catalog.d.ts +103 -0
  104. package/dist/holmes/governance/harness-catalog.js +348 -0
  105. package/dist/holmes/governance/human-channel.d.ts +38 -0
  106. package/dist/holmes/governance/human-channel.js +49 -0
  107. package/dist/holmes/governance/install-manifest.d.ts +22 -0
  108. package/dist/holmes/governance/install-manifest.js +160 -0
  109. package/dist/holmes/governance/ledger-rechain.js +2 -0
  110. package/dist/holmes/governance/ledger-store.conformance.d.ts +7 -1
  111. package/dist/holmes/governance/ledger-store.conformance.js +66 -0
  112. package/dist/holmes/governance/ledger-store.d.ts +88 -11
  113. package/dist/holmes/governance/ledger-store.js +186 -8
  114. package/dist/holmes/governance/provenance-chain.d.ts +16 -4
  115. package/dist/holmes/governance/provenance-chain.js +19 -4
  116. package/dist/holmes/governance/provenance-ledger.d.ts +6 -3
  117. package/dist/holmes/governance/provenance-ledger.js +53 -18
  118. package/dist/holmes/governance/push-gate-runner.d.ts +2 -0
  119. package/dist/holmes/governance/push-gate-runner.js +50 -9
  120. package/dist/holmes/governance/push-gate.d.ts +13 -2
  121. package/dist/holmes/governance/push-gate.js +106 -14
  122. package/dist/holmes/governance/recorded-pairs.d.ts +96 -0
  123. package/dist/holmes/governance/recorded-pairs.js +522 -0
  124. package/dist/holmes/governance/replica-id.d.ts +1 -1
  125. package/dist/holmes/governance/replica-id.js +1 -1
  126. package/dist/holmes/governance/spec-seam.d.ts +59 -0
  127. package/dist/holmes/governance/spec-seam.js +587 -0
  128. package/dist/holmes/governance/spend-kinds.d.ts +16 -0
  129. package/dist/holmes/governance/spend-kinds.js +18 -0
  130. package/dist/holmes/guardrail/advisory-line.d.ts +102 -0
  131. package/dist/holmes/guardrail/advisory-line.js +718 -0
  132. package/dist/holmes/guardrail/decision-ledger.d.ts +2 -1
  133. package/dist/holmes/guardrail/decision-ledger.js +2 -1
  134. package/dist/holmes/guardrail/impact-gate.d.ts +7 -0
  135. package/dist/holmes/guardrail/impact-gate.js +33 -6
  136. package/dist/holmes/guardrail/line-continuation.d.ts +8 -0
  137. package/dist/holmes/guardrail/line-continuation.js +39 -0
  138. package/dist/holmes/guardrail/operator-person.d.ts +28 -4
  139. package/dist/holmes/guardrail/operator-person.js +57 -11
  140. package/dist/holmes/guardrail/refusal-codes.d.ts +8 -3
  141. package/dist/holmes/guardrail/refusal-codes.js +64 -10
  142. package/dist/holmes/guardrail/risk-classifier.d.ts +15 -9
  143. package/dist/holmes/guardrail/risk-classifier.js +228 -16
  144. package/dist/holmes/guardrail/risk-gate.d.ts +2 -8
  145. package/dist/holmes/guardrail/risk-gate.js +7 -34
  146. package/dist/holmes/guardrail/risk-types.d.ts +10 -0
  147. package/dist/holmes/guardrail/run-plan-scope.js +3 -1
  148. package/dist/holmes/guardrail/run-refusals.js +24 -8
  149. package/dist/holmes/guardrail/run-stages.d.ts +5 -0
  150. package/dist/holmes/guardrail/run-stages.js +25 -5
  151. package/dist/holmes/guardrail/scope-judgment.js +1 -1
  152. package/dist/holmes/guardrail/shell-words.js +24 -3
  153. package/dist/holmes/guardrail/unresolved-write.d.ts +40 -0
  154. package/dist/holmes/guardrail/unresolved-write.js +556 -0
  155. package/dist/holmes/guardrail/write-target.d.ts +3 -0
  156. package/dist/holmes/guardrail/write-target.js +26 -5
  157. package/dist/holmes/hooks/pre-tool-use.d.ts +116 -1
  158. package/dist/holmes/hooks/pre-tool-use.js +1176 -170
  159. package/dist/holmes/hooks/rtm-refresh-child.d.ts +1 -1
  160. package/dist/holmes/hooks/rtm-refresh-child.js +1 -1
  161. package/dist/holmes/hooks/session-start.js +17 -1
  162. package/dist/holmes/hooks/stop.d.ts +23 -0
  163. package/dist/holmes/hooks/stop.js +86 -33
  164. package/dist/holmes/mcp/approval-adapters.js +3 -1
  165. package/dist/holmes/mcp/approval-commit.js +3 -2
  166. package/dist/holmes/mcp/approval-policy.d.ts +2 -1
  167. package/dist/holmes/mcp/approval-policy.js +3 -1
  168. package/dist/holmes/mcp/handlers/adoption.d.ts +1 -1
  169. package/dist/holmes/mcp/handlers/approval-ask.d.ts +3 -0
  170. package/dist/holmes/mcp/handlers/approval-ask.js +18 -7
  171. package/dist/holmes/mcp/handlers/entity-integration.d.ts +3 -3
  172. package/dist/holmes/mcp/handlers/entity-integration.js +29 -11
  173. package/dist/holmes/mcp/handlers/entity-renumber.d.ts +4 -4
  174. package/dist/holmes/mcp/handlers/entity-renumber.js +29 -11
  175. package/dist/holmes/mcp/handlers/entity-store.d.ts +2 -2
  176. package/dist/holmes/mcp/handlers/entity-store.js +133 -59
  177. package/dist/holmes/mcp/handlers/maintenance-analysis.d.ts +13 -0
  178. package/dist/holmes/mcp/handlers/maintenance-analysis.js +59 -2
  179. package/dist/holmes/mcp/handlers/operator-inspection.d.ts +9 -5
  180. package/dist/holmes/mcp/handlers/operator-inspection.js +79 -16
  181. package/dist/holmes/mcp/handlers/phase-queries.d.ts +1 -1
  182. package/dist/holmes/mcp/handlers/requirement-triage.d.ts +4 -4
  183. package/dist/holmes/mcp/handlers/review-evidence.d.ts +2 -2
  184. package/dist/holmes/mcp/handlers/review-evidence.js +35 -11
  185. package/dist/holmes/mcp/handlers/review-queries.d.ts +5 -0
  186. package/dist/holmes/mcp/handlers/review-queries.js +8 -1
  187. package/dist/holmes/mcp/handlers/review-verdict.d.ts +3 -0
  188. package/dist/holmes/mcp/handlers/review-verdict.js +10 -3
  189. package/dist/holmes/mcp/handlers/risk-assessment.d.ts +7 -2
  190. package/dist/holmes/mcp/handlers/risk-assessment.js +34 -12
  191. package/dist/holmes/mcp/handlers/runtime-binding-tool.d.ts +2 -2
  192. package/dist/holmes/mcp/handlers/slice-orchestration.d.ts +6 -1
  193. package/dist/holmes/mcp/handlers/slice-orchestration.js +14 -1
  194. package/dist/holmes/mcp/handlers/spec-approval.d.ts +15 -3
  195. package/dist/holmes/mcp/handlers/spec-approval.js +32 -5
  196. package/dist/holmes/mcp/handlers/spec-authoring.d.ts +2 -2
  197. package/dist/holmes/mcp/handlers/spec-authoring.js +19 -2
  198. package/dist/holmes/mcp/handlers/spec-lifecycle.d.ts +11 -38
  199. package/dist/holmes/mcp/handlers/spec-lifecycle.js +441 -94
  200. package/dist/holmes/mcp/handlers/spec-queries.d.ts +2 -2
  201. package/dist/holmes/mcp/handlers/taint-screening.d.ts +1 -1
  202. package/dist/holmes/mcp/handlers/taint-screening.js +2 -2
  203. package/dist/holmes/mcp/handlers/test-execution.d.ts +471 -4
  204. package/dist/holmes/mcp/handlers/test-execution.js +199 -21
  205. package/dist/holmes/mcp/handlers/workspace-queries.d.ts +2 -2
  206. package/dist/holmes/mcp/handlers.d.ts +557 -82
  207. package/dist/holmes/mcp/handlers.js +74 -23
  208. package/dist/holmes/mcp/harness-attestation.d.ts +78 -5
  209. package/dist/holmes/mcp/harness-attestation.js +325 -53
  210. package/dist/holmes/mcp/maintenance-analyze.d.ts +6 -1
  211. package/dist/holmes/mcp/maintenance-analyze.js +13 -11
  212. package/dist/holmes/mcp/maintenance-evidence.d.ts +9 -0
  213. package/dist/holmes/mcp/maintenance-evidence.js +35 -3
  214. package/dist/holmes/mcp/requirement-triage-evidence.js +22 -13
  215. package/dist/holmes/mcp/server-staleness.js +1 -1
  216. package/dist/holmes/mcp/server.js +9 -8
  217. package/dist/holmes/mcp/spec-id-guard.js +1 -1
  218. package/dist/holmes/mcp/tool-schemas.d.ts +13 -0
  219. package/dist/holmes/mcp/tool-schemas.js +42 -9
  220. package/dist/holmes/project/act-root.d.ts +19 -0
  221. package/dist/holmes/project/act-root.js +100 -0
  222. package/dist/holmes/project/baseline.d.ts +15 -2
  223. package/dist/holmes/project/baseline.js +14 -8
  224. package/dist/holmes/project/bounded-group.d.ts +47 -0
  225. package/dist/holmes/project/bounded-group.js +140 -0
  226. package/dist/holmes/project/bounded-io.d.ts +51 -2
  227. package/dist/holmes/project/bounded-io.js +140 -6
  228. package/dist/holmes/project/change-source.d.ts +4 -0
  229. package/dist/holmes/project/change-source.js +14 -6
  230. package/dist/holmes/project/instance-id.d.ts +10 -0
  231. package/dist/holmes/project/instance-id.js +233 -0
  232. package/dist/holmes/project/machine-identity.d.ts +25 -0
  233. package/dist/holmes/project/machine-identity.js +133 -0
  234. package/dist/holmes/project/owned-lock.d.ts +84 -0
  235. package/dist/holmes/project/owned-lock.js +263 -0
  236. package/dist/holmes/project/plain-lock-fs.d.ts +2 -0
  237. package/dist/holmes/project/plain-lock-fs.js +126 -0
  238. package/dist/holmes/project/request-lock.d.ts +85 -0
  239. package/dist/holmes/project/request-lock.js +459 -0
  240. package/dist/holmes/project/root.d.ts +12 -4
  241. package/dist/holmes/project/root.js +29 -8
  242. package/dist/holmes/project/run-records.d.ts +19 -0
  243. package/dist/holmes/project/run-records.js +94 -12
  244. package/dist/holmes/project/screen-text.d.ts +94 -0
  245. package/dist/holmes/project/screen-text.js +763 -0
  246. package/dist/holmes/project/spend-lock.d.ts +82 -0
  247. package/dist/holmes/project/spend-lock.js +413 -0
  248. package/dist/holmes/refresh/freshness-coordinator.d.ts +1 -1
  249. package/dist/holmes/refresh/freshness-coordinator.js +1 -1
  250. package/dist/holmes/review/condition-hunk-replay.js +4 -2
  251. package/dist/holmes/review/confidence-calibration.d.ts +1 -1
  252. package/dist/holmes/review/confidence-calibration.js +1 -1
  253. package/dist/holmes/review/content-verify.js +1 -1
  254. package/dist/holmes/review/evidence-mode.d.ts +21 -0
  255. package/dist/holmes/review/evidence-mode.js +33 -0
  256. package/dist/holmes/review/head-worktree.d.ts +55 -0
  257. package/dist/holmes/review/head-worktree.js +329 -0
  258. package/dist/holmes/review/hop-ablation.d.ts +4 -4
  259. package/dist/holmes/review/hop-ablation.js +5 -5
  260. package/dist/holmes/review/mrp.js +13 -3
  261. package/dist/holmes/review/point-in-time-replay.js +12 -3
  262. package/dist/holmes/review/rank-diagnosis.d.ts +2 -2
  263. package/dist/holmes/review/rank-diagnosis.js +1 -1
  264. package/dist/holmes/review/replay-corpus.d.ts +10 -4
  265. package/dist/holmes/review/replay-corpus.js +25 -9
  266. package/dist/holmes/review/requirement-triage.js +2 -2
  267. package/dist/holmes/review/run-provenance.d.ts +45 -0
  268. package/dist/holmes/review/run-provenance.js +314 -0
  269. package/dist/holmes/review/semantic-arm.d.ts +3 -3
  270. package/dist/holmes/review/semantic-arm.js +3 -3
  271. package/dist/holmes/review/test-evidence.d.ts +174 -0
  272. package/dist/holmes/review/test-evidence.js +330 -2
  273. package/dist/holmes/review/test-outcomes.js +3 -1
  274. package/dist/holmes/review/test-reports.d.ts +94 -0
  275. package/dist/holmes/review/test-reports.js +382 -0
  276. package/dist/holmes/review/test-runner.d.ts +180 -58
  277. package/dist/holmes/review/test-runner.js +560 -324
  278. package/dist/holmes/review/test-selection-breadth.js +2 -2
  279. package/dist/holmes/review/test-units.d.ts +159 -0
  280. package/dist/holmes/review/test-units.js +814 -0
  281. package/dist/holmes/review/verdicts.d.ts +105 -8
  282. package/dist/holmes/review/verdicts.js +640 -36
  283. package/dist/holmes/rtm/anchor-census.d.ts +9 -0
  284. package/dist/holmes/rtm/anchor-census.js +151 -0
  285. package/dist/holmes/rtm/anchor-ids.d.ts +25 -1
  286. package/dist/holmes/rtm/anchor-ids.js +45 -2
  287. package/dist/holmes/rtm/comment-ranges.d.ts +2 -0
  288. package/dist/holmes/rtm/comment-ranges.js +104 -0
  289. package/dist/holmes/rtm/counted-tests.d.ts +1 -0
  290. package/dist/holmes/rtm/counted-tests.js +63 -0
  291. package/dist/holmes/rtm/impact-advisory.d.ts +1 -1
  292. package/dist/holmes/rtm/known-defects.js +1 -1
  293. package/dist/holmes/rtm/localize.js +1 -1
  294. package/dist/holmes/rtm/taint-benchmark.d.ts +1 -1
  295. package/dist/holmes/rtm/taint-vocabulary.d.ts +1 -1
  296. package/dist/holmes/rtm/test-scope.d.ts +31 -0
  297. package/dist/holmes/rtm/test-scope.js +166 -5
  298. package/dist/holmes/rtm/trace-gaps.js +1 -1
  299. package/dist/holmes/runtime/dispatch.d.ts +10 -0
  300. package/dist/holmes/runtime/dispatch.js +12 -1
  301. package/dist/holmes/semantic/cloud-usage.d.ts +1 -1
  302. package/dist/holmes/semantic/cloud-usage.js +1 -1
  303. package/dist/holmes/semantic/unreadable-paths.d.ts +1 -1
  304. package/dist/holmes/semantic/unreadable-paths.js +1 -1
  305. package/dist/holmes/spec/approval-blockers.js +1 -1
  306. package/dist/holmes/spec/entity-store.d.ts +6 -1
  307. package/dist/holmes/spec/entity-store.js +16 -2
  308. package/dist/holmes/spec/entity-transaction.d.ts +7 -2
  309. package/dist/holmes/spec/entity-transaction.js +87 -170
  310. package/dist/holmes/spec/spec-store.d.ts +12 -6
  311. package/dist/holmes/spec/spec-store.js +4 -2
  312. package/dist/holmes/spec/spec-types.js +1 -1
  313. package/dist/holmes/spec/succession-guard.d.ts +56 -0
  314. package/dist/holmes/spec/succession-guard.js +187 -0
  315. package/dist/holmes/spec/succession.d.ts +70 -0
  316. package/dist/holmes/spec/succession.js +171 -0
  317. package/dist/holmes/spec/transition-policy.d.ts +2 -2
  318. package/docs/install-guide.md +9 -3
  319. package/docs/language-support.md +78 -0
  320. package/package.json +4 -3
  321. package/playbooks/author-slice/PLAYBOOK.md +1 -1
  322. package/playbooks/promote-slice/PLAYBOOK.md +5 -1
  323. package/playbooks/publish/PLAYBOOK.md +47 -34
  324. package/dist/holmes/review/manual-baseline.d.ts +0 -209
  325. package/dist/holmes/review/manual-baseline.js +0 -2847
@@ -1,209 +0,0 @@
1
- /**
2
- * The "strong model with plain file search" baseline (Goal Phase B).
3
- *
4
- * Every other arm in this directory compares the harness against ANOTHER PROGRAM. That answers "is
5
- * the spec layer worth its cost", never the Goal's actual claim: that the harness helps an agent
6
- * beat what a competent developer reaches by reading and grepping. So this arm is me — given only
7
- * the request text and a materialized tree, ranking by grep counts over the query terms, with the
8
- * truth WITHHELD until the picks were written to disk.
9
- *
10
- * The rule this slice was approved under: if the baseline beats the shipped harness, the claim of
11
- * surpassing a human does not hold, and that fact gets recorded rather than buried. It did beat it
12
- * on recall. The record below is that measurement, kept so a later change has to move it.
13
- *
14
- * The `shipped` lists below are the pipeline AS IT WAS when this baseline ran (S-404). A-SPEC-405
15
- * later changed the re-ranking in response to what this record showed; re-scoring the same eight
16
- * cases through the new pipeline gives Top-5 recall 0.175 -> 0.217 and Top-10 recall 0.471 -> 0.533,
17
- * still short of the 0.404 / 0.554 reached here by reading and grepping. The record is deliberately
18
- * NOT refreshed: it is the measurement that motivated the change, and rewriting it would erase the
19
- * comparison it exists to preserve.
20
- *
21
- * n=8. A sample this small cannot separate a real gap from noise; report the n alongside the
22
- * numbers and do not treat a large difference here as a conclusion.
23
- */
24
- import { type ReplayOutcome } from './evaluation-metrics';
25
- export interface ManualBaselineCase {
26
- /** Commit subject with spec citations stripped — the only input the picks were made from. */
27
- query: string;
28
- /** Files chosen by grep-ranked reading, frozen before the truth was read. */
29
- picks: string[];
30
- /** What the shipped pipeline returned for the same query on the same tree. */
31
- shipped: string[];
32
- truthFiles: string[];
33
- }
34
- export interface ManualBaselineRecord {
35
- corpus: string;
36
- measuredOn: string;
37
- caveat: string;
38
- cases: ManualBaselineCase[];
39
- }
40
- export declare const MANUAL_BASELINE_RECORD: ManualBaselineRecord;
41
- export declare const MANUAL_BASELINE_N: number;
42
- /** Score the frozen picks with the same metric code every other arm uses. */
43
- export declare function manualBaselineOutcomes(record: ManualBaselineRecord): ReplayOutcome[];
44
- export declare function shippedOutcomes(record: ManualBaselineRecord): ReplayOutcome[];
45
- /**
46
- * Per-metric verdict, computed from the record rather than asserted by hand: `true` only where the
47
- * shipped harness is strictly ahead of the grep-and-read baseline.
48
- */
49
- export declare function shippedBeatsManual(record: ManualBaselineRecord): {
50
- top1: boolean;
51
- top5Recall: boolean;
52
- top10Recall: boolean;
53
- top10Precision: boolean;
54
- };
55
- /**
56
- * @implements A-SPEC-408
57
- * The second blind set. Set 1 turned out to be markedly harder than the corpus average — Top-10
58
- * recall 0.404 against 0.688 for the other 22 cases of the same 30 — so its verdict could have been
59
- * a property of those eight commits rather than of the harness. These eight were drawn the same way
60
- * from further down the same slice, their truth was unread until the picks were on disk, and the
61
- * asymmetry came out in the same direction: the harness leads Top-1, reading and grepping leads
62
- * recall.
63
- *
64
- * One difference from set 1 that matters when reading the two together: these `shipped` lists come
65
- * from the pipeline AFTER A-SPEC-405/406/407, while set 1's are the S-404-era pipeline, kept
66
- * deliberately unrefreshed. So compare each set's two rows to each other, never set 1's harness row
67
- * to set 2's.
68
- */
69
- export declare const MANUAL_BASELINE_SET_2: ManualBaselineRecord;
70
- /**
71
- * @implements A-SPEC-410
72
- * The comparison once BOTH sides get the same thing.
73
- *
74
- * A-SPEC-404 and 408 compared the harness ALONE against a person reading and grepping, so the
75
- * presence of a judgement step was tangled up with the tools. Two asymmetries were then removed,
76
- * one per round, each on a set whose truth was unread until the picks were frozen:
77
- *
78
- * set "evidence-shown" (cases 32-39): the harness arm had been given file PATHS only while the
79
- * grep arm saw occurrence counts. Fixed by actually showing what A-SPEC-409 carries — lexical
80
- * score, matched symbols, via-specs, commit prose, graph-hop count, recency.
81
- * set "two-step" (cases 40-47): the grep arm judged TWICE (choosing the search terms, then the
82
- * files) while the harness arm judged once. Fixed by letting the harness be re-queried after
83
- * seeing its first result, which is what an agent actually does.
84
- *
85
- * Both sets: width 20 on each side, picks frozen before the truth was read.
86
- *
87
- * Pooled result (n=16). The verdict differs BY AXIS and is stated that way:
88
- * Top-1 harness+judgement 0.688 > grep+judgement 0.625
89
- * Top-5 recall harness+judgement 0.712 > grep+judgement 0.677
90
- * Top-10 recall harness+judgement 0.764 < grep+judgement 0.821
91
- * harness ALONE 0.438 / 0.505 / 0.651 — behind the human method on every recall axis.
92
- *
93
- * So the harness plus a judging caller leads at the top of the list and still trails on deep
94
- * recall, which is the axis the Goal is worded around. "Surpasses a human developer" does not hold.
95
- */
96
- export interface FairCase {
97
- set: string;
98
- query: string;
99
- /** The harness's own top ten, no judgement applied. */
100
- harnessAlone: string[];
101
- /** Ten chosen from the harness's evidence-annotated pool. */
102
- harnessJudged: string[];
103
- /** Ten chosen from grep output, the same judgement, the other tool. */
104
- grepJudged: string[];
105
- truthFiles: string[];
106
- }
107
- export declare const FAIR_COMPARISON: {
108
- measuredOn: string;
109
- caveat: string;
110
- cases: FairCase[];
111
- };
112
- /**
113
- * The third corpus, measured against a person instead of a program.
114
- *
115
- * A-SPEC-426 put the harness against a name-only matcher on this repository -- no spec store, a
116
- * third party's code -- and it won all four axes. That opponent was a program, and the note there
117
- * said the human comparison was still owed. This is it, under the A-SPEC-410 protocol: width 40 on
118
- * both sides, the harness's evidence actually shown, picks frozen before the truth was read, eight
119
- * cases whose answers I had never seen.
120
- *
121
- * grep and judgement win every axis, and not narrowly.
122
- *
123
- * The cause is not the reranking. Of nineteen truth files, NINE were anywhere in the forty the
124
- * harness exposed, and two cases offered none at all. The clearest example: for "properly format
125
- * markdown in the AskUser tool", the answer is packages/core/src/tools/ask-user.ts; one grep found
126
- * it immediately and it appears nowhere in the harness's forty. Reranking cannot recover a file
127
- * that was never offered.
128
- *
129
- * Read together with A-SPEC-426 rather than against it. Both are true, and the pair is the finding:
130
- * beating the automated baseline does not imply beating a person.
131
- */
132
- export interface ThirdCorpusCase extends FairCase {
133
- pool40: string[];
134
- }
135
- export declare const THIRD_CORPUS_COMPARISON: {
136
- corpus: string;
137
- opponent: string;
138
- note: string;
139
- caveat: string;
140
- harnessAloneTop10RecallHere: number;
141
- harnessAloneTop10RecallOn29: number;
142
- cases: ThirdCorpusCase[];
143
- };
144
- /** How much of the truth the exposed pool held at all -- the ceiling any reranking could reach. */
145
- export declare function poolCoverage(r: {
146
- cases: ThirdCorpusCase[];
147
- }): {
148
- truthTotal: number;
149
- inPool: number;
150
- casesWithNothing: number;
151
- };
152
- /**
153
- * What A-SPEC-428's window guarantee bought, measured two ways.
154
- *
155
- * Controlled first: the SAME eight cases A-SPEC-427 lost on, re-run after the change. Nine of
156
- * nineteen answers were inside the exposed forty before; fourteen after. Two cases offered nothing
157
- * at all; one after. Answers the lexical layer ranked in its top ten but the pipeline never showed:
158
- * five before, none after. Same commits, same queries — the increase belongs to the change.
159
- *
160
- * Then a fresh blind set of eight, truth unread until the picks were frozen. The gap to a person
161
- * with grep is much smaller than it was, and precision now favours the harness alone. It is still a
162
- * gap: grep and judgement lead both recall axes.
163
- *
164
- * One result here goes the other way and is kept because it does: on this set MY reranking was
165
- * WORSE than the harness's own ordering (Top-10 recall 0.446 against 0.527). The judgement step is
166
- * not a free improvement, and the earlier sets where it helped did not establish that it always
167
- * would.
168
- */
169
- export declare const WINDOW_GUARANTEE_GAIN: {
170
- readonly sameCases: {
171
- readonly corpus: "reference-gemini-cli, the eight cases of A-SPEC-427";
172
- readonly truthTotal: 19;
173
- readonly inPoolBefore: 9;
174
- readonly inPoolAfter: 14;
175
- readonly casesWithNothingBefore: 2;
176
- readonly casesWithNothingAfter: 1;
177
- readonly lexTop10BuriedBefore: 5;
178
- readonly lexTop10BuriedAfter: 0;
179
- };
180
- readonly freshSet: {
181
- readonly corpus: "reference-gemini-cli, eight cases whose truth was unread until the picks were frozen";
182
- readonly cases: 8;
183
- readonly harnessAlone: {
184
- readonly top1: 0.25;
185
- readonly top5Recall: 0.286;
186
- readonly top10Recall: 0.527;
187
- readonly top10Precision: 0.125;
188
- };
189
- readonly grepJudged: {
190
- readonly top1: 0.25;
191
- readonly top5Recall: 0.39;
192
- readonly top10Recall: 0.595;
193
- readonly top10Precision: 0.112;
194
- };
195
- readonly harnessJudged: {
196
- readonly top1: 0.125;
197
- readonly top5Recall: 0.348;
198
- readonly top10Recall: 0.446;
199
- readonly top10Precision: 0.1;
200
- };
201
- readonly poolCoverage: {
202
- readonly inPool: 13;
203
- readonly truthTotal: 18;
204
- readonly casesWithNothing: 1;
205
- };
206
- };
207
- readonly earlierSetHarnessAloneTop10Recall: 0.268;
208
- readonly caveat: "두 세트는 커밋이 다르므로 직접 비교하지 않는다 — 새 세트가 더 쉽다(하네스 단독 0.527 대 0.268). not comparable.";
209
- };