@hassangameryt/freeflow 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (690) hide show
  1. package/.claude-plugin/marketplace.json +4 -4
  2. package/.claude-plugin/plugin.json +5 -2
  3. package/.codex-plugin/plugin.json +8 -9
  4. package/README.md +190 -128
  5. package/assets/feedback-loop.png +0 -0
  6. package/assets/freeflow_workflow_map.png +0 -0
  7. package/assets/workflow.png +0 -0
  8. package/command-surface.json +51 -7
  9. package/hooks/freeflow-runtime-context.mjs +264 -136
  10. package/hooks/tests/check-runtime-context-hook.sh +223 -0
  11. package/package.json +23 -24
  12. package/pi-extension/dist/index.js +264 -80
  13. package/pi-extension/dist/integrations/mcp-config.js +29 -0
  14. package/pi-extension/dist/output-router/host-producer-identification.js +232 -0
  15. package/pi-extension/dist/output-router/native-safety-net.js +216 -0
  16. package/pi-extension/dist/output-router/observed-tool-routing.js +122 -0
  17. package/pi-extension/dist/output-router/renderers.js +1023 -0
  18. package/pi-extension/dist/output-router/router-tools.js +554 -0
  19. package/pi-extension/dist/output-router/schemas.js +383 -0
  20. package/pi-extension/dist/output-router/status.js +744 -0
  21. package/pi-extension/dist/output-router/utils.js +502 -0
  22. package/pi-extension/dist/runtime/runtime-context.js +798 -0
  23. package/pi-extension/dist/settings/settings-tui.js +314 -0
  24. package/pi-extension/dist/settings/settings-ui.js +1487 -0
  25. package/pi-extension/freeflow/index.js +1 -0
  26. package/pi-extension/src/index.ts +230 -56
  27. package/pi-extension/src/{host-producer-identification.ts → output-router/host-producer-identification.ts} +37 -7
  28. package/pi-extension/src/{native-safety-net.ts → output-router/native-safety-net.ts} +7 -11
  29. package/pi-extension/src/{observed-tool-routing.ts → output-router/observed-tool-routing.ts} +19 -11
  30. package/pi-extension/src/{renderers.ts → output-router/renderers.ts} +246 -55
  31. package/pi-extension/src/{router-tools.ts → output-router/router-tools.ts} +130 -29
  32. package/pi-extension/src/{schemas.ts → output-router/schemas.ts} +171 -43
  33. package/pi-extension/src/output-router/status.ts +791 -0
  34. package/pi-extension/src/{utils.ts → output-router/utils.ts} +80 -18
  35. package/pi-extension/src/runtime/runtime-context.ts +895 -0
  36. package/pi-extension/src/settings/settings-tui.ts +408 -0
  37. package/pi-extension/src/settings/settings-ui.ts +1730 -0
  38. package/pi-extension/tests/integration/pi-extension.test.js +4018 -0
  39. package/pi-extension/tests/output-router/observed-producers.test.js +132 -0
  40. package/pi-extension/tests/output-router/search.test.js +722 -0
  41. package/pi-extension/tests/settings/settings-tui.test.js +243 -0
  42. package/router/dist/benchmarks/benchmark-harness.d.ts +16 -13
  43. package/router/dist/benchmarks/benchmark-harness.js +62 -62
  44. package/router/dist/benchmarks/benchmarks.d.ts +80 -73
  45. package/router/dist/benchmarks/benchmarks.js +860 -761
  46. package/router/dist/benchmarks/codex-qa-benchmarks.d.ts +71 -67
  47. package/router/dist/benchmarks/codex-qa-benchmarks.js +429 -390
  48. package/router/dist/benchmarks/command-benchmarks.d.ts +107 -92
  49. package/router/dist/benchmarks/command-benchmarks.js +616 -571
  50. package/router/dist/benchmarks/context-mode-normalized-benchmarks.d.ts +64 -56
  51. package/router/dist/benchmarks/context-mode-normalized-benchmarks.js +679 -613
  52. package/router/dist/benchmarks/context-mode-real-deep-benchmark.d.ts +103 -95
  53. package/router/dist/benchmarks/context-mode-real-deep-benchmark.js +1188 -1103
  54. package/router/dist/benchmarks/context-saving-baseline.d.ts +47 -41
  55. package/router/dist/benchmarks/context-saving-baseline.js +405 -374
  56. package/router/dist/benchmarks/index-benchmarks.d.ts +74 -69
  57. package/router/dist/benchmarks/index-benchmarks.js +761 -661
  58. package/router/dist/benchmarks/storage-policy-benchmarks.d.ts +80 -69
  59. package/router/dist/benchmarks/storage-policy-benchmarks.js +549 -469
  60. package/router/dist/benchmarks/vault-index-benchmarks.d.ts +43 -34
  61. package/router/dist/benchmarks/vault-index-benchmarks.js +294 -229
  62. package/router/dist/config/config.d.ts +61 -54
  63. package/router/dist/config/config.js +551 -372
  64. package/router/dist/config/router-contract.d.ts +2 -2
  65. package/router/dist/config/router-contract.js +38 -38
  66. package/router/dist/config/schema.d.ts +19 -10
  67. package/router/dist/config/schema.js +586 -538
  68. package/router/dist/config/types.d.ts +413 -361
  69. package/router/dist/config/types.js +19 -8
  70. package/router/dist/evidence/bounded-evidence.d.ts +26 -26
  71. package/router/dist/evidence/bounded-evidence.js +265 -252
  72. package/router/dist/evidence/evidence-range-selector.d.ts +16 -14
  73. package/router/dist/evidence/evidence-range-selector.js +192 -191
  74. package/router/dist/evidence/evidence-search.d.ts +18 -16
  75. package/router/dist/evidence/evidence-search.js +401 -377
  76. package/router/dist/evidence/evidence.d.ts +17 -17
  77. package/router/dist/evidence/evidence.js +163 -159
  78. package/router/dist/evidence/failure-contracts.d.ts +25 -15
  79. package/router/dist/evidence/failure-contracts.js +132 -126
  80. package/router/dist/evidence/line-ranges.d.ts +17 -15
  81. package/router/dist/evidence/line-ranges.js +18 -18
  82. package/router/dist/experiments/local-index.d.ts +47 -41
  83. package/router/dist/experiments/local-index.js +477 -458
  84. package/router/dist/local/local-traversal.d.ts +18 -0
  85. package/router/dist/local/local-traversal.js +334 -0
  86. package/router/dist/processing/engine.d.ts +94 -70
  87. package/router/dist/processing/engine.js +462 -364
  88. package/router/dist/processing/reducers.d.ts +208 -194
  89. package/router/dist/processing/reducers.js +1434 -1256
  90. package/router/dist/processing/renderers.d.ts +24 -18
  91. package/router/dist/processing/renderers.js +161 -152
  92. package/router/dist/processing/scripts.d.ts +80 -67
  93. package/router/dist/processing/scripts.js +343 -290
  94. package/router/dist/repo/repo-traversal.d.ts +9 -9
  95. package/router/dist/repo/repo-traversal.js +205 -202
  96. package/router/dist/routing/observed-reducers.d.ts +25 -19
  97. package/router/dist/routing/observed-reducers.js +559 -545
  98. package/router/dist/routing/observed-routing.d.ts +23 -14
  99. package/router/dist/routing/observed-routing.js +319 -318
  100. package/router/dist/routing/parsers.d.ts +10 -10
  101. package/router/dist/routing/parsers.js +454 -406
  102. package/router/dist/routing/run-filters.d.ts +45 -41
  103. package/router/dist/routing/run-filters.js +215 -207
  104. package/router/dist/routing/run-reducers.d.ts +45 -26
  105. package/router/dist/routing/run-reducers.js +102 -90
  106. package/router/dist/sandbox/adapter-roots.d.ts +34 -26
  107. package/router/dist/sandbox/adapter-roots.js +43 -41
  108. package/router/dist/sandbox/eryx-python-adapter.d.ts +9 -5
  109. package/router/dist/sandbox/eryx-python-adapter.js +659 -588
  110. package/router/dist/sandbox/jq-wasm-adapter.d.ts +3 -3
  111. package/router/dist/sandbox/jq-wasm-adapter.js +319 -311
  112. package/router/dist/sandbox/quickjs-wasi-adapter.d.ts +9 -5
  113. package/router/dist/sandbox/quickjs-wasi-adapter.js +336 -333
  114. package/router/dist/sandbox/script-sandbox.d.ts +105 -84
  115. package/router/dist/sandbox/script-sandbox.js +294 -260
  116. package/router/dist/setup/script-transform-adapters.d.ts +23 -19
  117. package/router/dist/setup/script-transform-adapters.js +223 -204
  118. package/router/dist/tools/batch.d.ts +27 -16
  119. package/router/dist/tools/batch.js +493 -416
  120. package/router/dist/tools/run.d.ts +47 -35
  121. package/router/dist/tools/run.js +1171 -1086
  122. package/router/dist/tools/search.d.ts +53 -31
  123. package/router/dist/tools/search.js +1224 -1065
  124. package/router/dist/transform/engine.d.ts +121 -96
  125. package/router/dist/transform/engine.js +1853 -1736
  126. package/router/dist/vault/vault-index.d.ts +97 -78
  127. package/router/dist/vault/vault-index.js +347 -342
  128. package/router/dist/vault/vault.d.ts +109 -74
  129. package/router/dist/vault/vault.js +528 -533
  130. package/runtime/interaction-contract.md +16 -0
  131. package/skills/bypass/SKILL.md +52 -38
  132. package/skills/commit-work/SKILL.md +54 -46
  133. package/skills/commit-work/references/staging-decisions.md +46 -24
  134. package/skills/decision-gate/SKILL.md +48 -0
  135. package/skills/design-for-depth/SKILL.md +75 -59
  136. package/skills/design-for-depth/references/design-pressure-signals.md +56 -7
  137. package/skills/design-for-depth/references/interface-design-loop.md +156 -0
  138. package/skills/design-for-depth/references/software-design-philosophy.md +32 -3
  139. package/skills/diagnose-failure/SKILL.md +65 -60
  140. package/skills/diagnose-failure/references/diagnostic-loop-catalog.md +36 -0
  141. package/skills/diagnose-failure/references/flaky-and-performance.md +30 -18
  142. package/skills/discuss/SKILL.md +121 -0
  143. package/skills/discuss/references/checkpoints.md +66 -0
  144. package/skills/evaluate-skill/SKILL.md +62 -53
  145. package/skills/evaluate-skill/references/definition-schema.md +345 -0
  146. package/skills/evaluate-skill/references/evaluation-design.md +86 -0
  147. package/skills/evaluate-skill/references/execution-and-evidence.md +85 -0
  148. package/skills/evaluate-skill/references/review-and-revision.md +51 -0
  149. package/skills/evaluate-skill/scripts/lib/definitions.mjs +296 -0
  150. package/skills/evaluate-skill/scripts/lib/evidence.mjs +46 -0
  151. package/skills/evaluate-skill/scripts/lib/grade-comparison.mjs +60 -0
  152. package/skills/evaluate-skill/scripts/lib/grade-interaction.mjs +148 -0
  153. package/skills/evaluate-skill/scripts/lib/grade-support.mjs +126 -0
  154. package/skills/evaluate-skill/scripts/lib/grade-workspace.mjs +192 -0
  155. package/skills/evaluate-skill/scripts/lib/grade.mjs +56 -0
  156. package/skills/evaluate-skill/scripts/lib/pi.mjs +489 -0
  157. package/skills/evaluate-skill/scripts/lib/process.mjs +79 -0
  158. package/skills/evaluate-skill/scripts/lib/rpc.mjs +444 -0
  159. package/skills/evaluate-skill/scripts/lib/runner.mjs +346 -0
  160. package/skills/evaluate-skill/scripts/lib/sandbox.mjs +409 -0
  161. package/skills/evaluate-skill/scripts/lib/view.mjs +337 -0
  162. package/skills/evaluate-skill/scripts/pi-guard.mjs +164 -0
  163. package/skills/evaluate-skill/scripts/skill-eval.mjs +101 -0
  164. package/skills/execute-work/SKILL.md +99 -0
  165. package/skills/execute-work/references/code-practices.md +78 -0
  166. package/skills/execute-work/references/execution-loop.md +77 -0
  167. package/skills/finish-branch/SKILL.md +117 -0
  168. package/skills/finish-branch/references/integration-options.md +80 -0
  169. package/skills/handoff/SKILL.md +65 -39
  170. package/skills/handoff/references/templates.md +104 -27
  171. package/skills/launch-work/SKILL.md +116 -0
  172. package/skills/launch-work/references/launch-readiness.md +85 -0
  173. package/skills/migration-work/SKILL.md +115 -0
  174. package/skills/migration-work/references/migration-lifecycle.md +82 -0
  175. package/skills/mode-contract/SKILL.md +50 -52
  176. package/skills/output-router/SKILL.md +21 -17
  177. package/skills/output-router/references/safety-policy.md +1 -1
  178. package/skills/release-work/SKILL.md +118 -0
  179. package/skills/release-work/references/release-evidence.md +78 -0
  180. package/skills/review-artifact/SKILL.md +77 -69
  181. package/skills/review-artifact/references/reviewer-prompt.md +133 -42
  182. package/skills/review-work/SKILL.md +82 -57
  183. package/skills/review-work/references/reviewer-prompt.md +97 -60
  184. package/skills/review-work/references/security-risk-lens.md +67 -0
  185. package/skills/setup-freeflow/SKILL.md +58 -66
  186. package/skills/setup-freeflow/references/activation-contract.md +89 -69
  187. package/skills/setup-freeflow/references/host-setup.md +54 -17
  188. package/skills/setup-freeflow/references/output-router-setup.md +62 -54
  189. package/skills/simplify-code/SKILL.md +80 -0
  190. package/skills/simplify-code/references/simplification-patterns.md +70 -0
  191. package/skills/tdd/SKILL.md +96 -0
  192. package/skills/tdd/references/test-design.md +96 -0
  193. package/skills/track-work/SKILL.md +84 -0
  194. package/skills/track-work/references/working-record-schema.md +378 -0
  195. package/skills/verify-work/SKILL.md +59 -38
  196. package/skills/verify-work/references/browser-runtime-evidence.md +55 -0
  197. package/skills/verify-work/references/integration-evidence.md +51 -0
  198. package/skills/verify-work/references/performance-evidence.md +57 -0
  199. package/skills/workflow/SKILL.md +53 -59
  200. package/skills/workflow/references/domain-skill-composition.md +57 -0
  201. package/skills/workflow/references/workflow-loop.md +92 -0
  202. package/skills/write-plan/SKILL.md +85 -58
  203. package/skills/write-plan/references/plan-shapes.md +132 -46
  204. package/skills/write-skill/SKILL.md +82 -51
  205. package/skills/write-skill/references/activation-boundaries.md +62 -0
  206. package/skills/write-skill/references/agent-first-instructions.md +87 -0
  207. package/skills/write-skill/references/progressive-disclosure.md +63 -0
  208. package/skills/write-skill/scripts/lib/skill-author.mjs +653 -0
  209. package/skills/write-skill/scripts/skill-author.mjs +104 -0
  210. package/skills/write-spec/SKILL.md +65 -54
  211. package/skills/write-spec/references/artifact-standards.md +60 -41
  212. package/skills/write-spec/references/decision-records.md +77 -0
  213. package/skills/write-spec/references/spec-shapes.md +119 -60
  214. package/evals/README.md +0 -32
  215. package/evals/fixtures/copy-conventions.md +0 -31
  216. package/evals/fixtures/output-router/README.md +0 -12
  217. package/evals/fixtures/output-router/failed-command-output.txt +0 -9
  218. package/evals/fixtures/output-router/large-router-manual.md +0 -63
  219. package/evals/fixtures/output-router/native-large-output.txt +0 -40
  220. package/evals/fixtures/output-router/noisy-test-output.txt +0 -30
  221. package/evals/fixtures/output-router/verification-output.txt +0 -8
  222. package/evals/fixtures/tiny-artifact-review-app/README.md +0 -5
  223. package/evals/fixtures/tiny-artifact-review-app/docs/billing-policy.md +0 -10
  224. package/evals/fixtures/tiny-artifact-review-app/docs/specs/billing-webhook-api-draft.md +0 -21
  225. package/evals/fixtures/tiny-artifact-review-app/docs/specs/failed-payment-grace-period.md +0 -24
  226. package/evals/fixtures/tiny-artifact-review-app/docs/specs/immediate-downgrade.md +0 -17
  227. package/evals/fixtures/tiny-artifact-review-app/tests/billing.test.md +0 -8
  228. package/evals/fixtures/tiny-commit-ambiguous-app/.freeflow-eval-setup.sh +0 -15
  229. package/evals/fixtures/tiny-commit-ambiguous-app/README.md +0 -3
  230. package/evals/fixtures/tiny-commit-ambiguous-app/src/billing.ts +0 -5
  231. package/evals/fixtures/tiny-commit-ambiguous-app/src/settings-page.ts +0 -1
  232. package/evals/fixtures/tiny-commit-ambiguous-app/tests/settings.test.md +0 -5
  233. package/evals/fixtures/tiny-commit-app/.freeflow-eval-setup.sh +0 -14
  234. package/evals/fixtures/tiny-commit-app/README.md +0 -3
  235. package/evals/fixtures/tiny-commit-app/docs/specs/settings-title.md +0 -5
  236. package/evals/fixtures/tiny-commit-app/src/billing.ts +0 -5
  237. package/evals/fixtures/tiny-commit-app/src/settings-page.ts +0 -1
  238. package/evals/fixtures/tiny-commit-app/tests/settings.test.md +0 -5
  239. package/evals/fixtures/tiny-commit-everything-app/.freeflow-eval-setup.sh +0 -15
  240. package/evals/fixtures/tiny-commit-everything-app/README.md +0 -3
  241. package/evals/fixtures/tiny-commit-everything-app/docs/specs/settings-title.md +0 -5
  242. package/evals/fixtures/tiny-commit-everything-app/src/billing.ts +0 -5
  243. package/evals/fixtures/tiny-commit-everything-app/src/settings-page.ts +0 -1
  244. package/evals/fixtures/tiny-commit-everything-app/tests/settings.test.md +0 -5
  245. package/evals/fixtures/tiny-commit-staged-app/.freeflow-eval-setup.sh +0 -15
  246. package/evals/fixtures/tiny-commit-staged-app/README.md +0 -3
  247. package/evals/fixtures/tiny-commit-staged-app/docs/specs/settings-title.md +0 -5
  248. package/evals/fixtures/tiny-commit-staged-app/src/billing.ts +0 -5
  249. package/evals/fixtures/tiny-commit-staged-app/src/settings-page.ts +0 -1
  250. package/evals/fixtures/tiny-commit-staged-app/tests/settings.test.md +0 -5
  251. package/evals/fixtures/tiny-commit-staged-mixed-app/.freeflow-eval-setup.sh +0 -15
  252. package/evals/fixtures/tiny-commit-staged-mixed-app/README.md +0 -3
  253. package/evals/fixtures/tiny-commit-staged-mixed-app/docs/billing-policy.md +0 -3
  254. package/evals/fixtures/tiny-commit-staged-mixed-app/docs/release-notes.md +0 -3
  255. package/evals/fixtures/tiny-commit-staged-mixed-app/src/billing.ts +0 -5
  256. package/evals/fixtures/tiny-dashboard-stale-app/README.md +0 -11
  257. package/evals/fixtures/tiny-dashboard-stale-app/src/api-client.ts +0 -13
  258. package/evals/fixtures/tiny-dashboard-stale-app/src/dashboard-data.ts +0 -19
  259. package/evals/fixtures/tiny-dashboard-stale-app/tests/dashboard-data.test.md +0 -9
  260. package/evals/fixtures/tiny-decision-destination-app/README.md +0 -6
  261. package/evals/fixtures/tiny-decision-destination-app/src/risk.ts +0 -8
  262. package/evals/fixtures/tiny-decision-destination-app/tests/risk.test.md +0 -4
  263. package/evals/fixtures/tiny-design-depth-app/README.md +0 -15
  264. package/evals/fixtures/tiny-design-depth-app/src/routes.ts +0 -25
  265. package/evals/fixtures/tiny-design-depth-app/src/sendEmail.ts +0 -3
  266. package/evals/fixtures/tiny-design-depth-app/src/sendSms.ts +0 -3
  267. package/evals/fixtures/tiny-design-depth-app/src/telemetry.ts +0 -3
  268. package/evals/fixtures/tiny-design-depth-app/tests/routes.test.md +0 -12
  269. package/evals/fixtures/tiny-eval-method-app/README.md +0 -8
  270. package/evals/fixtures/tiny-eval-method-app/evals/reports/review-pr-eval-result.md +0 -5
  271. package/evals/fixtures/tiny-eval-method-app/evals/reports/triage-question-failure.md +0 -17
  272. package/evals/fixtures/tiny-eval-method-app/evals/runs/review-pr-with-skill-output.diff +0 -11
  273. package/evals/fixtures/tiny-eval-method-app/evals/runs/review-pr-with-skill-output.md +0 -3
  274. package/evals/fixtures/tiny-eval-method-app/skills/review-pr/SKILL.md +0 -10
  275. package/evals/fixtures/tiny-eval-method-app/skills/triage-question/SKILL.md +0 -10
  276. package/evals/fixtures/tiny-execute-plan-app/README.md +0 -5
  277. package/evals/fixtures/tiny-execute-plan-app/checks/settings-and-billing-check.sh +0 -5
  278. package/evals/fixtures/tiny-execute-plan-app/docs/billing-policy.md +0 -10
  279. package/evals/fixtures/tiny-execute-plan-app/docs/plans/immediate-downgrade-plan.md +0 -11
  280. package/evals/fixtures/tiny-execute-plan-app/docs/plans/settings-title-plan.md +0 -12
  281. package/evals/fixtures/tiny-execute-plan-app/docs/plans/settings-title-with-bad-check.md +0 -15
  282. package/evals/fixtures/tiny-execute-plan-app/docs/plans/unverified-billing-plan.md +0 -9
  283. package/evals/fixtures/tiny-execute-plan-app/docs/specs/failed-payment-grace-period.md +0 -9
  284. package/evals/fixtures/tiny-execute-plan-app/docs/specs/settings-title.md +0 -9
  285. package/evals/fixtures/tiny-execute-plan-app/src/billing.ts +0 -7
  286. package/evals/fixtures/tiny-execute-plan-app/src/settings-page.ts +0 -1
  287. package/evals/fixtures/tiny-execute-plan-app/tests/billing.test.md +0 -7
  288. package/evals/fixtures/tiny-execute-plan-app/tests/settings.test.md +0 -5
  289. package/evals/fixtures/tiny-execute-review-loop-app/README.md +0 -5
  290. package/evals/fixtures/tiny-execute-review-loop-app/docs/billing-policy.md +0 -9
  291. package/evals/fixtures/tiny-execute-review-loop-app/docs/plans/settings-title-plan.md +0 -12
  292. package/evals/fixtures/tiny-execute-review-loop-app/docs/review/previous-agent-summary.md +0 -15
  293. package/evals/fixtures/tiny-execute-review-loop-app/docs/review/review-pass-1-non-pass.md +0 -11
  294. package/evals/fixtures/tiny-execute-review-loop-app/docs/specs/settings-title.md +0 -9
  295. package/evals/fixtures/tiny-execute-review-loop-app/src/billing.ts +0 -7
  296. package/evals/fixtures/tiny-execute-review-loop-app/src/settings-page.ts +0 -1
  297. package/evals/fixtures/tiny-execute-review-loop-app/tests/billing.test.md +0 -7
  298. package/evals/fixtures/tiny-execute-review-loop-app/tests/settings.test.md +0 -5
  299. package/evals/fixtures/tiny-flaky-session-app/README.md +0 -10
  300. package/evals/fixtures/tiny-flaky-session-app/src/session-timeout.ts +0 -13
  301. package/evals/fixtures/tiny-flaky-session-app/tests/session-timeout.test.md +0 -8
  302. package/evals/fixtures/tiny-handoff-app/README.md +0 -5
  303. package/evals/fixtures/tiny-handoff-app/docs/billing-policy.md +0 -10
  304. package/evals/fixtures/tiny-handoff-app/docs/handoffs/old-agent-handoff.md +0 -8
  305. package/evals/fixtures/tiny-handoff-app/docs/plans/billing-retry-plan.md +0 -11
  306. package/evals/fixtures/tiny-handoff-app/src/billing.ts +0 -7
  307. package/evals/fixtures/tiny-handoff-report-app/README.md +0 -3
  308. package/evals/fixtures/tiny-handoff-report-app/evals/runs/handoff-3/hof-003/with-skill-output.diff +0 -1
  309. package/evals/fixtures/tiny-handoff-report-app/evals/runs/handoff-3/hof-003/with-skill-output.md +0 -1
  310. package/evals/fixtures/tiny-handoff-report-app/evals/scores/hof-003-score.txt +0 -4
  311. package/evals/fixtures/tiny-plan-app/README.md +0 -5
  312. package/evals/fixtures/tiny-plan-app/docs/billing-policy.md +0 -10
  313. package/evals/fixtures/tiny-plan-app/docs/specs/failed-payment-grace-period.md +0 -25
  314. package/evals/fixtures/tiny-plan-app/src/billing.ts +0 -17
  315. package/evals/fixtures/tiny-plan-app/src/settings-page.ts +0 -1
  316. package/evals/fixtures/tiny-plan-app/tests/billing.test.md +0 -8
  317. package/evals/fixtures/tiny-plan-app/tests/settings.test.md +0 -5
  318. package/evals/fixtures/tiny-plan-strict-api-app/README.md +0 -3
  319. package/evals/fixtures/tiny-plan-strict-api-app/docs/billing-policy.md +0 -5
  320. package/evals/fixtures/tiny-plan-strict-api-app/docs/specs/billing-webhook-api.md +0 -25
  321. package/evals/fixtures/tiny-plan-strict-api-app/src/billing.ts +0 -5
  322. package/evals/fixtures/tiny-plan-strict-api-app/src/webhooks.ts +0 -3
  323. package/evals/fixtures/tiny-plan-strict-api-app/tests/billing-webhook.test.md +0 -8
  324. package/evals/fixtures/tiny-post-setup-invalid-config-app/.freeflow/config.json +0 -3
  325. package/evals/fixtures/tiny-post-setup-invalid-config-app/AGENTS.md +0 -15
  326. package/evals/fixtures/tiny-post-setup-invalid-config-app/README.md +0 -16
  327. package/evals/fixtures/tiny-post-setup-invalid-config-app/docs/auth-review.md +0 -15
  328. package/evals/fixtures/tiny-post-setup-invalid-config-app/docs/billing-policy.md +0 -10
  329. package/evals/fixtures/tiny-post-setup-invalid-config-app/docs/copy-conventions.md +0 -15
  330. package/evals/fixtures/tiny-post-setup-invalid-config-app/src/auth-middleware.ts +0 -12
  331. package/evals/fixtures/tiny-post-setup-invalid-config-app/src/billing.ts +0 -20
  332. package/evals/fixtures/tiny-post-setup-invalid-config-app/src/login-copy.ts +0 -1
  333. package/evals/fixtures/tiny-post-setup-invalid-config-app/src/settings-page.ts +0 -1
  334. package/evals/fixtures/tiny-post-setup-invalid-config-app/tests/auth-middleware.test.md +0 -8
  335. package/evals/fixtures/tiny-post-setup-invalid-config-app/tests/billing.test.md +0 -8
  336. package/evals/fixtures/tiny-post-setup-invalid-config-app/tests/copy.test.md +0 -7
  337. package/evals/fixtures/tiny-post-setup-missing-config-app/AGENTS.md +0 -15
  338. package/evals/fixtures/tiny-post-setup-missing-config-app/README.md +0 -16
  339. package/evals/fixtures/tiny-post-setup-missing-config-app/docs/auth-review.md +0 -15
  340. package/evals/fixtures/tiny-post-setup-missing-config-app/docs/billing-policy.md +0 -10
  341. package/evals/fixtures/tiny-post-setup-missing-config-app/docs/copy-conventions.md +0 -15
  342. package/evals/fixtures/tiny-post-setup-missing-config-app/src/auth-middleware.ts +0 -12
  343. package/evals/fixtures/tiny-post-setup-missing-config-app/src/billing.ts +0 -20
  344. package/evals/fixtures/tiny-post-setup-missing-config-app/src/login-copy.ts +0 -1
  345. package/evals/fixtures/tiny-post-setup-missing-config-app/src/settings-page.ts +0 -1
  346. package/evals/fixtures/tiny-post-setup-missing-config-app/tests/auth-middleware.test.md +0 -8
  347. package/evals/fixtures/tiny-post-setup-missing-config-app/tests/billing.test.md +0 -8
  348. package/evals/fixtures/tiny-post-setup-missing-config-app/tests/copy.test.md +0 -7
  349. package/evals/fixtures/tiny-post-setup-source-conflict-app/.freeflow/config.json +0 -3
  350. package/evals/fixtures/tiny-post-setup-source-conflict-app/AGENTS.md +0 -15
  351. package/evals/fixtures/tiny-post-setup-source-conflict-app/README.md +0 -16
  352. package/evals/fixtures/tiny-post-setup-source-conflict-app/docs/auth-review.md +0 -15
  353. package/evals/fixtures/tiny-post-setup-source-conflict-app/docs/billing-policy.md +0 -10
  354. package/evals/fixtures/tiny-post-setup-source-conflict-app/docs/copy-conventions.md +0 -15
  355. package/evals/fixtures/tiny-post-setup-source-conflict-app/src/auth-middleware.ts +0 -12
  356. package/evals/fixtures/tiny-post-setup-source-conflict-app/src/billing.ts +0 -20
  357. package/evals/fixtures/tiny-post-setup-source-conflict-app/src/login-copy.ts +0 -1
  358. package/evals/fixtures/tiny-post-setup-source-conflict-app/src/settings-page.ts +0 -1
  359. package/evals/fixtures/tiny-post-setup-source-conflict-app/tests/auth-middleware.test.md +0 -8
  360. package/evals/fixtures/tiny-post-setup-source-conflict-app/tests/billing.test.md +0 -8
  361. package/evals/fixtures/tiny-post-setup-source-conflict-app/tests/copy.test.md +0 -7
  362. package/evals/fixtures/tiny-report-performance-app/README.md +0 -11
  363. package/evals/fixtures/tiny-report-performance-app/docs/performance.md +0 -10
  364. package/evals/fixtures/tiny-report-performance-app/src/report-summary.ts +0 -29
  365. package/evals/fixtures/tiny-report-performance-app/tests/report-performance.test.md +0 -8
  366. package/evals/fixtures/tiny-review-request-app/README.md +0 -3
  367. package/evals/fixtures/tiny-review-request-app/docs/billing-policy.md +0 -5
  368. package/evals/fixtures/tiny-review-request-app/docs/plans/billing-webhook-api.md +0 -19
  369. package/evals/fixtures/tiny-review-request-app/docs/review/previous-agent-summary.md +0 -20
  370. package/evals/fixtures/tiny-review-request-app/docs/specs/billing-webhook-api.md +0 -25
  371. package/evals/fixtures/tiny-review-request-app/src/billing.ts +0 -11
  372. package/evals/fixtures/tiny-review-request-app/src/webhooks.ts +0 -19
  373. package/evals/fixtures/tiny-review-request-app/tests/billing-webhook.test.md +0 -8
  374. package/evals/fixtures/tiny-review-work-app/README.md +0 -5
  375. package/evals/fixtures/tiny-review-work-app/docs/billing-policy.md +0 -9
  376. package/evals/fixtures/tiny-review-work-app/docs/plans/settings-title-plan.md +0 -12
  377. package/evals/fixtures/tiny-review-work-app/docs/review/previous-agent-summary.md +0 -15
  378. package/evals/fixtures/tiny-review-work-app/docs/specs/settings-title.md +0 -9
  379. package/evals/fixtures/tiny-review-work-app/src/billing.ts +0 -7
  380. package/evals/fixtures/tiny-review-work-app/src/settings-page.ts +0 -1
  381. package/evals/fixtures/tiny-review-work-app/tests/billing.test.md +0 -7
  382. package/evals/fixtures/tiny-review-work-app/tests/settings.test.md +0 -5
  383. package/evals/fixtures/tiny-saas-app/README.md +0 -16
  384. package/evals/fixtures/tiny-saas-app/docs/auth-review.md +0 -15
  385. package/evals/fixtures/tiny-saas-app/docs/billing-policy.md +0 -10
  386. package/evals/fixtures/tiny-saas-app/docs/copy-conventions.md +0 -15
  387. package/evals/fixtures/tiny-saas-app/docs/specs/settings-title.md +0 -13
  388. package/evals/fixtures/tiny-saas-app/src/auth-middleware.ts +0 -12
  389. package/evals/fixtures/tiny-saas-app/src/billing.ts +0 -20
  390. package/evals/fixtures/tiny-saas-app/src/login-copy.ts +0 -1
  391. package/evals/fixtures/tiny-saas-app/src/settings-page.ts +0 -1
  392. package/evals/fixtures/tiny-saas-app/tests/auth-middleware.test.md +0 -8
  393. package/evals/fixtures/tiny-saas-app/tests/billing.test.md +0 -8
  394. package/evals/fixtures/tiny-saas-app/tests/copy.test.md +0 -7
  395. package/evals/fixtures/tiny-setup-agents-app/AGENTS.md +0 -6
  396. package/evals/fixtures/tiny-setup-both-app/AGENTS.md +0 -5
  397. package/evals/fixtures/tiny-setup-both-app/CLAUDE.md +0 -5
  398. package/evals/fixtures/tiny-setup-claude-app/CLAUDE.md +0 -6
  399. package/evals/fixtures/tiny-setup-codex-rules-app/.codex/rules/shell.rules +0 -5
  400. package/evals/fixtures/tiny-setup-codex-rules-app/AGENTS.md +0 -6
  401. package/evals/fixtures/tiny-setup-conflict-app/AGENTS.md +0 -6
  402. package/evals/fixtures/tiny-setup-existing-block-app/AGENTS.md +0 -14
  403. package/evals/fixtures/tiny-setup-four-rules-app/CLAUDE.md +0 -6
  404. package/evals/fixtures/tiny-skill-dev-app/README.md +0 -10
  405. package/evals/fixtures/tiny-skill-dev-app/evals/prompts/review-pr-001.txt +0 -8
  406. package/evals/fixtures/tiny-skill-dev-app/evals/reports/review-pr-failure.md +0 -17
  407. package/evals/fixtures/tiny-skill-dev-app/skills/review-pr/SKILL.md +0 -13
  408. package/evals/fixtures/tiny-team-app/README.md +0 -5
  409. package/evals/fixtures/tiny-team-app/docs/product.md +0 -6
  410. package/evals/fixtures/tiny-team-app/src/teams.ts +0 -9
  411. package/evals/fixtures/tiny-team-app/src/users.ts +0 -5
  412. package/evals/fixtures/tiny-team-app/tests/teams.test.md +0 -7
  413. package/evals/fixtures/tiny-verify-app/README.md +0 -5
  414. package/evals/fixtures/tiny-verify-app/src/settings-nav.ts +0 -1
  415. package/evals/fixtures/tiny-verify-app/src/settings-page.ts +0 -1
  416. package/evals/fixtures/tiny-verify-app/tests/settings.test.md +0 -7
  417. package/evals/prompts/aon-001.txt +0 -2
  418. package/evals/prompts/byp-001.txt +0 -2
  419. package/evals/prompts/byp-002.txt +0 -2
  420. package/evals/prompts/cap-001.txt +0 -3
  421. package/evals/prompts/cap-002.txt +0 -3
  422. package/evals/prompts/cmd-001.txt +0 -3
  423. package/evals/prompts/cmd-002.txt +0 -3
  424. package/evals/prompts/cmd-003.txt +0 -6
  425. package/evals/prompts/cmd-004.txt +0 -5
  426. package/evals/prompts/cmd-005.txt +0 -5
  427. package/evals/prompts/cmd-006.txt +0 -5
  428. package/evals/prompts/cmd-007.txt +0 -5
  429. package/evals/prompts/cmd-008.txt +0 -5
  430. package/evals/prompts/cmd-009.txt +0 -8
  431. package/evals/prompts/cmd-010.txt +0 -5
  432. package/evals/prompts/cmd-011.txt +0 -5
  433. package/evals/prompts/cmd-012.txt +0 -5
  434. package/evals/prompts/cmd-013.txt +0 -5
  435. package/evals/prompts/cmd-014.txt +0 -7
  436. package/evals/prompts/cmd-015.txt +0 -7
  437. package/evals/prompts/cmd-016.txt +0 -5
  438. package/evals/prompts/cmt-001.txt +0 -1
  439. package/evals/prompts/cmt-002.txt +0 -1
  440. package/evals/prompts/cmt-003.txt +0 -1
  441. package/evals/prompts/cmt-004.txt +0 -1
  442. package/evals/prompts/cmt-005.txt +0 -1
  443. package/evals/prompts/dfd-001.txt +0 -3
  444. package/evals/prompts/dia-001.txt +0 -3
  445. package/evals/prompts/dia-002.txt +0 -3
  446. package/evals/prompts/dia-003.txt +0 -3
  447. package/evals/prompts/dis-001.txt +0 -5
  448. package/evals/prompts/dis-002.txt +0 -5
  449. package/evals/prompts/esk-001.txt +0 -5
  450. package/evals/prompts/esk-002.txt +0 -5
  451. package/evals/prompts/esk-003.txt +0 -5
  452. package/evals/prompts/grc-001.txt +0 -3
  453. package/evals/prompts/hof-001.txt +0 -3
  454. package/evals/prompts/hof-002.txt +0 -1
  455. package/evals/prompts/hof-003.txt +0 -1
  456. package/evals/prompts/hof-004.txt +0 -1
  457. package/evals/prompts/hof-005.txt +0 -1
  458. package/evals/prompts/ivg-001.txt +0 -1
  459. package/evals/prompts/ivg-002.txt +0 -11
  460. package/evals/prompts/ivg-003.txt +0 -1
  461. package/evals/prompts/ivg-004.txt +0 -1
  462. package/evals/prompts/mode-001.txt +0 -3
  463. package/evals/prompts/mode-002.txt +0 -1
  464. package/evals/prompts/mode-003.txt +0 -3
  465. package/evals/prompts/mode-004.txt +0 -3
  466. package/evals/prompts/mode-005.txt +0 -3
  467. package/evals/prompts/mode-006.txt +0 -3
  468. package/evals/prompts/mode-007.txt +0 -3
  469. package/evals/prompts/otr-001.txt +0 -3
  470. package/evals/prompts/otr-002.txt +0 -3
  471. package/evals/prompts/rar-001.txt +0 -5
  472. package/evals/prompts/rar-002.txt +0 -3
  473. package/evals/prompts/rar-003.txt +0 -6
  474. package/evals/prompts/rar-004.txt +0 -15
  475. package/evals/prompts/rar-005.txt +0 -10
  476. package/evals/prompts/rbr-001.txt +0 -3
  477. package/evals/prompts/rbr-002.txt +0 -3
  478. package/evals/prompts/rbr-003.txt +0 -7
  479. package/evals/prompts/rev-001.txt +0 -3
  480. package/evals/prompts/rev-002.txt +0 -5
  481. package/evals/prompts/rev-003.txt +0 -6
  482. package/evals/prompts/rev-004.txt +0 -1
  483. package/evals/prompts/rev-005.txt +0 -15
  484. package/evals/prompts/rev-006.txt +0 -5
  485. package/evals/prompts/stp-001.txt +0 -1
  486. package/evals/prompts/stp-002.txt +0 -1
  487. package/evals/prompts/stp-003.txt +0 -1
  488. package/evals/prompts/stp-004.txt +0 -1
  489. package/evals/prompts/stp-005.txt +0 -1
  490. package/evals/prompts/stp-006.txt +0 -1
  491. package/evals/prompts/stp-007.txt +0 -1
  492. package/evals/prompts/stp-008.txt +0 -1
  493. package/evals/prompts/stp-009.txt +0 -1
  494. package/evals/prompts/stp-010.txt +0 -1
  495. package/evals/prompts/stp-011.txt +0 -1
  496. package/evals/prompts/vfy-003.txt +0 -1
  497. package/evals/prompts/wpl-001.txt +0 -3
  498. package/evals/prompts/wpl-002.txt +0 -5
  499. package/evals/prompts/wpl-003.txt +0 -3
  500. package/evals/prompts/wpl-004.txt +0 -5
  501. package/evals/prompts/wpl-005.txt +0 -1
  502. package/evals/prompts/wrc-001.txt +0 -10
  503. package/evals/prompts/wrc-002.txt +0 -3
  504. package/evals/prompts/wsk-001.txt +0 -6
  505. package/evals/prompts/wsp-001.txt +0 -5
  506. package/evals/prompts/wsp-002.txt +0 -15
  507. package/evals/prompts/wsp-003.txt +0 -3
  508. package/evals/prompts/wsp-004.txt +0 -15
  509. package/evals/prompts/wsp-005.txt +0 -5
  510. package/evals/prompts/wsp-006.txt +0 -12
  511. package/evals/prompts/wsp-007.txt +0 -9
  512. package/evals/prompts/wsp-008.txt +0 -13
  513. package/evals/prompts/xpl-001.txt +0 -1
  514. package/evals/prompts/xpl-002.txt +0 -3
  515. package/evals/prompts/xpl-003.txt +0 -3
  516. package/evals/prompts/xpl-004.txt +0 -3
  517. package/evals/prompts/xpl-005.txt +0 -5
  518. package/evals/prompts/xpl-006.txt +0 -3
  519. package/evals/registries/adversarial-evals.json +0 -39
  520. package/evals/registries/fixture-evals.json +0 -2513
  521. package/evals/registries/handoff-evals.json +0 -74
  522. package/evals/registries/legacy-evals.json +0 -147
  523. package/evals/registries/skill-evidence.json +0 -193
  524. package/evals/registries/verify-work-evals.json +0 -46
  525. package/evals/reports/acceptance/v0.1-acceptance-report.md +0 -130
  526. package/evals/reports/by-command-surface/command-surface-1-report.md +0 -45
  527. package/evals/reports/by-command-surface/command-surface-10-report.md +0 -76
  528. package/evals/reports/by-command-surface/command-surface-11-report.md +0 -77
  529. package/evals/reports/by-command-surface/command-surface-12-report.md +0 -91
  530. package/evals/reports/by-command-surface/command-surface-13-report.md +0 -64
  531. package/evals/reports/by-command-surface/command-surface-14-report.md +0 -66
  532. package/evals/reports/by-command-surface/command-surface-15-report.md +0 -124
  533. package/evals/reports/by-command-surface/command-surface-16-report.md +0 -39
  534. package/evals/reports/by-command-surface/command-surface-2-report.md +0 -80
  535. package/evals/reports/by-command-surface/command-surface-3-report.md +0 -86
  536. package/evals/reports/by-command-surface/command-surface-4-report.md +0 -76
  537. package/evals/reports/by-command-surface/command-surface-5-report.md +0 -80
  538. package/evals/reports/by-command-surface/command-surface-6-report.md +0 -74
  539. package/evals/reports/by-command-surface/command-surface-7-report.md +0 -84
  540. package/evals/reports/by-command-surface/command-surface-8-report.md +0 -79
  541. package/evals/reports/by-command-surface/command-surface-9-report.md +0 -79
  542. package/evals/reports/by-command-surface/command-surface-matrix.md +0 -91
  543. package/evals/reports/by-skill/bypass-1-report.md +0 -124
  544. package/evals/reports/by-skill/capture-decisions-1-report.md +0 -99
  545. package/evals/reports/by-skill/capture-decisions-2-report.md +0 -102
  546. package/evals/reports/by-skill/commit-work-1-report.md +0 -165
  547. package/evals/reports/by-skill/commit-work-2-report.md +0 -109
  548. package/evals/reports/by-skill/design-for-depth-1-report.md +0 -95
  549. package/evals/reports/by-skill/diagnose-failure-1-report.md +0 -91
  550. package/evals/reports/by-skill/diagnose-failure-2-report.md +0 -94
  551. package/evals/reports/by-skill/discover-1-report.md +0 -118
  552. package/evals/reports/by-skill/evaluate-skill-2-report.md +0 -71
  553. package/evals/reports/by-skill/execute-plan-1-report.md +0 -103
  554. package/evals/reports/by-skill/execute-plan-2-report.md +0 -112
  555. package/evals/reports/by-skill/execute-plan-3-report.md +0 -87
  556. package/evals/reports/by-skill/grill-context-1-report.md +0 -103
  557. package/evals/reports/by-skill/handoff-1-report.md +0 -123
  558. package/evals/reports/by-skill/handoff-2-report.md +0 -105
  559. package/evals/reports/by-skill/handoff-3-report.md +0 -100
  560. package/evals/reports/by-skill/handoff-7-report.md +0 -77
  561. package/evals/reports/by-skill/interview-gate-1-report.md +0 -63
  562. package/evals/reports/by-skill/interview-gate-2-report.md +0 -105
  563. package/evals/reports/by-skill/mode-contract-1-report.md +0 -74
  564. package/evals/reports/by-skill/mode-contract-2-report.md +0 -87
  565. package/evals/reports/by-skill/mode-contract-3-report.md +0 -65
  566. package/evals/reports/by-skill/mode-contract-4-report.md +0 -71
  567. package/evals/reports/by-skill/mode-contract-5-report.md +0 -70
  568. package/evals/reports/by-skill/output-router-1-report.md +0 -76
  569. package/evals/reports/by-skill/output-router-2-report.md +0 -65
  570. package/evals/reports/by-skill/research-1-report.md +0 -144
  571. package/evals/reports/by-skill/research-brief-1-report.md +0 -94
  572. package/evals/reports/by-skill/research-brief-2-report.md +0 -75
  573. package/evals/reports/by-skill/review-artifact-1-report.md +0 -91
  574. package/evals/reports/by-skill/review-artifact-2-report.md +0 -79
  575. package/evals/reports/by-skill/review-artifact-3-report.md +0 -120
  576. package/evals/reports/by-skill/review-artifact-4-report.md +0 -89
  577. package/evals/reports/by-skill/review-artifact-5-report.md +0 -66
  578. package/evals/reports/by-skill/review-work-1-report.md +0 -85
  579. package/evals/reports/by-skill/review-work-2-report.md +0 -87
  580. package/evals/reports/by-skill/review-work-3-report.md +0 -104
  581. package/evals/reports/by-skill/review-work-4-report.md +0 -108
  582. package/evals/reports/by-skill/review-work-5-report.md +0 -73
  583. package/evals/reports/by-skill/review-work-6-report.md +0 -66
  584. package/evals/reports/by-skill/setup-freeflow-1-report.md +0 -199
  585. package/evals/reports/by-skill/setup-freeflow-2-report.md +0 -105
  586. package/evals/reports/by-skill/setup-freeflow-3-report.md +0 -105
  587. package/evals/reports/by-skill/setup-freeflow-4-report.md +0 -88
  588. package/evals/reports/by-skill/setup-freeflow-5-report.md +0 -115
  589. package/evals/reports/by-skill/skill-authoring-1-report.md +0 -81
  590. package/evals/reports/by-skill/verify-work-1-report.md +0 -111
  591. package/evals/reports/by-skill/verify-work-2-report.md +0 -120
  592. package/evals/reports/by-skill/workflow-1-report.md +0 -65
  593. package/evals/reports/by-skill/write-plan-1-report.md +0 -123
  594. package/evals/reports/by-skill/write-plan-2-report.md +0 -98
  595. package/evals/reports/by-skill/write-plan-3-report.md +0 -110
  596. package/evals/reports/by-skill/write-spec-1-report.md +0 -88
  597. package/evals/reports/by-skill/write-spec-2-report.md +0 -89
  598. package/evals/reports/by-skill/write-spec-3-report.md +0 -90
  599. package/evals/reports/by-skill/write-spec-4-report.md +0 -167
  600. package/evals/reports/harness/claude-harness-1-report.md +0 -103
  601. package/evals/reports/harness/eval-harness-1-report.md +0 -91
  602. package/evals/reports/iterations/iteration-1-report.md +0 -71
  603. package/evals/reports/iterations/iteration-2-report.md +0 -78
  604. package/evals/reports/iterations/iteration-3-report.md +0 -118
  605. package/evals/reports/iterations/iteration-4-report.md +0 -131
  606. package/evals/reports/iterations/iteration-5-report.md +0 -149
  607. package/evals/reports/iterations/iteration-6-report.md +0 -136
  608. package/evals/reports/runtime/always-on-runtime-1-report.md +0 -107
  609. package/evals/reports/runtime/context-mode-normalized-benchmark-1-report.md +0 -52
  610. package/evals/reports/runtime/context-mode-real-deep-baseline-1-report.md +0 -410
  611. package/evals/reports/runtime/context-mode-real-deep-final-slice-11-report.md +0 -203
  612. package/evals/reports/runtime/context-mode-real-deep-final-slice-11-review.md +0 -51
  613. package/evals/reports/runtime/context-saving-native-tools-baseline-1-report.md +0 -99
  614. package/evals/reports/runtime/eryx-python-compatibility-reprobe-1-report.md +0 -166
  615. package/evals/reports/runtime/eryx-python-proof-spike-1-report.md +0 -96
  616. package/evals/reports/runtime/eryx-python-proof-spike-2-report.md +0 -151
  617. package/evals/reports/runtime/eryx-python-script-derive-execution-1-report.md +0 -233
  618. package/evals/reports/runtime/jq-script-derive-execution-1-report.md +0 -215
  619. package/evals/reports/runtime/jq-wasm-proof-spike-1-report.md +0 -67
  620. package/evals/reports/runtime/output-router-benchmark-1-report.md +0 -81
  621. package/evals/reports/runtime/output-router-capture-derive-provider-eval-1-report.md +0 -35
  622. package/evals/reports/runtime/output-router-codex-large-index-sqlite-adhoc-report.md +0 -150
  623. package/evals/reports/runtime/output-router-codex-qa-benchmark-1-report.md +0 -46
  624. package/evals/reports/runtime/output-router-command-benchmark-1-report.md +0 -62
  625. package/evals/reports/runtime/output-router-index-benchmark-1-report.md +0 -64
  626. package/evals/reports/runtime/output-router-regression-1-report.md +0 -81
  627. package/evals/reports/runtime/output-router-transform-eval-1-report.md +0 -28
  628. package/evals/reports/runtime/pi-observed-routing-eval-1-report.md +0 -44
  629. package/evals/reports/runtime/quickjs-script-derive-execution-1-report.md +0 -159
  630. package/evals/reports/runtime/quickjs-wasi-proof-spike-1-report.md +0 -55
  631. package/evals/reports/runtime/render-format-row-protocol-1-report.md +0 -58
  632. package/evals/reports/runtime/script-sandbox-adapter-selection-review-1-report.md +0 -84
  633. package/evals/reports/runtime/script-sandbox-probe-resource-hardening-1-report.md +0 -51
  634. package/evals/reports/runtime/storage-policy-benchmark-1-report.md +0 -176
  635. package/evals/reports/runtime/vault-index-storage-spike-1-report.md +0 -39
  636. package/evals/reports/runtime/workflow-context-hook-1-report.md +0 -82
  637. package/evals/runbooks/adversarial-eval-runbook.md +0 -28
  638. package/evals/runbooks/fixture-eval-runbook.md +0 -137
  639. package/evals/runbooks/handoff-eval-runbook.md +0 -22
  640. package/evals/runbooks/verify-work-eval-runbook.md +0 -27
  641. package/evals/scripts/audit-command-surface.sh +0 -160
  642. package/evals/scripts/check-activation-contract.sh +0 -121
  643. package/evals/scripts/check-runtime-context-hook.sh +0 -183
  644. package/evals/scripts/grade-fixture-eval.sh +0 -446
  645. package/evals/scripts/run-claude-fixture-eval.sh +0 -26
  646. package/evals/scripts/run-codex-fixture-eval.sh +0 -26
  647. package/evals/scripts/run-eryx-python-proof-spike.js +0 -559
  648. package/evals/scripts/run-fixture-eval-by-id.sh +0 -23
  649. package/evals/scripts/run-fixture-eval.sh +0 -464
  650. package/evals/scripts/run-jq-wasm-proof-spike.js +0 -299
  651. package/evals/scripts/run-output-router-transform-eval.js +0 -115
  652. package/evals/scripts/run-pi-observed-routing-eval.js +0 -478
  653. package/evals/scripts/run-quickjs-wasi-proof-spike.js +0 -339
  654. package/evals/scripts/skill-evidence.sh +0 -316
  655. package/evals/scripts/validate-release-metadata.sh +0 -486
  656. package/evals/suites/v0.1-acceptance-suite.md +0 -131
  657. package/pi-extension/dist/host-producer-identification.js +0 -186
  658. package/pi-extension/dist/mcp-config.js +0 -30
  659. package/pi-extension/dist/native-safety-net.js +0 -209
  660. package/pi-extension/dist/observed-tool-routing.js +0 -114
  661. package/pi-extension/dist/renderers.js +0 -835
  662. package/pi-extension/dist/router-tools.js +0 -438
  663. package/pi-extension/dist/runtime-context.js +0 -288
  664. package/pi-extension/dist/schemas.js +0 -254
  665. package/pi-extension/dist/status.js +0 -449
  666. package/pi-extension/dist/utils.js +0 -467
  667. package/pi-extension/src/runtime-context.ts +0 -347
  668. package/pi-extension/src/status.ts +0 -505
  669. package/plugin-docs/README.md +0 -40
  670. package/plugin-docs/adr/0001-three-modes.md +0 -27
  671. package/plugin-docs/adr/0002-evals-before-hooks.md +0 -17
  672. package/plugin-docs/adr/0003-release-boundary.md +0 -21
  673. package/plugin-docs/adr/0004-discover-replaces-shallow-discovery-skills.md +0 -26
  674. package/plugin-docs/adr/README.md +0 -10
  675. package/plugin-docs/architecture.md +0 -89
  676. package/plugin-docs/output-router.md +0 -396
  677. package/plugin-docs/release-evidence.md +0 -77
  678. package/plugin-docs/skills.md +0 -44
  679. package/plugin-docs/workflow.md +0 -57
  680. package/skills/diagnose-failure/references/feedback-loop-catalog.md +0 -25
  681. package/skills/discover/ARTIFACT-DESTINATIONS.md +0 -58
  682. package/skills/discover/CHECKPOINTS.md +0 -74
  683. package/skills/discover/SKILL.md +0 -173
  684. package/skills/evaluate-skill/references/eval-patterns.md +0 -41
  685. package/skills/evaluate-skill/references/grading-priority.md +0 -33
  686. package/skills/execute-plan/SKILL.md +0 -133
  687. package/skills/execute-plan/references/execution-map.md +0 -122
  688. package/skills/interview-gate/SKILL.md +0 -90
  689. package/skills/workflow/references/workflow-map.md +0 -104
  690. /package/pi-extension/src/{mcp-config.ts → integrations/mcp-config.ts} +0 -0
@@ -1,47 +1,56 @@
1
1
  export interface RunVaultIndexStorageBenchmarkOptions {
2
- iterations?: number;
3
- generatedAt?: string;
2
+ iterations?: number;
3
+ generatedAt?: string;
4
4
  }
5
5
  export interface VaultIndexStorageBenchmarkReport {
6
- generatedAt: string;
7
- iterations: number;
8
- summary: VaultIndexStorageBenchmarkSummary;
9
- candidates: VaultIndexStorageCandidateResult[];
6
+ generatedAt: string;
7
+ iterations: number;
8
+ summary: VaultIndexStorageBenchmarkSummary;
9
+ candidates: VaultIndexStorageCandidateResult[];
10
10
  }
11
11
  export interface VaultIndexStorageBenchmarkSummary {
12
- selectedEngine: string;
13
- selectedReason: string;
14
- fixtures: number;
15
- localSidecarPassed: boolean;
16
- scannerBaselinePassed: boolean;
17
- sqliteFtsStatus: "not-run" | "available" | "failed";
18
- localAppendMs: LatencyPair;
19
- localQueryMs: LatencyPair;
20
- scanQueryMs: LatencyPair;
21
- localQueryReductionPercent: number;
12
+ selectedEngine: string;
13
+ selectedReason: string;
14
+ fixtures: number;
15
+ localSidecarPassed: boolean;
16
+ scannerBaselinePassed: boolean;
17
+ sqliteFtsStatus: "not-run" | "available" | "failed";
18
+ localAppendMs: LatencyPair;
19
+ localQueryMs: LatencyPair;
20
+ scanQueryMs: LatencyPair;
21
+ localQueryReductionPercent: number;
22
22
  }
23
23
  export interface LatencyPair {
24
- p50: number;
25
- p95: number;
24
+ p50: number;
25
+ p95: number;
26
26
  }
27
27
  export interface VaultIndexStorageCandidateResult {
28
- candidate: string;
29
- status: "pass" | "fail" | "not-run";
30
- adopted: boolean;
31
- appendMs?: LatencyPair;
32
- queryMs?: LatencyPair;
33
- rawBytes?: number;
34
- resultBytes?: number;
35
- reductionPercent?: number;
36
- checks: string[];
37
- notes: string[];
28
+ candidate: string;
29
+ status: "pass" | "fail" | "not-run";
30
+ adopted: boolean;
31
+ appendMs?: LatencyPair;
32
+ queryMs?: LatencyPair;
33
+ rawBytes?: number;
34
+ resultBytes?: number;
35
+ reductionPercent?: number;
36
+ checks: string[];
37
+ notes: string[];
38
38
  }
39
- export declare function runVaultIndexStorageBenchmark(options?: RunVaultIndexStorageBenchmarkOptions): Promise<VaultIndexStorageBenchmarkReport>;
39
+ export declare function runVaultIndexStorageBenchmark(
40
+ options?: RunVaultIndexStorageBenchmarkOptions,
41
+ ): Promise<VaultIndexStorageBenchmarkReport>;
40
42
  export declare function renderVaultIndexStorageBenchmarkReport(report: VaultIndexStorageBenchmarkReport): string;
41
- export declare function writeVaultIndexStorageBenchmarkReport(report: VaultIndexStorageBenchmarkReport, reportPath: string): Promise<void>;
42
- export declare function writeVaultIndexStorageBenchmarkReports(report: VaultIndexStorageBenchmarkReport, markdownReportPath: string, options?: {
43
+ export declare function writeVaultIndexStorageBenchmarkReport(
44
+ report: VaultIndexStorageBenchmarkReport,
45
+ reportPath: string,
46
+ ): Promise<void>;
47
+ export declare function writeVaultIndexStorageBenchmarkReports(
48
+ report: VaultIndexStorageBenchmarkReport,
49
+ markdownReportPath: string,
50
+ options?: {
43
51
  jsonReportPath?: string | false;
44
- }): Promise<{
45
- markdown: string;
46
- json?: string;
52
+ },
53
+ ): Promise<{
54
+ markdown: string;
55
+ json?: string;
47
56
  }>;
@@ -3,255 +3,320 @@ import { tmpdir } from "node:os";
3
3
  import { dirname, resolve } from "node:path";
4
4
  import { fileURLToPath } from "node:url";
5
5
  import { performance } from "node:perf_hooks";
6
- import { escapeMarkdownTableCell as escapeTable, formatPercent, latencySummary, normalizeIterations, parseBenchmarkCliArgs, reductionPercent, writeBenchmarkReportPair, } from "./benchmark-harness.js";
6
+ import {
7
+ escapeMarkdownTableCell as escapeTable,
8
+ formatPercent,
9
+ latencySummary,
10
+ normalizeIterations,
11
+ parseBenchmarkCliArgs,
12
+ reductionPercent,
13
+ writeBenchmarkReportPair,
14
+ } from "./benchmark-harness.js";
7
15
  import { createLocalVaultIndex } from "../vault/vault-index.js";
8
- import { createVault, readOutputText, storeCommandOutput, storeMetadataOutput, storeTextOutput } from "../vault/vault.js";
16
+ import {
17
+ createVault,
18
+ readOutputText,
19
+ storeCommandOutput,
20
+ storeMetadataOutput,
21
+ storeTextOutput,
22
+ } from "../vault/vault.js";
9
23
  const DEFAULT_ITERATIONS = 3;
10
- const REPORT_PATH = "evals/reports/runtime/vault-index-storage-spike-1-report.md";
24
+ const REPORT_PATH = "router/evals/reports/vault-index-storage-spike-1-report.md";
11
25
  export async function runVaultIndexStorageBenchmark(options = {}) {
12
- const iterations = normalizeIterations(options.iterations, DEFAULT_ITERATIONS);
13
- const localAppendLatencies = [];
14
- const localQueryLatencies = [];
15
- const scanQueryLatencies = [];
16
- let localPassed = true;
17
- let scanPassed = true;
18
- let localRawBytes = 0;
19
- let localResultBytes = 0;
20
- let fixtureCount = 0;
21
- for (let iteration = 0; iteration < iterations; iteration += 1) {
22
- const tmpRoot = await mkdtemp(resolve(tmpdir(), "freeflow-vault-index-bench-"));
23
- try {
24
- const vault = createVault({ root: tmpRoot });
25
- const index = createLocalVaultIndex(vault);
26
- const emptyStatus = await index.status();
27
- localPassed &&= emptyStatus.entryCount === 0;
28
- const fixtures = await createBenchmarkFixtures(vault, iteration);
29
- fixtureCount = fixtures.length;
30
- for (const fixture of fixtures) {
31
- localAppendLatencies.push(fixture.appendMs);
32
- const visibilityStarted = performance.now();
33
- const visible = await index.queryVault(fixture.query, { sessionId: "vault-index-benchmark" }, { topK: 3 });
34
- localQueryLatencies.push(performance.now() - visibilityStarted);
35
- localPassed &&= visible.matches.some((match) => match.outputId === fixture.expectedOutputId);
36
- if (fixture.forbiddenQuery) {
37
- const forbidden = await index.queryVault(fixture.forbiddenQuery, { sessionId: "vault-index-benchmark" }, { topK: 3 });
38
- localPassed &&= forbidden.matches.length === 0;
39
- }
40
- }
41
- const status = await index.status();
42
- localPassed &&= status.outputCount === fixtures.length;
43
- for (const fixture of fixtures) {
44
- const started = performance.now();
45
- const scanned = await scanVaultFixture(vault, fixture);
46
- scanQueryLatencies.push(performance.now() - started);
47
- scanPassed &&= scanned;
48
- }
49
- const queryResult = await index.queryVault("target", { sessionId: "vault-index-benchmark" }, { topK: 10 });
50
- localRawBytes += fixtures.reduce((total, fixture) => total + Buffer.byteLength(fixture.text ?? JSON.stringify(fixture.record), "utf8"), 0);
51
- localResultBytes += Buffer.byteLength(JSON.stringify(queryResult.matches), "utf8");
52
- }
53
- finally {
54
- await rm(tmpRoot, { recursive: true, force: true });
26
+ const iterations = normalizeIterations(options.iterations, DEFAULT_ITERATIONS);
27
+ const localAppendLatencies = [];
28
+ const localQueryLatencies = [];
29
+ const scanQueryLatencies = [];
30
+ let localPassed = true;
31
+ let scanPassed = true;
32
+ let localRawBytes = 0;
33
+ let localResultBytes = 0;
34
+ let fixtureCount = 0;
35
+ for (let iteration = 0; iteration < iterations; iteration += 1) {
36
+ const tmpRoot = await mkdtemp(resolve(tmpdir(), "freeflow-vault-index-bench-"));
37
+ try {
38
+ const vault = createVault({ root: tmpRoot });
39
+ const index = createLocalVaultIndex(vault);
40
+ const emptyStatus = await index.status();
41
+ localPassed &&= emptyStatus.entryCount === 0;
42
+ const fixtures = await createBenchmarkFixtures(vault, iteration);
43
+ fixtureCount = fixtures.length;
44
+ for (const fixture of fixtures) {
45
+ localAppendLatencies.push(fixture.appendMs);
46
+ const visibilityStarted = performance.now();
47
+ const visible = await index.queryVault(fixture.query, { sessionId: "vault-index-benchmark" }, { topK: 3 });
48
+ localQueryLatencies.push(performance.now() - visibilityStarted);
49
+ localPassed &&= visible.matches.some((match) => match.outputId === fixture.expectedOutputId);
50
+ if (fixture.forbiddenQuery) {
51
+ const forbidden = await index.queryVault(
52
+ fixture.forbiddenQuery,
53
+ { sessionId: "vault-index-benchmark" },
54
+ { topK: 3 },
55
+ );
56
+ localPassed &&= forbidden.matches.length === 0;
55
57
  }
58
+ }
59
+ const status = await index.status();
60
+ localPassed &&= status.outputCount === fixtures.length;
61
+ for (const fixture of fixtures) {
62
+ const started = performance.now();
63
+ const scanned = await scanVaultFixture(vault, fixture);
64
+ scanQueryLatencies.push(performance.now() - started);
65
+ scanPassed &&= scanned;
66
+ }
67
+ const queryResult = await index.queryVault("target", { sessionId: "vault-index-benchmark" }, { topK: 10 });
68
+ localRawBytes += fixtures.reduce(
69
+ (total, fixture) => total + Buffer.byteLength(fixture.text ?? JSON.stringify(fixture.record), "utf8"),
70
+ 0,
71
+ );
72
+ localResultBytes += Buffer.byteLength(JSON.stringify(queryResult.matches), "utf8");
73
+ } finally {
74
+ await rm(tmpRoot, { recursive: true, force: true });
56
75
  }
57
- const localAppend = latencySummary(localAppendLatencies);
58
- const localQuery = latencySummary(localQueryLatencies);
59
- const scanQuery = latencySummary(scanQueryLatencies);
60
- const localReduction = reductionPercent(localRawBytes, localResultBytes);
61
- const sqliteStatus = "not-run";
62
- const candidates = [
63
- {
64
- candidate: "local-json-sidecar",
65
- status: localPassed ? "pass" : "fail",
66
- adopted: true,
67
- appendMs: localAppend,
68
- queryMs: localQuery,
69
- rawBytes: localRawBytes,
70
- resultBytes: localResultBytes,
71
- reductionPercent: localReduction,
72
- checks: [
73
- "fresh index starts empty",
74
- "each persisted append is queryable immediately",
75
- "metadata-only raw content is not indexed",
76
- "no native dependency",
77
- ],
78
- notes: ["Selected for the Slice 11 interface and exercised through the Slice 12 write path because it is portable and dependency-free."],
79
- },
80
- {
81
- candidate: "vault-session-scan-baseline",
82
- status: scanPassed ? "pass" : "fail",
83
- adopted: false,
84
- queryMs: scanQuery,
85
- checks: ["finds fixture evidence by reading vault records directly"],
86
- notes: ["Baseline remains source-truth fallback but query cost grows with vaulted text."],
87
- },
88
- {
89
- candidate: "sqlite-fts",
90
- status: sqliteStatus,
91
- adopted: false,
92
- checks: ["not introduced during Slice 11"],
93
- notes: ["Deferred: adding a native dependency or relying on experimental runtime SQLite needs explicit owner approval."],
94
- },
95
- ];
96
- return {
97
- generatedAt: options.generatedAt ?? new Date().toISOString(),
98
- iterations,
99
- summary: {
100
- selectedEngine: "local-json-sidecar",
101
- selectedReason: "Portable deterministic sidecar files satisfy automatic incremental write-through semantics without adding a native dependency.",
102
- fixtures: fixtureCount,
103
- localSidecarPassed: localPassed,
104
- scannerBaselinePassed: scanPassed,
105
- sqliteFtsStatus: sqliteStatus,
106
- localAppendMs: localAppend,
107
- localQueryMs: localQuery,
108
- scanQueryMs: scanQuery,
109
- localQueryReductionPercent: localReduction,
110
- },
111
- candidates,
112
- };
76
+ }
77
+ const localAppend = latencySummary(localAppendLatencies);
78
+ const localQuery = latencySummary(localQueryLatencies);
79
+ const scanQuery = latencySummary(scanQueryLatencies);
80
+ const localReduction = reductionPercent(localRawBytes, localResultBytes);
81
+ const sqliteStatus = "not-run";
82
+ const candidates = [
83
+ {
84
+ candidate: "local-json-sidecar",
85
+ status: localPassed ? "pass" : "fail",
86
+ adopted: true,
87
+ appendMs: localAppend,
88
+ queryMs: localQuery,
89
+ rawBytes: localRawBytes,
90
+ resultBytes: localResultBytes,
91
+ reductionPercent: localReduction,
92
+ checks: [
93
+ "fresh index starts empty",
94
+ "each persisted append is queryable immediately",
95
+ "metadata-only raw content is not indexed",
96
+ "no native dependency",
97
+ ],
98
+ notes: [
99
+ "Selected for the Slice 11 interface and exercised through the Slice 12 write path because it is portable and dependency-free.",
100
+ ],
101
+ },
102
+ {
103
+ candidate: "vault-session-scan-baseline",
104
+ status: scanPassed ? "pass" : "fail",
105
+ adopted: false,
106
+ queryMs: scanQuery,
107
+ checks: ["finds fixture evidence by reading vault records directly"],
108
+ notes: ["Baseline remains source-truth fallback but query cost grows with vaulted text."],
109
+ },
110
+ {
111
+ candidate: "sqlite-fts",
112
+ status: sqliteStatus,
113
+ adopted: false,
114
+ checks: ["not introduced during Slice 11"],
115
+ notes: [
116
+ "Deferred: adding a native dependency or relying on experimental runtime SQLite needs explicit owner approval.",
117
+ ],
118
+ },
119
+ ];
120
+ return {
121
+ generatedAt: options.generatedAt ?? new Date().toISOString(),
122
+ iterations,
123
+ summary: {
124
+ selectedEngine: "local-json-sidecar",
125
+ selectedReason:
126
+ "Portable deterministic sidecar files satisfy automatic incremental write-through semantics without adding a native dependency.",
127
+ fixtures: fixtureCount,
128
+ localSidecarPassed: localPassed,
129
+ scannerBaselinePassed: scanPassed,
130
+ sqliteFtsStatus: sqliteStatus,
131
+ localAppendMs: localAppend,
132
+ localQueryMs: localQuery,
133
+ scanQueryMs: scanQuery,
134
+ localQueryReductionPercent: localReduction,
135
+ },
136
+ candidates,
137
+ };
113
138
  }
114
139
  async function createBenchmarkFixtures(vault, iteration) {
115
- const commandText = [
116
- "setup noise ".repeat(100),
117
- `alpha target evidence iteration ${iteration}`,
118
- "tail noise ".repeat(100),
119
- ].join("\n");
120
- let started = performance.now();
121
- const command = await storeCommandOutput(vault, {
122
- sessionId: "vault-index-benchmark",
123
- command: "npm test",
124
- stdout: commandText,
125
- stderr: "",
126
- executionStatus: "success",
127
- exitCode: 0,
128
- createdAt: `2026-06-24T00:00:${String(iteration).padStart(2, "0")}.000Z`,
129
- });
130
- const commandAppendMs = performance.now() - started;
131
- const observedText = [
132
- "github issue search",
133
- `beta target issue https://github.com/acme/freeflow/issues/${iteration + 1}`,
134
- "body noise ".repeat(160),
135
- ].join("\n");
136
- started = performance.now();
137
- const observed = await storeTextOutput(vault, {
138
- sessionId: "vault-index-benchmark",
139
- sourceKind: "mcp",
140
- raw: observedText,
141
- producer: { kind: "mcp", server: "github", tool: "search_issues" },
142
- createdAt: `2026-06-24T00:01:${String(iteration).padStart(2, "0")}.000Z`,
143
- });
144
- const observedAppendMs = performance.now() - started;
145
- started = performance.now();
146
- const metadata = await storeMetadataOutput(vault, {
147
- sessionId: "vault-index-benchmark",
148
- sourceKind: "mcp",
149
- rawLineCount: 3,
150
- rawByteCount: 4096,
151
- rawSha256: "b".repeat(64),
152
- metadata: { producer: { kind: "mcp", server: "gmail", tool: "search" }, note: "gamma target metadata only" },
153
- producer: { kind: "mcp", server: "gmail", tool: "search" },
154
- createdAt: `2026-06-24T00:02:${String(iteration).padStart(2, "0")}.000Z`,
155
- });
156
- const metadataAppendMs = performance.now() - started;
157
- return [
158
- { record: command, appendMs: commandAppendMs, text: commandText, stream: "combined", query: "alpha target", expectedOutputId: command.outputId },
159
- { record: observed, appendMs: observedAppendMs, text: observedText, stream: "raw", query: "beta target github", expectedOutputId: observed.outputId },
160
- { record: metadata, appendMs: metadataAppendMs, query: "gmail gamma target", expectedOutputId: metadata.outputId, forbiddenQuery: "PRIVATE_RAW_SECRET" },
161
- ];
140
+ const commandText = [
141
+ "setup noise ".repeat(100),
142
+ `alpha target evidence iteration ${iteration}`,
143
+ "tail noise ".repeat(100),
144
+ ].join("\n");
145
+ let started = performance.now();
146
+ const command = await storeCommandOutput(vault, {
147
+ sessionId: "vault-index-benchmark",
148
+ command: "npm test",
149
+ stdout: commandText,
150
+ stderr: "",
151
+ executionStatus: "success",
152
+ exitCode: 0,
153
+ createdAt: `2026-06-24T00:00:${String(iteration).padStart(2, "0")}.000Z`,
154
+ });
155
+ const commandAppendMs = performance.now() - started;
156
+ const observedText = [
157
+ "github issue search",
158
+ `beta target issue https://github.com/acme/freeflow/issues/${iteration + 1}`,
159
+ "body noise ".repeat(160),
160
+ ].join("\n");
161
+ started = performance.now();
162
+ const observed = await storeTextOutput(vault, {
163
+ sessionId: "vault-index-benchmark",
164
+ sourceKind: "mcp",
165
+ raw: observedText,
166
+ producer: { kind: "mcp", server: "github", tool: "search_issues" },
167
+ createdAt: `2026-06-24T00:01:${String(iteration).padStart(2, "0")}.000Z`,
168
+ });
169
+ const observedAppendMs = performance.now() - started;
170
+ started = performance.now();
171
+ const metadata = await storeMetadataOutput(vault, {
172
+ sessionId: "vault-index-benchmark",
173
+ sourceKind: "mcp",
174
+ rawLineCount: 3,
175
+ rawByteCount: 4096,
176
+ rawSha256: "b".repeat(64),
177
+ metadata: { producer: { kind: "mcp", server: "gmail", tool: "search" }, note: "gamma target metadata only" },
178
+ producer: { kind: "mcp", server: "gmail", tool: "search" },
179
+ createdAt: `2026-06-24T00:02:${String(iteration).padStart(2, "0")}.000Z`,
180
+ });
181
+ const metadataAppendMs = performance.now() - started;
182
+ return [
183
+ {
184
+ record: command,
185
+ appendMs: commandAppendMs,
186
+ text: commandText,
187
+ stream: "combined",
188
+ query: "alpha target",
189
+ expectedOutputId: command.outputId,
190
+ },
191
+ {
192
+ record: observed,
193
+ appendMs: observedAppendMs,
194
+ text: observedText,
195
+ stream: "raw",
196
+ query: "beta target github",
197
+ expectedOutputId: observed.outputId,
198
+ },
199
+ {
200
+ record: metadata,
201
+ appendMs: metadataAppendMs,
202
+ query: "gmail gamma target",
203
+ expectedOutputId: metadata.outputId,
204
+ forbiddenQuery: "PRIVATE_RAW_SECRET",
205
+ },
206
+ ];
162
207
  }
163
208
  async function scanVaultFixture(vault, fixture) {
164
- const text = await textForScan(vault, fixture.record, fixture.stream);
165
- return fixture.query.toLowerCase().split(/\s+/).every((term) => text.toLowerCase().includes(term));
209
+ const text = await textForScan(vault, fixture.record, fixture.stream);
210
+ return fixture.query
211
+ .toLowerCase()
212
+ .split(/\s+/)
213
+ .every((term) => text.toLowerCase().includes(term));
166
214
  }
167
215
  async function textForScan(vault, record, stream) {
168
- if (record.kind === "command") {
169
- return readOutputText(vault, "vault-index-benchmark", record.outputId, stream ?? "combined");
170
- }
171
- if (record.kind === "text") {
172
- return readOutputText(vault, "vault-index-benchmark", record.outputId, "raw");
173
- }
174
- return JSON.stringify(record);
216
+ if (record.kind === "command") {
217
+ return readOutputText(vault, "vault-index-benchmark", record.outputId, stream ?? "combined");
218
+ }
219
+ if (record.kind === "text") {
220
+ return readOutputText(vault, "vault-index-benchmark", record.outputId, "raw");
221
+ }
222
+ return JSON.stringify(record);
175
223
  }
176
224
  export function renderVaultIndexStorageBenchmarkReport(report) {
177
- const date = report.generatedAt.slice(0, 10);
178
- const lines = [
179
- "# Vault Index Storage Spike Report - Iteration 1",
180
- "",
181
- `Date: ${date}`,
182
- "",
183
- "## Scope",
184
- "",
185
- "Slice 11/12 storage-interface and write-path spike for the Freeflow vault index. The benchmark checks that a fresh index starts empty, persisted appends become queryable incrementally through automatic write-through indexing, metadata-only records do not index raw content, and the selected storage engine stays hidden behind the vault-index interface.",
186
- "",
187
- "## Command",
188
- "",
189
- "```sh",
190
- "npm run bench:router:vault-index",
191
- "```",
192
- "",
193
- "## Summary",
194
- "",
195
- `- Fixtures per iteration: ${report.summary.fixtures}`,
196
- `- Iterations: ${report.iterations}`,
197
- `- Selected engine: ${report.summary.selectedEngine}`,
198
- `- Selected reason: ${report.summary.selectedReason}`,
199
- `- Local sidecar passed: ${report.summary.localSidecarPassed ? "yes" : "no"}`,
200
- `- Scanner baseline passed: ${report.summary.scannerBaselinePassed ? "yes" : "no"}`,
201
- `- SQLite FTS status: ${report.summary.sqliteFtsStatus}`,
202
- `- Local append p50/p95: ${report.summary.localAppendMs.p50.toFixed(2)}/${report.summary.localAppendMs.p95.toFixed(2)} ms`,
203
- `- Local query p50/p95: ${report.summary.localQueryMs.p50.toFixed(2)}/${report.summary.localQueryMs.p95.toFixed(2)} ms`,
204
- `- Scan query p50/p95: ${report.summary.scanQueryMs.p50.toFixed(2)}/${report.summary.scanQueryMs.p95.toFixed(2)} ms`,
205
- `- Local query result reduction: ${formatPercent(report.summary.localQueryReductionPercent)}`,
206
- "",
207
- "## Candidates",
208
- "",
209
- "| candidate | status | adopted | append p50/p95 ms | query p50/p95 ms | raw/result bytes | reduction | checks | notes |",
210
- "| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |",
211
- ];
212
- for (const candidate of report.candidates) {
213
- lines.push([
214
- escapeTable(candidate.candidate),
215
- candidate.status,
216
- candidate.adopted ? "yes" : "no",
217
- candidate.appendMs ? `${candidate.appendMs.p50.toFixed(2)}/${candidate.appendMs.p95.toFixed(2)}` : "-",
218
- candidate.queryMs ? `${candidate.queryMs.p50.toFixed(2)}/${candidate.queryMs.p95.toFixed(2)}` : "-",
219
- candidate.rawBytes !== undefined && candidate.resultBytes !== undefined ? `${candidate.rawBytes}/${candidate.resultBytes}` : "-",
220
- candidate.reductionPercent !== undefined ? formatPercent(candidate.reductionPercent) : "-",
221
- escapeTable(candidate.checks.join("; ")),
222
- escapeTable(candidate.notes.join("; ")),
223
- ].join(" | ").replace(/^/, "| ").replace(/$/, " |"));
224
- }
225
- lines.push("", "## Decision", "", "Use the deterministic local JSON sidecar behind the vault-index interface for the next slices. Do not adopt SQLite/FTS in this slice because introducing a native dependency or relying on experimental runtime SQLite needs separate owner approval. The vault remains source truth; the index is a rebuildable sidecar.", "");
226
- return lines.join("\n");
225
+ const date = report.generatedAt.slice(0, 10);
226
+ const lines = [
227
+ "# Vault Index Storage Spike Report - Iteration 1",
228
+ "",
229
+ `Date: ${date}`,
230
+ "",
231
+ "## Scope",
232
+ "",
233
+ "Slice 11/12 storage-interface and write-path spike for the Freeflow vault index. The benchmark checks that a fresh index starts empty, persisted appends become queryable incrementally through automatic write-through indexing, metadata-only records do not index raw content, and the selected storage engine stays hidden behind the vault-index interface.",
234
+ "",
235
+ "## Command",
236
+ "",
237
+ "```sh",
238
+ "npm run bench:router:vault-index",
239
+ "```",
240
+ "",
241
+ "## Summary",
242
+ "",
243
+ `- Fixtures per iteration: ${report.summary.fixtures}`,
244
+ `- Iterations: ${report.iterations}`,
245
+ `- Selected engine: ${report.summary.selectedEngine}`,
246
+ `- Selected reason: ${report.summary.selectedReason}`,
247
+ `- Local sidecar passed: ${report.summary.localSidecarPassed ? "yes" : "no"}`,
248
+ `- Scanner baseline passed: ${report.summary.scannerBaselinePassed ? "yes" : "no"}`,
249
+ `- SQLite FTS status: ${report.summary.sqliteFtsStatus}`,
250
+ `- Local append p50/p95: ${report.summary.localAppendMs.p50.toFixed(2)}/${report.summary.localAppendMs.p95.toFixed(2)} ms`,
251
+ `- Local query p50/p95: ${report.summary.localQueryMs.p50.toFixed(2)}/${report.summary.localQueryMs.p95.toFixed(2)} ms`,
252
+ `- Scan query p50/p95: ${report.summary.scanQueryMs.p50.toFixed(2)}/${report.summary.scanQueryMs.p95.toFixed(2)} ms`,
253
+ `- Local query result reduction: ${formatPercent(report.summary.localQueryReductionPercent)}`,
254
+ "",
255
+ "## Candidates",
256
+ "",
257
+ "| candidate | status | adopted | append p50/p95 ms | query p50/p95 ms | raw/result bytes | reduction | checks | notes |",
258
+ "| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |",
259
+ ];
260
+ for (const candidate of report.candidates) {
261
+ lines.push(
262
+ [
263
+ escapeTable(candidate.candidate),
264
+ candidate.status,
265
+ candidate.adopted ? "yes" : "no",
266
+ candidate.appendMs ? `${candidate.appendMs.p50.toFixed(2)}/${candidate.appendMs.p95.toFixed(2)}` : "-",
267
+ candidate.queryMs ? `${candidate.queryMs.p50.toFixed(2)}/${candidate.queryMs.p95.toFixed(2)}` : "-",
268
+ candidate.rawBytes !== undefined && candidate.resultBytes !== undefined
269
+ ? `${candidate.rawBytes}/${candidate.resultBytes}`
270
+ : "-",
271
+ candidate.reductionPercent !== undefined ? formatPercent(candidate.reductionPercent) : "-",
272
+ escapeTable(candidate.checks.join("; ")),
273
+ escapeTable(candidate.notes.join("; ")),
274
+ ]
275
+ .join(" | ")
276
+ .replace(/^/, "| ")
277
+ .replace(/$/, " |"),
278
+ );
279
+ }
280
+ lines.push(
281
+ "",
282
+ "## Decision",
283
+ "",
284
+ "Use the deterministic local JSON sidecar behind the vault-index interface for the next slices. Do not adopt SQLite/FTS in this slice because introducing a native dependency or relying on experimental runtime SQLite needs separate owner approval. The vault remains source truth; the index is a rebuildable sidecar.",
285
+ "",
286
+ );
287
+ return lines.join("\n");
227
288
  }
228
289
  export async function writeVaultIndexStorageBenchmarkReport(report, reportPath) {
229
- await mkdir(dirname(reportPath), { recursive: true });
230
- await writeFile(reportPath, renderVaultIndexStorageBenchmarkReport(report), "utf8");
290
+ await mkdir(dirname(reportPath), { recursive: true });
291
+ await writeFile(reportPath, renderVaultIndexStorageBenchmarkReport(report), "utf8");
231
292
  }
232
293
  export async function writeVaultIndexStorageBenchmarkReports(report, markdownReportPath, options = {}) {
233
- return writeBenchmarkReportPair({
234
- report,
235
- markdownReportPath,
236
- jsonReportPath: options.jsonReportPath,
237
- renderMarkdown: renderVaultIndexStorageBenchmarkReport,
238
- });
294
+ return writeBenchmarkReportPair({
295
+ report,
296
+ markdownReportPath,
297
+ jsonReportPath: options.jsonReportPath,
298
+ renderMarkdown: renderVaultIndexStorageBenchmarkReport,
299
+ });
239
300
  }
240
301
  async function main() {
241
- const args = parseBenchmarkCliArgs(process.argv.slice(2), { reportPath: REPORT_PATH });
242
- const benchmarkOptions = args.iterations === undefined ? {} : { iterations: args.iterations };
243
- const report = await runVaultIndexStorageBenchmark(benchmarkOptions);
244
- const writeOptions = args.jsonReportPath === undefined ? {} : { jsonReportPath: args.jsonReportPath };
245
- const written = await writeVaultIndexStorageBenchmarkReports(report, resolve(process.cwd(), args.reportPath), writeOptions);
246
- console.log(`Wrote ${written.markdown}`);
247
- if (written.json) {
248
- console.log(`Wrote ${written.json}`);
249
- }
302
+ const args = parseBenchmarkCliArgs(process.argv.slice(2), { reportPath: REPORT_PATH });
303
+ const benchmarkOptions = args.iterations === undefined ? {} : { iterations: args.iterations };
304
+ const report = await runVaultIndexStorageBenchmark(benchmarkOptions);
305
+ const writeOptions = args.jsonReportPath === undefined ? {} : { jsonReportPath: args.jsonReportPath };
306
+ const written = await writeVaultIndexStorageBenchmarkReports(
307
+ report,
308
+ resolve(process.cwd(), args.reportPath),
309
+ writeOptions,
310
+ );
311
+ console.log(`Wrote ${written.markdown}`);
312
+ if (written.json) {
313
+ console.log(`Wrote ${written.json}`);
314
+ }
250
315
  }
251
316
  const isMain = process.argv[1] !== undefined && fileURLToPath(import.meta.url) === resolve(process.argv[1]);
252
317
  if (isMain) {
253
- main().catch((error) => {
254
- console.error(error);
255
- process.exitCode = 1;
256
- });
318
+ main().catch((error) => {
319
+ console.error(error);
320
+ process.exitCode = 1;
321
+ });
257
322
  }