@hassangameryt/freeflow 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (692) hide show
  1. package/.claude-plugin/marketplace.json +4 -4
  2. package/.claude-plugin/plugin.json +6 -2
  3. package/.codex-plugin/plugin.json +9 -9
  4. package/README.md +190 -127
  5. package/assets/feedback-loop.png +0 -0
  6. package/assets/freeflow_workflow_map.png +0 -0
  7. package/assets/workflow.png +0 -0
  8. package/{skills → capabilities}/output-router/SKILL.md +21 -17
  9. package/{skills → capabilities}/output-router/references/safety-policy.md +1 -1
  10. package/capabilities/output-router/references/setup.md +110 -0
  11. package/command-surface.json +62 -7
  12. package/hooks/freeflow-runtime-context.mjs +572 -154
  13. package/hooks/hooks.json +26 -1
  14. package/hooks/tests/check-runtime-context-hook.sh +375 -0
  15. package/package.json +24 -24
  16. package/pi-extension/dist/index.js +264 -80
  17. package/pi-extension/dist/integrations/mcp-config.js +29 -0
  18. package/pi-extension/dist/output-router/host-producer-identification.js +232 -0
  19. package/pi-extension/dist/output-router/native-safety-net.js +216 -0
  20. package/pi-extension/dist/output-router/observed-tool-routing.js +122 -0
  21. package/pi-extension/dist/output-router/renderers.js +1023 -0
  22. package/pi-extension/dist/output-router/router-tools.js +554 -0
  23. package/pi-extension/dist/output-router/schemas.js +383 -0
  24. package/pi-extension/dist/output-router/status.js +744 -0
  25. package/pi-extension/dist/output-router/utils.js +502 -0
  26. package/pi-extension/dist/runtime/runtime-context.js +798 -0
  27. package/pi-extension/dist/settings/settings-tui.js +314 -0
  28. package/pi-extension/dist/settings/settings-ui.js +1487 -0
  29. package/pi-extension/freeflow/index.js +1 -0
  30. package/pi-extension/src/index.ts +230 -56
  31. package/pi-extension/src/{host-producer-identification.ts → output-router/host-producer-identification.ts} +37 -7
  32. package/pi-extension/src/{native-safety-net.ts → output-router/native-safety-net.ts} +8 -12
  33. package/pi-extension/src/{observed-tool-routing.ts → output-router/observed-tool-routing.ts} +19 -11
  34. package/pi-extension/src/{renderers.ts → output-router/renderers.ts} +246 -55
  35. package/pi-extension/src/{router-tools.ts → output-router/router-tools.ts} +130 -29
  36. package/pi-extension/src/{schemas.ts → output-router/schemas.ts} +171 -43
  37. package/pi-extension/src/output-router/status.ts +791 -0
  38. package/pi-extension/src/{utils.ts → output-router/utils.ts} +80 -18
  39. package/pi-extension/src/runtime/runtime-context.ts +895 -0
  40. package/pi-extension/src/settings/settings-tui.ts +408 -0
  41. package/pi-extension/src/settings/settings-ui.ts +1730 -0
  42. package/pi-extension/tests/integration/pi-extension.test.js +4020 -0
  43. package/pi-extension/tests/output-router/observed-producers.test.js +132 -0
  44. package/pi-extension/tests/output-router/search.test.js +722 -0
  45. package/pi-extension/tests/settings/settings-tui.test.js +243 -0
  46. package/router/dist/benchmarks/benchmark-harness.d.ts +16 -13
  47. package/router/dist/benchmarks/benchmark-harness.js +62 -62
  48. package/router/dist/benchmarks/benchmarks.d.ts +80 -73
  49. package/router/dist/benchmarks/benchmarks.js +860 -761
  50. package/router/dist/benchmarks/codex-qa-benchmarks.d.ts +71 -67
  51. package/router/dist/benchmarks/codex-qa-benchmarks.js +429 -390
  52. package/router/dist/benchmarks/command-benchmarks.d.ts +107 -92
  53. package/router/dist/benchmarks/command-benchmarks.js +616 -571
  54. package/router/dist/benchmarks/context-mode-normalized-benchmarks.d.ts +64 -56
  55. package/router/dist/benchmarks/context-mode-normalized-benchmarks.js +679 -613
  56. package/router/dist/benchmarks/context-mode-real-deep-benchmark.d.ts +103 -95
  57. package/router/dist/benchmarks/context-mode-real-deep-benchmark.js +1188 -1103
  58. package/router/dist/benchmarks/context-saving-baseline.d.ts +47 -41
  59. package/router/dist/benchmarks/context-saving-baseline.js +405 -374
  60. package/router/dist/benchmarks/index-benchmarks.d.ts +74 -69
  61. package/router/dist/benchmarks/index-benchmarks.js +761 -661
  62. package/router/dist/benchmarks/storage-policy-benchmarks.d.ts +80 -69
  63. package/router/dist/benchmarks/storage-policy-benchmarks.js +549 -469
  64. package/router/dist/benchmarks/vault-index-benchmarks.d.ts +43 -34
  65. package/router/dist/benchmarks/vault-index-benchmarks.js +294 -229
  66. package/router/dist/config/config.d.ts +62 -55
  67. package/router/dist/config/config.js +552 -373
  68. package/router/dist/config/router-contract.d.ts +2 -2
  69. package/router/dist/config/router-contract.js +38 -38
  70. package/router/dist/config/schema.d.ts +19 -10
  71. package/router/dist/config/schema.js +586 -538
  72. package/router/dist/config/types.d.ts +413 -361
  73. package/router/dist/config/types.js +19 -8
  74. package/router/dist/evidence/bounded-evidence.d.ts +26 -26
  75. package/router/dist/evidence/bounded-evidence.js +265 -252
  76. package/router/dist/evidence/evidence-range-selector.d.ts +16 -14
  77. package/router/dist/evidence/evidence-range-selector.js +192 -191
  78. package/router/dist/evidence/evidence-search.d.ts +18 -16
  79. package/router/dist/evidence/evidence-search.js +401 -377
  80. package/router/dist/evidence/evidence.d.ts +17 -17
  81. package/router/dist/evidence/evidence.js +163 -159
  82. package/router/dist/evidence/failure-contracts.d.ts +25 -15
  83. package/router/dist/evidence/failure-contracts.js +132 -126
  84. package/router/dist/evidence/line-ranges.d.ts +17 -15
  85. package/router/dist/evidence/line-ranges.js +18 -18
  86. package/router/dist/experiments/local-index.d.ts +47 -41
  87. package/router/dist/experiments/local-index.js +477 -458
  88. package/router/dist/local/local-traversal.d.ts +18 -0
  89. package/router/dist/local/local-traversal.js +334 -0
  90. package/router/dist/processing/engine.d.ts +94 -70
  91. package/router/dist/processing/engine.js +462 -364
  92. package/router/dist/processing/reducers.d.ts +208 -194
  93. package/router/dist/processing/reducers.js +1434 -1256
  94. package/router/dist/processing/renderers.d.ts +24 -18
  95. package/router/dist/processing/renderers.js +161 -152
  96. package/router/dist/processing/scripts.d.ts +80 -67
  97. package/router/dist/processing/scripts.js +343 -290
  98. package/router/dist/repo/repo-traversal.d.ts +9 -9
  99. package/router/dist/repo/repo-traversal.js +205 -202
  100. package/router/dist/routing/observed-reducers.d.ts +25 -19
  101. package/router/dist/routing/observed-reducers.js +559 -545
  102. package/router/dist/routing/observed-routing.d.ts +23 -14
  103. package/router/dist/routing/observed-routing.js +319 -318
  104. package/router/dist/routing/parsers.d.ts +10 -10
  105. package/router/dist/routing/parsers.js +454 -406
  106. package/router/dist/routing/run-filters.d.ts +45 -41
  107. package/router/dist/routing/run-filters.js +215 -207
  108. package/router/dist/routing/run-reducers.d.ts +45 -26
  109. package/router/dist/routing/run-reducers.js +102 -90
  110. package/router/dist/sandbox/adapter-roots.d.ts +34 -26
  111. package/router/dist/sandbox/adapter-roots.js +43 -41
  112. package/router/dist/sandbox/eryx-python-adapter.d.ts +9 -5
  113. package/router/dist/sandbox/eryx-python-adapter.js +659 -588
  114. package/router/dist/sandbox/jq-wasm-adapter.d.ts +3 -3
  115. package/router/dist/sandbox/jq-wasm-adapter.js +319 -311
  116. package/router/dist/sandbox/quickjs-wasi-adapter.d.ts +9 -5
  117. package/router/dist/sandbox/quickjs-wasi-adapter.js +336 -333
  118. package/router/dist/sandbox/script-sandbox.d.ts +105 -84
  119. package/router/dist/sandbox/script-sandbox.js +294 -260
  120. package/router/dist/setup/script-transform-adapters.d.ts +23 -19
  121. package/router/dist/setup/script-transform-adapters.js +223 -204
  122. package/router/dist/tools/batch.d.ts +27 -16
  123. package/router/dist/tools/batch.js +493 -416
  124. package/router/dist/tools/run.d.ts +47 -35
  125. package/router/dist/tools/run.js +1172 -1087
  126. package/router/dist/tools/search.d.ts +53 -31
  127. package/router/dist/tools/search.js +1224 -1065
  128. package/router/dist/transform/engine.d.ts +121 -96
  129. package/router/dist/transform/engine.js +1853 -1736
  130. package/router/dist/vault/vault-index.d.ts +97 -78
  131. package/router/dist/vault/vault-index.js +347 -342
  132. package/router/dist/vault/vault.d.ts +109 -74
  133. package/router/dist/vault/vault.js +528 -533
  134. package/runtime/interaction-contract.md +16 -0
  135. package/skills/bypass/SKILL.md +52 -38
  136. package/skills/commit-work/SKILL.md +54 -46
  137. package/skills/commit-work/references/staging-decisions.md +46 -24
  138. package/skills/decision-gate/SKILL.md +48 -0
  139. package/skills/design-for-depth/SKILL.md +75 -59
  140. package/skills/design-for-depth/references/design-pressure-signals.md +56 -7
  141. package/skills/design-for-depth/references/interface-design-loop.md +156 -0
  142. package/skills/design-for-depth/references/software-design-philosophy.md +32 -3
  143. package/skills/diagnose-failure/SKILL.md +65 -60
  144. package/skills/diagnose-failure/references/diagnostic-loop-catalog.md +36 -0
  145. package/skills/diagnose-failure/references/flaky-and-performance.md +30 -18
  146. package/skills/discuss/SKILL.md +121 -0
  147. package/skills/discuss/references/checkpoints.md +66 -0
  148. package/skills/evaluate-skill/SKILL.md +62 -53
  149. package/skills/evaluate-skill/references/definition-schema.md +345 -0
  150. package/skills/evaluate-skill/references/evaluation-design.md +86 -0
  151. package/skills/evaluate-skill/references/execution-and-evidence.md +85 -0
  152. package/skills/evaluate-skill/references/review-and-revision.md +51 -0
  153. package/skills/evaluate-skill/scripts/lib/definitions.mjs +296 -0
  154. package/skills/evaluate-skill/scripts/lib/evidence.mjs +46 -0
  155. package/skills/evaluate-skill/scripts/lib/grade-comparison.mjs +60 -0
  156. package/skills/evaluate-skill/scripts/lib/grade-interaction.mjs +148 -0
  157. package/skills/evaluate-skill/scripts/lib/grade-support.mjs +126 -0
  158. package/skills/evaluate-skill/scripts/lib/grade-workspace.mjs +192 -0
  159. package/skills/evaluate-skill/scripts/lib/grade.mjs +56 -0
  160. package/skills/evaluate-skill/scripts/lib/pi.mjs +489 -0
  161. package/skills/evaluate-skill/scripts/lib/process.mjs +79 -0
  162. package/skills/evaluate-skill/scripts/lib/rpc.mjs +444 -0
  163. package/skills/evaluate-skill/scripts/lib/runner.mjs +346 -0
  164. package/skills/evaluate-skill/scripts/lib/sandbox.mjs +409 -0
  165. package/skills/evaluate-skill/scripts/lib/view.mjs +337 -0
  166. package/skills/evaluate-skill/scripts/pi-guard.mjs +164 -0
  167. package/skills/evaluate-skill/scripts/skill-eval.mjs +101 -0
  168. package/skills/execute-work/SKILL.md +99 -0
  169. package/skills/execute-work/references/code-practices.md +78 -0
  170. package/skills/execute-work/references/execution-loop.md +77 -0
  171. package/skills/finish-branch/SKILL.md +117 -0
  172. package/skills/finish-branch/references/integration-options.md +80 -0
  173. package/skills/handoff/SKILL.md +65 -39
  174. package/skills/handoff/references/templates.md +104 -27
  175. package/skills/launch-work/SKILL.md +116 -0
  176. package/skills/launch-work/references/launch-readiness.md +85 -0
  177. package/skills/migration-work/SKILL.md +115 -0
  178. package/skills/migration-work/references/migration-lifecycle.md +82 -0
  179. package/skills/mode-contract/SKILL.md +72 -52
  180. package/skills/release-work/SKILL.md +118 -0
  181. package/skills/release-work/references/release-evidence.md +78 -0
  182. package/skills/review-artifact/SKILL.md +77 -69
  183. package/skills/review-artifact/references/reviewer-prompt.md +133 -42
  184. package/skills/review-work/SKILL.md +82 -57
  185. package/skills/review-work/references/reviewer-prompt.md +97 -60
  186. package/skills/review-work/references/security-risk-lens.md +67 -0
  187. package/skills/setup-freeflow/SKILL.md +54 -68
  188. package/skills/setup-freeflow/references/activation-contract.md +84 -69
  189. package/skills/setup-freeflow/references/host-setup.md +50 -17
  190. package/skills/simplify-code/SKILL.md +80 -0
  191. package/skills/simplify-code/references/simplification-patterns.md +70 -0
  192. package/skills/tdd/SKILL.md +96 -0
  193. package/skills/tdd/references/test-design.md +96 -0
  194. package/skills/track-work/SKILL.md +84 -0
  195. package/skills/track-work/references/working-record-schema.md +378 -0
  196. package/skills/verify-work/SKILL.md +59 -38
  197. package/skills/verify-work/references/browser-runtime-evidence.md +55 -0
  198. package/skills/verify-work/references/integration-evidence.md +51 -0
  199. package/skills/verify-work/references/performance-evidence.md +57 -0
  200. package/skills/workflow/SKILL.md +53 -59
  201. package/skills/workflow/references/domain-skill-composition.md +57 -0
  202. package/skills/workflow/references/workflow-loop.md +92 -0
  203. package/skills/write-plan/SKILL.md +85 -58
  204. package/skills/write-plan/references/plan-shapes.md +132 -46
  205. package/skills/write-skill/SKILL.md +82 -51
  206. package/skills/write-skill/references/activation-boundaries.md +62 -0
  207. package/skills/write-skill/references/agent-first-instructions.md +87 -0
  208. package/skills/write-skill/references/progressive-disclosure.md +63 -0
  209. package/skills/write-skill/scripts/lib/skill-author.mjs +653 -0
  210. package/skills/write-skill/scripts/skill-author.mjs +104 -0
  211. package/skills/write-spec/SKILL.md +65 -54
  212. package/skills/write-spec/references/artifact-standards.md +60 -41
  213. package/skills/write-spec/references/decision-records.md +77 -0
  214. package/skills/write-spec/references/spec-shapes.md +119 -60
  215. package/evals/README.md +0 -32
  216. package/evals/fixtures/copy-conventions.md +0 -31
  217. package/evals/fixtures/output-router/README.md +0 -12
  218. package/evals/fixtures/output-router/failed-command-output.txt +0 -9
  219. package/evals/fixtures/output-router/large-router-manual.md +0 -63
  220. package/evals/fixtures/output-router/native-large-output.txt +0 -40
  221. package/evals/fixtures/output-router/noisy-test-output.txt +0 -30
  222. package/evals/fixtures/output-router/verification-output.txt +0 -8
  223. package/evals/fixtures/tiny-artifact-review-app/README.md +0 -5
  224. package/evals/fixtures/tiny-artifact-review-app/docs/billing-policy.md +0 -10
  225. package/evals/fixtures/tiny-artifact-review-app/docs/specs/billing-webhook-api-draft.md +0 -21
  226. package/evals/fixtures/tiny-artifact-review-app/docs/specs/failed-payment-grace-period.md +0 -24
  227. package/evals/fixtures/tiny-artifact-review-app/docs/specs/immediate-downgrade.md +0 -17
  228. package/evals/fixtures/tiny-artifact-review-app/tests/billing.test.md +0 -8
  229. package/evals/fixtures/tiny-commit-ambiguous-app/.freeflow-eval-setup.sh +0 -15
  230. package/evals/fixtures/tiny-commit-ambiguous-app/README.md +0 -3
  231. package/evals/fixtures/tiny-commit-ambiguous-app/src/billing.ts +0 -5
  232. package/evals/fixtures/tiny-commit-ambiguous-app/src/settings-page.ts +0 -1
  233. package/evals/fixtures/tiny-commit-ambiguous-app/tests/settings.test.md +0 -5
  234. package/evals/fixtures/tiny-commit-app/.freeflow-eval-setup.sh +0 -14
  235. package/evals/fixtures/tiny-commit-app/README.md +0 -3
  236. package/evals/fixtures/tiny-commit-app/docs/specs/settings-title.md +0 -5
  237. package/evals/fixtures/tiny-commit-app/src/billing.ts +0 -5
  238. package/evals/fixtures/tiny-commit-app/src/settings-page.ts +0 -1
  239. package/evals/fixtures/tiny-commit-app/tests/settings.test.md +0 -5
  240. package/evals/fixtures/tiny-commit-everything-app/.freeflow-eval-setup.sh +0 -15
  241. package/evals/fixtures/tiny-commit-everything-app/README.md +0 -3
  242. package/evals/fixtures/tiny-commit-everything-app/docs/specs/settings-title.md +0 -5
  243. package/evals/fixtures/tiny-commit-everything-app/src/billing.ts +0 -5
  244. package/evals/fixtures/tiny-commit-everything-app/src/settings-page.ts +0 -1
  245. package/evals/fixtures/tiny-commit-everything-app/tests/settings.test.md +0 -5
  246. package/evals/fixtures/tiny-commit-staged-app/.freeflow-eval-setup.sh +0 -15
  247. package/evals/fixtures/tiny-commit-staged-app/README.md +0 -3
  248. package/evals/fixtures/tiny-commit-staged-app/docs/specs/settings-title.md +0 -5
  249. package/evals/fixtures/tiny-commit-staged-app/src/billing.ts +0 -5
  250. package/evals/fixtures/tiny-commit-staged-app/src/settings-page.ts +0 -1
  251. package/evals/fixtures/tiny-commit-staged-app/tests/settings.test.md +0 -5
  252. package/evals/fixtures/tiny-commit-staged-mixed-app/.freeflow-eval-setup.sh +0 -15
  253. package/evals/fixtures/tiny-commit-staged-mixed-app/README.md +0 -3
  254. package/evals/fixtures/tiny-commit-staged-mixed-app/docs/billing-policy.md +0 -3
  255. package/evals/fixtures/tiny-commit-staged-mixed-app/docs/release-notes.md +0 -3
  256. package/evals/fixtures/tiny-commit-staged-mixed-app/src/billing.ts +0 -5
  257. package/evals/fixtures/tiny-dashboard-stale-app/README.md +0 -11
  258. package/evals/fixtures/tiny-dashboard-stale-app/src/api-client.ts +0 -13
  259. package/evals/fixtures/tiny-dashboard-stale-app/src/dashboard-data.ts +0 -19
  260. package/evals/fixtures/tiny-dashboard-stale-app/tests/dashboard-data.test.md +0 -9
  261. package/evals/fixtures/tiny-decision-destination-app/README.md +0 -6
  262. package/evals/fixtures/tiny-decision-destination-app/src/risk.ts +0 -8
  263. package/evals/fixtures/tiny-decision-destination-app/tests/risk.test.md +0 -4
  264. package/evals/fixtures/tiny-design-depth-app/README.md +0 -15
  265. package/evals/fixtures/tiny-design-depth-app/src/routes.ts +0 -25
  266. package/evals/fixtures/tiny-design-depth-app/src/sendEmail.ts +0 -3
  267. package/evals/fixtures/tiny-design-depth-app/src/sendSms.ts +0 -3
  268. package/evals/fixtures/tiny-design-depth-app/src/telemetry.ts +0 -3
  269. package/evals/fixtures/tiny-design-depth-app/tests/routes.test.md +0 -12
  270. package/evals/fixtures/tiny-eval-method-app/README.md +0 -8
  271. package/evals/fixtures/tiny-eval-method-app/evals/reports/review-pr-eval-result.md +0 -5
  272. package/evals/fixtures/tiny-eval-method-app/evals/reports/triage-question-failure.md +0 -17
  273. package/evals/fixtures/tiny-eval-method-app/evals/runs/review-pr-with-skill-output.diff +0 -11
  274. package/evals/fixtures/tiny-eval-method-app/evals/runs/review-pr-with-skill-output.md +0 -3
  275. package/evals/fixtures/tiny-eval-method-app/skills/review-pr/SKILL.md +0 -10
  276. package/evals/fixtures/tiny-eval-method-app/skills/triage-question/SKILL.md +0 -10
  277. package/evals/fixtures/tiny-execute-plan-app/README.md +0 -5
  278. package/evals/fixtures/tiny-execute-plan-app/checks/settings-and-billing-check.sh +0 -5
  279. package/evals/fixtures/tiny-execute-plan-app/docs/billing-policy.md +0 -10
  280. package/evals/fixtures/tiny-execute-plan-app/docs/plans/immediate-downgrade-plan.md +0 -11
  281. package/evals/fixtures/tiny-execute-plan-app/docs/plans/settings-title-plan.md +0 -12
  282. package/evals/fixtures/tiny-execute-plan-app/docs/plans/settings-title-with-bad-check.md +0 -15
  283. package/evals/fixtures/tiny-execute-plan-app/docs/plans/unverified-billing-plan.md +0 -9
  284. package/evals/fixtures/tiny-execute-plan-app/docs/specs/failed-payment-grace-period.md +0 -9
  285. package/evals/fixtures/tiny-execute-plan-app/docs/specs/settings-title.md +0 -9
  286. package/evals/fixtures/tiny-execute-plan-app/src/billing.ts +0 -7
  287. package/evals/fixtures/tiny-execute-plan-app/src/settings-page.ts +0 -1
  288. package/evals/fixtures/tiny-execute-plan-app/tests/billing.test.md +0 -7
  289. package/evals/fixtures/tiny-execute-plan-app/tests/settings.test.md +0 -5
  290. package/evals/fixtures/tiny-execute-review-loop-app/README.md +0 -5
  291. package/evals/fixtures/tiny-execute-review-loop-app/docs/billing-policy.md +0 -9
  292. package/evals/fixtures/tiny-execute-review-loop-app/docs/plans/settings-title-plan.md +0 -12
  293. package/evals/fixtures/tiny-execute-review-loop-app/docs/review/previous-agent-summary.md +0 -15
  294. package/evals/fixtures/tiny-execute-review-loop-app/docs/review/review-pass-1-non-pass.md +0 -11
  295. package/evals/fixtures/tiny-execute-review-loop-app/docs/specs/settings-title.md +0 -9
  296. package/evals/fixtures/tiny-execute-review-loop-app/src/billing.ts +0 -7
  297. package/evals/fixtures/tiny-execute-review-loop-app/src/settings-page.ts +0 -1
  298. package/evals/fixtures/tiny-execute-review-loop-app/tests/billing.test.md +0 -7
  299. package/evals/fixtures/tiny-execute-review-loop-app/tests/settings.test.md +0 -5
  300. package/evals/fixtures/tiny-flaky-session-app/README.md +0 -10
  301. package/evals/fixtures/tiny-flaky-session-app/src/session-timeout.ts +0 -13
  302. package/evals/fixtures/tiny-flaky-session-app/tests/session-timeout.test.md +0 -8
  303. package/evals/fixtures/tiny-handoff-app/README.md +0 -5
  304. package/evals/fixtures/tiny-handoff-app/docs/billing-policy.md +0 -10
  305. package/evals/fixtures/tiny-handoff-app/docs/handoffs/old-agent-handoff.md +0 -8
  306. package/evals/fixtures/tiny-handoff-app/docs/plans/billing-retry-plan.md +0 -11
  307. package/evals/fixtures/tiny-handoff-app/src/billing.ts +0 -7
  308. package/evals/fixtures/tiny-handoff-report-app/README.md +0 -3
  309. package/evals/fixtures/tiny-handoff-report-app/evals/runs/handoff-3/hof-003/with-skill-output.diff +0 -1
  310. package/evals/fixtures/tiny-handoff-report-app/evals/runs/handoff-3/hof-003/with-skill-output.md +0 -1
  311. package/evals/fixtures/tiny-handoff-report-app/evals/scores/hof-003-score.txt +0 -4
  312. package/evals/fixtures/tiny-plan-app/README.md +0 -5
  313. package/evals/fixtures/tiny-plan-app/docs/billing-policy.md +0 -10
  314. package/evals/fixtures/tiny-plan-app/docs/specs/failed-payment-grace-period.md +0 -25
  315. package/evals/fixtures/tiny-plan-app/src/billing.ts +0 -17
  316. package/evals/fixtures/tiny-plan-app/src/settings-page.ts +0 -1
  317. package/evals/fixtures/tiny-plan-app/tests/billing.test.md +0 -8
  318. package/evals/fixtures/tiny-plan-app/tests/settings.test.md +0 -5
  319. package/evals/fixtures/tiny-plan-strict-api-app/README.md +0 -3
  320. package/evals/fixtures/tiny-plan-strict-api-app/docs/billing-policy.md +0 -5
  321. package/evals/fixtures/tiny-plan-strict-api-app/docs/specs/billing-webhook-api.md +0 -25
  322. package/evals/fixtures/tiny-plan-strict-api-app/src/billing.ts +0 -5
  323. package/evals/fixtures/tiny-plan-strict-api-app/src/webhooks.ts +0 -3
  324. package/evals/fixtures/tiny-plan-strict-api-app/tests/billing-webhook.test.md +0 -8
  325. package/evals/fixtures/tiny-post-setup-invalid-config-app/.freeflow/config.json +0 -3
  326. package/evals/fixtures/tiny-post-setup-invalid-config-app/AGENTS.md +0 -15
  327. package/evals/fixtures/tiny-post-setup-invalid-config-app/README.md +0 -16
  328. package/evals/fixtures/tiny-post-setup-invalid-config-app/docs/auth-review.md +0 -15
  329. package/evals/fixtures/tiny-post-setup-invalid-config-app/docs/billing-policy.md +0 -10
  330. package/evals/fixtures/tiny-post-setup-invalid-config-app/docs/copy-conventions.md +0 -15
  331. package/evals/fixtures/tiny-post-setup-invalid-config-app/src/auth-middleware.ts +0 -12
  332. package/evals/fixtures/tiny-post-setup-invalid-config-app/src/billing.ts +0 -20
  333. package/evals/fixtures/tiny-post-setup-invalid-config-app/src/login-copy.ts +0 -1
  334. package/evals/fixtures/tiny-post-setup-invalid-config-app/src/settings-page.ts +0 -1
  335. package/evals/fixtures/tiny-post-setup-invalid-config-app/tests/auth-middleware.test.md +0 -8
  336. package/evals/fixtures/tiny-post-setup-invalid-config-app/tests/billing.test.md +0 -8
  337. package/evals/fixtures/tiny-post-setup-invalid-config-app/tests/copy.test.md +0 -7
  338. package/evals/fixtures/tiny-post-setup-missing-config-app/AGENTS.md +0 -15
  339. package/evals/fixtures/tiny-post-setup-missing-config-app/README.md +0 -16
  340. package/evals/fixtures/tiny-post-setup-missing-config-app/docs/auth-review.md +0 -15
  341. package/evals/fixtures/tiny-post-setup-missing-config-app/docs/billing-policy.md +0 -10
  342. package/evals/fixtures/tiny-post-setup-missing-config-app/docs/copy-conventions.md +0 -15
  343. package/evals/fixtures/tiny-post-setup-missing-config-app/src/auth-middleware.ts +0 -12
  344. package/evals/fixtures/tiny-post-setup-missing-config-app/src/billing.ts +0 -20
  345. package/evals/fixtures/tiny-post-setup-missing-config-app/src/login-copy.ts +0 -1
  346. package/evals/fixtures/tiny-post-setup-missing-config-app/src/settings-page.ts +0 -1
  347. package/evals/fixtures/tiny-post-setup-missing-config-app/tests/auth-middleware.test.md +0 -8
  348. package/evals/fixtures/tiny-post-setup-missing-config-app/tests/billing.test.md +0 -8
  349. package/evals/fixtures/tiny-post-setup-missing-config-app/tests/copy.test.md +0 -7
  350. package/evals/fixtures/tiny-post-setup-source-conflict-app/.freeflow/config.json +0 -3
  351. package/evals/fixtures/tiny-post-setup-source-conflict-app/AGENTS.md +0 -15
  352. package/evals/fixtures/tiny-post-setup-source-conflict-app/README.md +0 -16
  353. package/evals/fixtures/tiny-post-setup-source-conflict-app/docs/auth-review.md +0 -15
  354. package/evals/fixtures/tiny-post-setup-source-conflict-app/docs/billing-policy.md +0 -10
  355. package/evals/fixtures/tiny-post-setup-source-conflict-app/docs/copy-conventions.md +0 -15
  356. package/evals/fixtures/tiny-post-setup-source-conflict-app/src/auth-middleware.ts +0 -12
  357. package/evals/fixtures/tiny-post-setup-source-conflict-app/src/billing.ts +0 -20
  358. package/evals/fixtures/tiny-post-setup-source-conflict-app/src/login-copy.ts +0 -1
  359. package/evals/fixtures/tiny-post-setup-source-conflict-app/src/settings-page.ts +0 -1
  360. package/evals/fixtures/tiny-post-setup-source-conflict-app/tests/auth-middleware.test.md +0 -8
  361. package/evals/fixtures/tiny-post-setup-source-conflict-app/tests/billing.test.md +0 -8
  362. package/evals/fixtures/tiny-post-setup-source-conflict-app/tests/copy.test.md +0 -7
  363. package/evals/fixtures/tiny-report-performance-app/README.md +0 -11
  364. package/evals/fixtures/tiny-report-performance-app/docs/performance.md +0 -10
  365. package/evals/fixtures/tiny-report-performance-app/src/report-summary.ts +0 -29
  366. package/evals/fixtures/tiny-report-performance-app/tests/report-performance.test.md +0 -8
  367. package/evals/fixtures/tiny-review-request-app/README.md +0 -3
  368. package/evals/fixtures/tiny-review-request-app/docs/billing-policy.md +0 -5
  369. package/evals/fixtures/tiny-review-request-app/docs/plans/billing-webhook-api.md +0 -19
  370. package/evals/fixtures/tiny-review-request-app/docs/review/previous-agent-summary.md +0 -20
  371. package/evals/fixtures/tiny-review-request-app/docs/specs/billing-webhook-api.md +0 -25
  372. package/evals/fixtures/tiny-review-request-app/src/billing.ts +0 -11
  373. package/evals/fixtures/tiny-review-request-app/src/webhooks.ts +0 -19
  374. package/evals/fixtures/tiny-review-request-app/tests/billing-webhook.test.md +0 -8
  375. package/evals/fixtures/tiny-review-work-app/README.md +0 -5
  376. package/evals/fixtures/tiny-review-work-app/docs/billing-policy.md +0 -9
  377. package/evals/fixtures/tiny-review-work-app/docs/plans/settings-title-plan.md +0 -12
  378. package/evals/fixtures/tiny-review-work-app/docs/review/previous-agent-summary.md +0 -15
  379. package/evals/fixtures/tiny-review-work-app/docs/specs/settings-title.md +0 -9
  380. package/evals/fixtures/tiny-review-work-app/src/billing.ts +0 -7
  381. package/evals/fixtures/tiny-review-work-app/src/settings-page.ts +0 -1
  382. package/evals/fixtures/tiny-review-work-app/tests/billing.test.md +0 -7
  383. package/evals/fixtures/tiny-review-work-app/tests/settings.test.md +0 -5
  384. package/evals/fixtures/tiny-saas-app/README.md +0 -16
  385. package/evals/fixtures/tiny-saas-app/docs/auth-review.md +0 -15
  386. package/evals/fixtures/tiny-saas-app/docs/billing-policy.md +0 -10
  387. package/evals/fixtures/tiny-saas-app/docs/copy-conventions.md +0 -15
  388. package/evals/fixtures/tiny-saas-app/docs/specs/settings-title.md +0 -13
  389. package/evals/fixtures/tiny-saas-app/src/auth-middleware.ts +0 -12
  390. package/evals/fixtures/tiny-saas-app/src/billing.ts +0 -20
  391. package/evals/fixtures/tiny-saas-app/src/login-copy.ts +0 -1
  392. package/evals/fixtures/tiny-saas-app/src/settings-page.ts +0 -1
  393. package/evals/fixtures/tiny-saas-app/tests/auth-middleware.test.md +0 -8
  394. package/evals/fixtures/tiny-saas-app/tests/billing.test.md +0 -8
  395. package/evals/fixtures/tiny-saas-app/tests/copy.test.md +0 -7
  396. package/evals/fixtures/tiny-setup-agents-app/AGENTS.md +0 -6
  397. package/evals/fixtures/tiny-setup-both-app/AGENTS.md +0 -5
  398. package/evals/fixtures/tiny-setup-both-app/CLAUDE.md +0 -5
  399. package/evals/fixtures/tiny-setup-claude-app/CLAUDE.md +0 -6
  400. package/evals/fixtures/tiny-setup-codex-rules-app/.codex/rules/shell.rules +0 -5
  401. package/evals/fixtures/tiny-setup-codex-rules-app/AGENTS.md +0 -6
  402. package/evals/fixtures/tiny-setup-conflict-app/AGENTS.md +0 -6
  403. package/evals/fixtures/tiny-setup-existing-block-app/AGENTS.md +0 -14
  404. package/evals/fixtures/tiny-setup-four-rules-app/CLAUDE.md +0 -6
  405. package/evals/fixtures/tiny-skill-dev-app/README.md +0 -10
  406. package/evals/fixtures/tiny-skill-dev-app/evals/prompts/review-pr-001.txt +0 -8
  407. package/evals/fixtures/tiny-skill-dev-app/evals/reports/review-pr-failure.md +0 -17
  408. package/evals/fixtures/tiny-skill-dev-app/skills/review-pr/SKILL.md +0 -13
  409. package/evals/fixtures/tiny-team-app/README.md +0 -5
  410. package/evals/fixtures/tiny-team-app/docs/product.md +0 -6
  411. package/evals/fixtures/tiny-team-app/src/teams.ts +0 -9
  412. package/evals/fixtures/tiny-team-app/src/users.ts +0 -5
  413. package/evals/fixtures/tiny-team-app/tests/teams.test.md +0 -7
  414. package/evals/fixtures/tiny-verify-app/README.md +0 -5
  415. package/evals/fixtures/tiny-verify-app/src/settings-nav.ts +0 -1
  416. package/evals/fixtures/tiny-verify-app/src/settings-page.ts +0 -1
  417. package/evals/fixtures/tiny-verify-app/tests/settings.test.md +0 -7
  418. package/evals/prompts/aon-001.txt +0 -2
  419. package/evals/prompts/byp-001.txt +0 -2
  420. package/evals/prompts/byp-002.txt +0 -2
  421. package/evals/prompts/cap-001.txt +0 -3
  422. package/evals/prompts/cap-002.txt +0 -3
  423. package/evals/prompts/cmd-001.txt +0 -3
  424. package/evals/prompts/cmd-002.txt +0 -3
  425. package/evals/prompts/cmd-003.txt +0 -6
  426. package/evals/prompts/cmd-004.txt +0 -5
  427. package/evals/prompts/cmd-005.txt +0 -5
  428. package/evals/prompts/cmd-006.txt +0 -5
  429. package/evals/prompts/cmd-007.txt +0 -5
  430. package/evals/prompts/cmd-008.txt +0 -5
  431. package/evals/prompts/cmd-009.txt +0 -8
  432. package/evals/prompts/cmd-010.txt +0 -5
  433. package/evals/prompts/cmd-011.txt +0 -5
  434. package/evals/prompts/cmd-012.txt +0 -5
  435. package/evals/prompts/cmd-013.txt +0 -5
  436. package/evals/prompts/cmd-014.txt +0 -7
  437. package/evals/prompts/cmd-015.txt +0 -7
  438. package/evals/prompts/cmd-016.txt +0 -5
  439. package/evals/prompts/cmt-001.txt +0 -1
  440. package/evals/prompts/cmt-002.txt +0 -1
  441. package/evals/prompts/cmt-003.txt +0 -1
  442. package/evals/prompts/cmt-004.txt +0 -1
  443. package/evals/prompts/cmt-005.txt +0 -1
  444. package/evals/prompts/dfd-001.txt +0 -3
  445. package/evals/prompts/dia-001.txt +0 -3
  446. package/evals/prompts/dia-002.txt +0 -3
  447. package/evals/prompts/dia-003.txt +0 -3
  448. package/evals/prompts/dis-001.txt +0 -5
  449. package/evals/prompts/dis-002.txt +0 -5
  450. package/evals/prompts/esk-001.txt +0 -5
  451. package/evals/prompts/esk-002.txt +0 -5
  452. package/evals/prompts/esk-003.txt +0 -5
  453. package/evals/prompts/grc-001.txt +0 -3
  454. package/evals/prompts/hof-001.txt +0 -3
  455. package/evals/prompts/hof-002.txt +0 -1
  456. package/evals/prompts/hof-003.txt +0 -1
  457. package/evals/prompts/hof-004.txt +0 -1
  458. package/evals/prompts/hof-005.txt +0 -1
  459. package/evals/prompts/ivg-001.txt +0 -1
  460. package/evals/prompts/ivg-002.txt +0 -11
  461. package/evals/prompts/ivg-003.txt +0 -1
  462. package/evals/prompts/ivg-004.txt +0 -1
  463. package/evals/prompts/mode-001.txt +0 -3
  464. package/evals/prompts/mode-002.txt +0 -1
  465. package/evals/prompts/mode-003.txt +0 -3
  466. package/evals/prompts/mode-004.txt +0 -3
  467. package/evals/prompts/mode-005.txt +0 -3
  468. package/evals/prompts/mode-006.txt +0 -3
  469. package/evals/prompts/mode-007.txt +0 -3
  470. package/evals/prompts/otr-001.txt +0 -3
  471. package/evals/prompts/otr-002.txt +0 -3
  472. package/evals/prompts/rar-001.txt +0 -5
  473. package/evals/prompts/rar-002.txt +0 -3
  474. package/evals/prompts/rar-003.txt +0 -6
  475. package/evals/prompts/rar-004.txt +0 -15
  476. package/evals/prompts/rar-005.txt +0 -10
  477. package/evals/prompts/rbr-001.txt +0 -3
  478. package/evals/prompts/rbr-002.txt +0 -3
  479. package/evals/prompts/rbr-003.txt +0 -7
  480. package/evals/prompts/rev-001.txt +0 -3
  481. package/evals/prompts/rev-002.txt +0 -5
  482. package/evals/prompts/rev-003.txt +0 -6
  483. package/evals/prompts/rev-004.txt +0 -1
  484. package/evals/prompts/rev-005.txt +0 -15
  485. package/evals/prompts/rev-006.txt +0 -5
  486. package/evals/prompts/stp-001.txt +0 -1
  487. package/evals/prompts/stp-002.txt +0 -1
  488. package/evals/prompts/stp-003.txt +0 -1
  489. package/evals/prompts/stp-004.txt +0 -1
  490. package/evals/prompts/stp-005.txt +0 -1
  491. package/evals/prompts/stp-006.txt +0 -1
  492. package/evals/prompts/stp-007.txt +0 -1
  493. package/evals/prompts/stp-008.txt +0 -1
  494. package/evals/prompts/stp-009.txt +0 -1
  495. package/evals/prompts/stp-010.txt +0 -1
  496. package/evals/prompts/stp-011.txt +0 -1
  497. package/evals/prompts/vfy-003.txt +0 -1
  498. package/evals/prompts/wpl-001.txt +0 -3
  499. package/evals/prompts/wpl-002.txt +0 -5
  500. package/evals/prompts/wpl-003.txt +0 -3
  501. package/evals/prompts/wpl-004.txt +0 -5
  502. package/evals/prompts/wpl-005.txt +0 -1
  503. package/evals/prompts/wrc-001.txt +0 -10
  504. package/evals/prompts/wrc-002.txt +0 -3
  505. package/evals/prompts/wsk-001.txt +0 -6
  506. package/evals/prompts/wsp-001.txt +0 -5
  507. package/evals/prompts/wsp-002.txt +0 -15
  508. package/evals/prompts/wsp-003.txt +0 -3
  509. package/evals/prompts/wsp-004.txt +0 -15
  510. package/evals/prompts/wsp-005.txt +0 -5
  511. package/evals/prompts/wsp-006.txt +0 -12
  512. package/evals/prompts/wsp-007.txt +0 -9
  513. package/evals/prompts/wsp-008.txt +0 -13
  514. package/evals/prompts/xpl-001.txt +0 -1
  515. package/evals/prompts/xpl-002.txt +0 -3
  516. package/evals/prompts/xpl-003.txt +0 -3
  517. package/evals/prompts/xpl-004.txt +0 -3
  518. package/evals/prompts/xpl-005.txt +0 -5
  519. package/evals/prompts/xpl-006.txt +0 -3
  520. package/evals/registries/adversarial-evals.json +0 -39
  521. package/evals/registries/fixture-evals.json +0 -2513
  522. package/evals/registries/handoff-evals.json +0 -74
  523. package/evals/registries/legacy-evals.json +0 -147
  524. package/evals/registries/skill-evidence.json +0 -193
  525. package/evals/registries/verify-work-evals.json +0 -46
  526. package/evals/reports/acceptance/v0.1-acceptance-report.md +0 -130
  527. package/evals/reports/by-command-surface/command-surface-1-report.md +0 -45
  528. package/evals/reports/by-command-surface/command-surface-10-report.md +0 -76
  529. package/evals/reports/by-command-surface/command-surface-11-report.md +0 -77
  530. package/evals/reports/by-command-surface/command-surface-12-report.md +0 -91
  531. package/evals/reports/by-command-surface/command-surface-13-report.md +0 -64
  532. package/evals/reports/by-command-surface/command-surface-14-report.md +0 -66
  533. package/evals/reports/by-command-surface/command-surface-15-report.md +0 -124
  534. package/evals/reports/by-command-surface/command-surface-16-report.md +0 -39
  535. package/evals/reports/by-command-surface/command-surface-2-report.md +0 -80
  536. package/evals/reports/by-command-surface/command-surface-3-report.md +0 -86
  537. package/evals/reports/by-command-surface/command-surface-4-report.md +0 -76
  538. package/evals/reports/by-command-surface/command-surface-5-report.md +0 -80
  539. package/evals/reports/by-command-surface/command-surface-6-report.md +0 -74
  540. package/evals/reports/by-command-surface/command-surface-7-report.md +0 -84
  541. package/evals/reports/by-command-surface/command-surface-8-report.md +0 -79
  542. package/evals/reports/by-command-surface/command-surface-9-report.md +0 -79
  543. package/evals/reports/by-command-surface/command-surface-matrix.md +0 -91
  544. package/evals/reports/by-skill/bypass-1-report.md +0 -124
  545. package/evals/reports/by-skill/capture-decisions-1-report.md +0 -99
  546. package/evals/reports/by-skill/capture-decisions-2-report.md +0 -102
  547. package/evals/reports/by-skill/commit-work-1-report.md +0 -165
  548. package/evals/reports/by-skill/commit-work-2-report.md +0 -109
  549. package/evals/reports/by-skill/design-for-depth-1-report.md +0 -95
  550. package/evals/reports/by-skill/diagnose-failure-1-report.md +0 -91
  551. package/evals/reports/by-skill/diagnose-failure-2-report.md +0 -94
  552. package/evals/reports/by-skill/discover-1-report.md +0 -118
  553. package/evals/reports/by-skill/evaluate-skill-2-report.md +0 -71
  554. package/evals/reports/by-skill/execute-plan-1-report.md +0 -103
  555. package/evals/reports/by-skill/execute-plan-2-report.md +0 -112
  556. package/evals/reports/by-skill/execute-plan-3-report.md +0 -87
  557. package/evals/reports/by-skill/grill-context-1-report.md +0 -103
  558. package/evals/reports/by-skill/handoff-1-report.md +0 -123
  559. package/evals/reports/by-skill/handoff-2-report.md +0 -105
  560. package/evals/reports/by-skill/handoff-3-report.md +0 -100
  561. package/evals/reports/by-skill/handoff-7-report.md +0 -77
  562. package/evals/reports/by-skill/interview-gate-1-report.md +0 -63
  563. package/evals/reports/by-skill/interview-gate-2-report.md +0 -105
  564. package/evals/reports/by-skill/mode-contract-1-report.md +0 -74
  565. package/evals/reports/by-skill/mode-contract-2-report.md +0 -87
  566. package/evals/reports/by-skill/mode-contract-3-report.md +0 -65
  567. package/evals/reports/by-skill/mode-contract-4-report.md +0 -71
  568. package/evals/reports/by-skill/mode-contract-5-report.md +0 -70
  569. package/evals/reports/by-skill/output-router-1-report.md +0 -76
  570. package/evals/reports/by-skill/output-router-2-report.md +0 -65
  571. package/evals/reports/by-skill/research-1-report.md +0 -144
  572. package/evals/reports/by-skill/research-brief-1-report.md +0 -94
  573. package/evals/reports/by-skill/research-brief-2-report.md +0 -75
  574. package/evals/reports/by-skill/review-artifact-1-report.md +0 -91
  575. package/evals/reports/by-skill/review-artifact-2-report.md +0 -79
  576. package/evals/reports/by-skill/review-artifact-3-report.md +0 -120
  577. package/evals/reports/by-skill/review-artifact-4-report.md +0 -89
  578. package/evals/reports/by-skill/review-artifact-5-report.md +0 -66
  579. package/evals/reports/by-skill/review-work-1-report.md +0 -85
  580. package/evals/reports/by-skill/review-work-2-report.md +0 -87
  581. package/evals/reports/by-skill/review-work-3-report.md +0 -104
  582. package/evals/reports/by-skill/review-work-4-report.md +0 -108
  583. package/evals/reports/by-skill/review-work-5-report.md +0 -73
  584. package/evals/reports/by-skill/review-work-6-report.md +0 -66
  585. package/evals/reports/by-skill/setup-freeflow-1-report.md +0 -199
  586. package/evals/reports/by-skill/setup-freeflow-2-report.md +0 -105
  587. package/evals/reports/by-skill/setup-freeflow-3-report.md +0 -105
  588. package/evals/reports/by-skill/setup-freeflow-4-report.md +0 -88
  589. package/evals/reports/by-skill/setup-freeflow-5-report.md +0 -115
  590. package/evals/reports/by-skill/skill-authoring-1-report.md +0 -81
  591. package/evals/reports/by-skill/verify-work-1-report.md +0 -111
  592. package/evals/reports/by-skill/verify-work-2-report.md +0 -120
  593. package/evals/reports/by-skill/workflow-1-report.md +0 -65
  594. package/evals/reports/by-skill/write-plan-1-report.md +0 -123
  595. package/evals/reports/by-skill/write-plan-2-report.md +0 -98
  596. package/evals/reports/by-skill/write-plan-3-report.md +0 -110
  597. package/evals/reports/by-skill/write-spec-1-report.md +0 -88
  598. package/evals/reports/by-skill/write-spec-2-report.md +0 -89
  599. package/evals/reports/by-skill/write-spec-3-report.md +0 -90
  600. package/evals/reports/by-skill/write-spec-4-report.md +0 -167
  601. package/evals/reports/harness/claude-harness-1-report.md +0 -103
  602. package/evals/reports/harness/eval-harness-1-report.md +0 -91
  603. package/evals/reports/iterations/iteration-1-report.md +0 -71
  604. package/evals/reports/iterations/iteration-2-report.md +0 -78
  605. package/evals/reports/iterations/iteration-3-report.md +0 -118
  606. package/evals/reports/iterations/iteration-4-report.md +0 -131
  607. package/evals/reports/iterations/iteration-5-report.md +0 -149
  608. package/evals/reports/iterations/iteration-6-report.md +0 -136
  609. package/evals/reports/runtime/always-on-runtime-1-report.md +0 -107
  610. package/evals/reports/runtime/context-mode-normalized-benchmark-1-report.md +0 -52
  611. package/evals/reports/runtime/context-mode-real-deep-baseline-1-report.md +0 -410
  612. package/evals/reports/runtime/context-mode-real-deep-final-slice-11-report.md +0 -203
  613. package/evals/reports/runtime/context-mode-real-deep-final-slice-11-review.md +0 -51
  614. package/evals/reports/runtime/context-saving-native-tools-baseline-1-report.md +0 -99
  615. package/evals/reports/runtime/eryx-python-compatibility-reprobe-1-report.md +0 -166
  616. package/evals/reports/runtime/eryx-python-proof-spike-1-report.md +0 -96
  617. package/evals/reports/runtime/eryx-python-proof-spike-2-report.md +0 -151
  618. package/evals/reports/runtime/eryx-python-script-derive-execution-1-report.md +0 -233
  619. package/evals/reports/runtime/jq-script-derive-execution-1-report.md +0 -215
  620. package/evals/reports/runtime/jq-wasm-proof-spike-1-report.md +0 -67
  621. package/evals/reports/runtime/output-router-benchmark-1-report.md +0 -81
  622. package/evals/reports/runtime/output-router-capture-derive-provider-eval-1-report.md +0 -35
  623. package/evals/reports/runtime/output-router-codex-large-index-sqlite-adhoc-report.md +0 -150
  624. package/evals/reports/runtime/output-router-codex-qa-benchmark-1-report.md +0 -46
  625. package/evals/reports/runtime/output-router-command-benchmark-1-report.md +0 -62
  626. package/evals/reports/runtime/output-router-index-benchmark-1-report.md +0 -64
  627. package/evals/reports/runtime/output-router-regression-1-report.md +0 -81
  628. package/evals/reports/runtime/output-router-transform-eval-1-report.md +0 -28
  629. package/evals/reports/runtime/pi-observed-routing-eval-1-report.md +0 -44
  630. package/evals/reports/runtime/quickjs-script-derive-execution-1-report.md +0 -159
  631. package/evals/reports/runtime/quickjs-wasi-proof-spike-1-report.md +0 -55
  632. package/evals/reports/runtime/render-format-row-protocol-1-report.md +0 -58
  633. package/evals/reports/runtime/script-sandbox-adapter-selection-review-1-report.md +0 -84
  634. package/evals/reports/runtime/script-sandbox-probe-resource-hardening-1-report.md +0 -51
  635. package/evals/reports/runtime/storage-policy-benchmark-1-report.md +0 -176
  636. package/evals/reports/runtime/vault-index-storage-spike-1-report.md +0 -39
  637. package/evals/reports/runtime/workflow-context-hook-1-report.md +0 -82
  638. package/evals/runbooks/adversarial-eval-runbook.md +0 -28
  639. package/evals/runbooks/fixture-eval-runbook.md +0 -137
  640. package/evals/runbooks/handoff-eval-runbook.md +0 -22
  641. package/evals/runbooks/verify-work-eval-runbook.md +0 -27
  642. package/evals/scripts/audit-command-surface.sh +0 -160
  643. package/evals/scripts/check-activation-contract.sh +0 -121
  644. package/evals/scripts/check-runtime-context-hook.sh +0 -183
  645. package/evals/scripts/grade-fixture-eval.sh +0 -446
  646. package/evals/scripts/run-claude-fixture-eval.sh +0 -26
  647. package/evals/scripts/run-codex-fixture-eval.sh +0 -26
  648. package/evals/scripts/run-eryx-python-proof-spike.js +0 -559
  649. package/evals/scripts/run-fixture-eval-by-id.sh +0 -23
  650. package/evals/scripts/run-fixture-eval.sh +0 -464
  651. package/evals/scripts/run-jq-wasm-proof-spike.js +0 -299
  652. package/evals/scripts/run-output-router-transform-eval.js +0 -115
  653. package/evals/scripts/run-pi-observed-routing-eval.js +0 -478
  654. package/evals/scripts/run-quickjs-wasi-proof-spike.js +0 -339
  655. package/evals/scripts/skill-evidence.sh +0 -316
  656. package/evals/scripts/validate-release-metadata.sh +0 -486
  657. package/evals/suites/v0.1-acceptance-suite.md +0 -131
  658. package/pi-extension/dist/host-producer-identification.js +0 -186
  659. package/pi-extension/dist/mcp-config.js +0 -30
  660. package/pi-extension/dist/native-safety-net.js +0 -209
  661. package/pi-extension/dist/observed-tool-routing.js +0 -114
  662. package/pi-extension/dist/renderers.js +0 -835
  663. package/pi-extension/dist/router-tools.js +0 -438
  664. package/pi-extension/dist/runtime-context.js +0 -288
  665. package/pi-extension/dist/schemas.js +0 -254
  666. package/pi-extension/dist/status.js +0 -449
  667. package/pi-extension/dist/utils.js +0 -467
  668. package/pi-extension/src/runtime-context.ts +0 -347
  669. package/pi-extension/src/status.ts +0 -505
  670. package/plugin-docs/README.md +0 -40
  671. package/plugin-docs/adr/0001-three-modes.md +0 -27
  672. package/plugin-docs/adr/0002-evals-before-hooks.md +0 -17
  673. package/plugin-docs/adr/0003-release-boundary.md +0 -21
  674. package/plugin-docs/adr/0004-discover-replaces-shallow-discovery-skills.md +0 -26
  675. package/plugin-docs/adr/README.md +0 -10
  676. package/plugin-docs/architecture.md +0 -89
  677. package/plugin-docs/output-router.md +0 -396
  678. package/plugin-docs/release-evidence.md +0 -77
  679. package/plugin-docs/skills.md +0 -44
  680. package/plugin-docs/workflow.md +0 -57
  681. package/skills/diagnose-failure/references/feedback-loop-catalog.md +0 -25
  682. package/skills/discover/ARTIFACT-DESTINATIONS.md +0 -58
  683. package/skills/discover/CHECKPOINTS.md +0 -74
  684. package/skills/discover/SKILL.md +0 -173
  685. package/skills/evaluate-skill/references/eval-patterns.md +0 -41
  686. package/skills/evaluate-skill/references/grading-priority.md +0 -33
  687. package/skills/execute-plan/SKILL.md +0 -133
  688. package/skills/execute-plan/references/execution-map.md +0 -122
  689. package/skills/interview-gate/SKILL.md +0 -90
  690. package/skills/setup-freeflow/references/output-router-setup.md +0 -102
  691. package/skills/workflow/references/workflow-map.md +0 -104
  692. /package/pi-extension/src/{mcp-config.ts → integrations/mcp-config.ts} +0 -0
@@ -1,47 +1,56 @@
1
1
  export interface RunVaultIndexStorageBenchmarkOptions {
2
- iterations?: number;
3
- generatedAt?: string;
2
+ iterations?: number;
3
+ generatedAt?: string;
4
4
  }
5
5
  export interface VaultIndexStorageBenchmarkReport {
6
- generatedAt: string;
7
- iterations: number;
8
- summary: VaultIndexStorageBenchmarkSummary;
9
- candidates: VaultIndexStorageCandidateResult[];
6
+ generatedAt: string;
7
+ iterations: number;
8
+ summary: VaultIndexStorageBenchmarkSummary;
9
+ candidates: VaultIndexStorageCandidateResult[];
10
10
  }
11
11
  export interface VaultIndexStorageBenchmarkSummary {
12
- selectedEngine: string;
13
- selectedReason: string;
14
- fixtures: number;
15
- localSidecarPassed: boolean;
16
- scannerBaselinePassed: boolean;
17
- sqliteFtsStatus: "not-run" | "available" | "failed";
18
- localAppendMs: LatencyPair;
19
- localQueryMs: LatencyPair;
20
- scanQueryMs: LatencyPair;
21
- localQueryReductionPercent: number;
12
+ selectedEngine: string;
13
+ selectedReason: string;
14
+ fixtures: number;
15
+ localSidecarPassed: boolean;
16
+ scannerBaselinePassed: boolean;
17
+ sqliteFtsStatus: "not-run" | "available" | "failed";
18
+ localAppendMs: LatencyPair;
19
+ localQueryMs: LatencyPair;
20
+ scanQueryMs: LatencyPair;
21
+ localQueryReductionPercent: number;
22
22
  }
23
23
  export interface LatencyPair {
24
- p50: number;
25
- p95: number;
24
+ p50: number;
25
+ p95: number;
26
26
  }
27
27
  export interface VaultIndexStorageCandidateResult {
28
- candidate: string;
29
- status: "pass" | "fail" | "not-run";
30
- adopted: boolean;
31
- appendMs?: LatencyPair;
32
- queryMs?: LatencyPair;
33
- rawBytes?: number;
34
- resultBytes?: number;
35
- reductionPercent?: number;
36
- checks: string[];
37
- notes: string[];
28
+ candidate: string;
29
+ status: "pass" | "fail" | "not-run";
30
+ adopted: boolean;
31
+ appendMs?: LatencyPair;
32
+ queryMs?: LatencyPair;
33
+ rawBytes?: number;
34
+ resultBytes?: number;
35
+ reductionPercent?: number;
36
+ checks: string[];
37
+ notes: string[];
38
38
  }
39
- export declare function runVaultIndexStorageBenchmark(options?: RunVaultIndexStorageBenchmarkOptions): Promise<VaultIndexStorageBenchmarkReport>;
39
+ export declare function runVaultIndexStorageBenchmark(
40
+ options?: RunVaultIndexStorageBenchmarkOptions,
41
+ ): Promise<VaultIndexStorageBenchmarkReport>;
40
42
  export declare function renderVaultIndexStorageBenchmarkReport(report: VaultIndexStorageBenchmarkReport): string;
41
- export declare function writeVaultIndexStorageBenchmarkReport(report: VaultIndexStorageBenchmarkReport, reportPath: string): Promise<void>;
42
- export declare function writeVaultIndexStorageBenchmarkReports(report: VaultIndexStorageBenchmarkReport, markdownReportPath: string, options?: {
43
+ export declare function writeVaultIndexStorageBenchmarkReport(
44
+ report: VaultIndexStorageBenchmarkReport,
45
+ reportPath: string,
46
+ ): Promise<void>;
47
+ export declare function writeVaultIndexStorageBenchmarkReports(
48
+ report: VaultIndexStorageBenchmarkReport,
49
+ markdownReportPath: string,
50
+ options?: {
43
51
  jsonReportPath?: string | false;
44
- }): Promise<{
45
- markdown: string;
46
- json?: string;
52
+ },
53
+ ): Promise<{
54
+ markdown: string;
55
+ json?: string;
47
56
  }>;
@@ -3,255 +3,320 @@ import { tmpdir } from "node:os";
3
3
  import { dirname, resolve } from "node:path";
4
4
  import { fileURLToPath } from "node:url";
5
5
  import { performance } from "node:perf_hooks";
6
- import { escapeMarkdownTableCell as escapeTable, formatPercent, latencySummary, normalizeIterations, parseBenchmarkCliArgs, reductionPercent, writeBenchmarkReportPair, } from "./benchmark-harness.js";
6
+ import {
7
+ escapeMarkdownTableCell as escapeTable,
8
+ formatPercent,
9
+ latencySummary,
10
+ normalizeIterations,
11
+ parseBenchmarkCliArgs,
12
+ reductionPercent,
13
+ writeBenchmarkReportPair,
14
+ } from "./benchmark-harness.js";
7
15
  import { createLocalVaultIndex } from "../vault/vault-index.js";
8
- import { createVault, readOutputText, storeCommandOutput, storeMetadataOutput, storeTextOutput } from "../vault/vault.js";
16
+ import {
17
+ createVault,
18
+ readOutputText,
19
+ storeCommandOutput,
20
+ storeMetadataOutput,
21
+ storeTextOutput,
22
+ } from "../vault/vault.js";
9
23
  const DEFAULT_ITERATIONS = 3;
10
- const REPORT_PATH = "evals/reports/runtime/vault-index-storage-spike-1-report.md";
24
+ const REPORT_PATH = "router/evals/reports/vault-index-storage-spike-1-report.md";
11
25
  export async function runVaultIndexStorageBenchmark(options = {}) {
12
- const iterations = normalizeIterations(options.iterations, DEFAULT_ITERATIONS);
13
- const localAppendLatencies = [];
14
- const localQueryLatencies = [];
15
- const scanQueryLatencies = [];
16
- let localPassed = true;
17
- let scanPassed = true;
18
- let localRawBytes = 0;
19
- let localResultBytes = 0;
20
- let fixtureCount = 0;
21
- for (let iteration = 0; iteration < iterations; iteration += 1) {
22
- const tmpRoot = await mkdtemp(resolve(tmpdir(), "freeflow-vault-index-bench-"));
23
- try {
24
- const vault = createVault({ root: tmpRoot });
25
- const index = createLocalVaultIndex(vault);
26
- const emptyStatus = await index.status();
27
- localPassed &&= emptyStatus.entryCount === 0;
28
- const fixtures = await createBenchmarkFixtures(vault, iteration);
29
- fixtureCount = fixtures.length;
30
- for (const fixture of fixtures) {
31
- localAppendLatencies.push(fixture.appendMs);
32
- const visibilityStarted = performance.now();
33
- const visible = await index.queryVault(fixture.query, { sessionId: "vault-index-benchmark" }, { topK: 3 });
34
- localQueryLatencies.push(performance.now() - visibilityStarted);
35
- localPassed &&= visible.matches.some((match) => match.outputId === fixture.expectedOutputId);
36
- if (fixture.forbiddenQuery) {
37
- const forbidden = await index.queryVault(fixture.forbiddenQuery, { sessionId: "vault-index-benchmark" }, { topK: 3 });
38
- localPassed &&= forbidden.matches.length === 0;
39
- }
40
- }
41
- const status = await index.status();
42
- localPassed &&= status.outputCount === fixtures.length;
43
- for (const fixture of fixtures) {
44
- const started = performance.now();
45
- const scanned = await scanVaultFixture(vault, fixture);
46
- scanQueryLatencies.push(performance.now() - started);
47
- scanPassed &&= scanned;
48
- }
49
- const queryResult = await index.queryVault("target", { sessionId: "vault-index-benchmark" }, { topK: 10 });
50
- localRawBytes += fixtures.reduce((total, fixture) => total + Buffer.byteLength(fixture.text ?? JSON.stringify(fixture.record), "utf8"), 0);
51
- localResultBytes += Buffer.byteLength(JSON.stringify(queryResult.matches), "utf8");
52
- }
53
- finally {
54
- await rm(tmpRoot, { recursive: true, force: true });
26
+ const iterations = normalizeIterations(options.iterations, DEFAULT_ITERATIONS);
27
+ const localAppendLatencies = [];
28
+ const localQueryLatencies = [];
29
+ const scanQueryLatencies = [];
30
+ let localPassed = true;
31
+ let scanPassed = true;
32
+ let localRawBytes = 0;
33
+ let localResultBytes = 0;
34
+ let fixtureCount = 0;
35
+ for (let iteration = 0; iteration < iterations; iteration += 1) {
36
+ const tmpRoot = await mkdtemp(resolve(tmpdir(), "freeflow-vault-index-bench-"));
37
+ try {
38
+ const vault = createVault({ root: tmpRoot });
39
+ const index = createLocalVaultIndex(vault);
40
+ const emptyStatus = await index.status();
41
+ localPassed &&= emptyStatus.entryCount === 0;
42
+ const fixtures = await createBenchmarkFixtures(vault, iteration);
43
+ fixtureCount = fixtures.length;
44
+ for (const fixture of fixtures) {
45
+ localAppendLatencies.push(fixture.appendMs);
46
+ const visibilityStarted = performance.now();
47
+ const visible = await index.queryVault(fixture.query, { sessionId: "vault-index-benchmark" }, { topK: 3 });
48
+ localQueryLatencies.push(performance.now() - visibilityStarted);
49
+ localPassed &&= visible.matches.some((match) => match.outputId === fixture.expectedOutputId);
50
+ if (fixture.forbiddenQuery) {
51
+ const forbidden = await index.queryVault(
52
+ fixture.forbiddenQuery,
53
+ { sessionId: "vault-index-benchmark" },
54
+ { topK: 3 },
55
+ );
56
+ localPassed &&= forbidden.matches.length === 0;
55
57
  }
58
+ }
59
+ const status = await index.status();
60
+ localPassed &&= status.outputCount === fixtures.length;
61
+ for (const fixture of fixtures) {
62
+ const started = performance.now();
63
+ const scanned = await scanVaultFixture(vault, fixture);
64
+ scanQueryLatencies.push(performance.now() - started);
65
+ scanPassed &&= scanned;
66
+ }
67
+ const queryResult = await index.queryVault("target", { sessionId: "vault-index-benchmark" }, { topK: 10 });
68
+ localRawBytes += fixtures.reduce(
69
+ (total, fixture) => total + Buffer.byteLength(fixture.text ?? JSON.stringify(fixture.record), "utf8"),
70
+ 0,
71
+ );
72
+ localResultBytes += Buffer.byteLength(JSON.stringify(queryResult.matches), "utf8");
73
+ } finally {
74
+ await rm(tmpRoot, { recursive: true, force: true });
56
75
  }
57
- const localAppend = latencySummary(localAppendLatencies);
58
- const localQuery = latencySummary(localQueryLatencies);
59
- const scanQuery = latencySummary(scanQueryLatencies);
60
- const localReduction = reductionPercent(localRawBytes, localResultBytes);
61
- const sqliteStatus = "not-run";
62
- const candidates = [
63
- {
64
- candidate: "local-json-sidecar",
65
- status: localPassed ? "pass" : "fail",
66
- adopted: true,
67
- appendMs: localAppend,
68
- queryMs: localQuery,
69
- rawBytes: localRawBytes,
70
- resultBytes: localResultBytes,
71
- reductionPercent: localReduction,
72
- checks: [
73
- "fresh index starts empty",
74
- "each persisted append is queryable immediately",
75
- "metadata-only raw content is not indexed",
76
- "no native dependency",
77
- ],
78
- notes: ["Selected for the Slice 11 interface and exercised through the Slice 12 write path because it is portable and dependency-free."],
79
- },
80
- {
81
- candidate: "vault-session-scan-baseline",
82
- status: scanPassed ? "pass" : "fail",
83
- adopted: false,
84
- queryMs: scanQuery,
85
- checks: ["finds fixture evidence by reading vault records directly"],
86
- notes: ["Baseline remains source-truth fallback but query cost grows with vaulted text."],
87
- },
88
- {
89
- candidate: "sqlite-fts",
90
- status: sqliteStatus,
91
- adopted: false,
92
- checks: ["not introduced during Slice 11"],
93
- notes: ["Deferred: adding a native dependency or relying on experimental runtime SQLite needs explicit owner approval."],
94
- },
95
- ];
96
- return {
97
- generatedAt: options.generatedAt ?? new Date().toISOString(),
98
- iterations,
99
- summary: {
100
- selectedEngine: "local-json-sidecar",
101
- selectedReason: "Portable deterministic sidecar files satisfy automatic incremental write-through semantics without adding a native dependency.",
102
- fixtures: fixtureCount,
103
- localSidecarPassed: localPassed,
104
- scannerBaselinePassed: scanPassed,
105
- sqliteFtsStatus: sqliteStatus,
106
- localAppendMs: localAppend,
107
- localQueryMs: localQuery,
108
- scanQueryMs: scanQuery,
109
- localQueryReductionPercent: localReduction,
110
- },
111
- candidates,
112
- };
76
+ }
77
+ const localAppend = latencySummary(localAppendLatencies);
78
+ const localQuery = latencySummary(localQueryLatencies);
79
+ const scanQuery = latencySummary(scanQueryLatencies);
80
+ const localReduction = reductionPercent(localRawBytes, localResultBytes);
81
+ const sqliteStatus = "not-run";
82
+ const candidates = [
83
+ {
84
+ candidate: "local-json-sidecar",
85
+ status: localPassed ? "pass" : "fail",
86
+ adopted: true,
87
+ appendMs: localAppend,
88
+ queryMs: localQuery,
89
+ rawBytes: localRawBytes,
90
+ resultBytes: localResultBytes,
91
+ reductionPercent: localReduction,
92
+ checks: [
93
+ "fresh index starts empty",
94
+ "each persisted append is queryable immediately",
95
+ "metadata-only raw content is not indexed",
96
+ "no native dependency",
97
+ ],
98
+ notes: [
99
+ "Selected for the Slice 11 interface and exercised through the Slice 12 write path because it is portable and dependency-free.",
100
+ ],
101
+ },
102
+ {
103
+ candidate: "vault-session-scan-baseline",
104
+ status: scanPassed ? "pass" : "fail",
105
+ adopted: false,
106
+ queryMs: scanQuery,
107
+ checks: ["finds fixture evidence by reading vault records directly"],
108
+ notes: ["Baseline remains source-truth fallback but query cost grows with vaulted text."],
109
+ },
110
+ {
111
+ candidate: "sqlite-fts",
112
+ status: sqliteStatus,
113
+ adopted: false,
114
+ checks: ["not introduced during Slice 11"],
115
+ notes: [
116
+ "Deferred: adding a native dependency or relying on experimental runtime SQLite needs explicit owner approval.",
117
+ ],
118
+ },
119
+ ];
120
+ return {
121
+ generatedAt: options.generatedAt ?? new Date().toISOString(),
122
+ iterations,
123
+ summary: {
124
+ selectedEngine: "local-json-sidecar",
125
+ selectedReason:
126
+ "Portable deterministic sidecar files satisfy automatic incremental write-through semantics without adding a native dependency.",
127
+ fixtures: fixtureCount,
128
+ localSidecarPassed: localPassed,
129
+ scannerBaselinePassed: scanPassed,
130
+ sqliteFtsStatus: sqliteStatus,
131
+ localAppendMs: localAppend,
132
+ localQueryMs: localQuery,
133
+ scanQueryMs: scanQuery,
134
+ localQueryReductionPercent: localReduction,
135
+ },
136
+ candidates,
137
+ };
113
138
  }
114
139
  async function createBenchmarkFixtures(vault, iteration) {
115
- const commandText = [
116
- "setup noise ".repeat(100),
117
- `alpha target evidence iteration ${iteration}`,
118
- "tail noise ".repeat(100),
119
- ].join("\n");
120
- let started = performance.now();
121
- const command = await storeCommandOutput(vault, {
122
- sessionId: "vault-index-benchmark",
123
- command: "npm test",
124
- stdout: commandText,
125
- stderr: "",
126
- executionStatus: "success",
127
- exitCode: 0,
128
- createdAt: `2026-06-24T00:00:${String(iteration).padStart(2, "0")}.000Z`,
129
- });
130
- const commandAppendMs = performance.now() - started;
131
- const observedText = [
132
- "github issue search",
133
- `beta target issue https://github.com/acme/freeflow/issues/${iteration + 1}`,
134
- "body noise ".repeat(160),
135
- ].join("\n");
136
- started = performance.now();
137
- const observed = await storeTextOutput(vault, {
138
- sessionId: "vault-index-benchmark",
139
- sourceKind: "mcp",
140
- raw: observedText,
141
- producer: { kind: "mcp", server: "github", tool: "search_issues" },
142
- createdAt: `2026-06-24T00:01:${String(iteration).padStart(2, "0")}.000Z`,
143
- });
144
- const observedAppendMs = performance.now() - started;
145
- started = performance.now();
146
- const metadata = await storeMetadataOutput(vault, {
147
- sessionId: "vault-index-benchmark",
148
- sourceKind: "mcp",
149
- rawLineCount: 3,
150
- rawByteCount: 4096,
151
- rawSha256: "b".repeat(64),
152
- metadata: { producer: { kind: "mcp", server: "gmail", tool: "search" }, note: "gamma target metadata only" },
153
- producer: { kind: "mcp", server: "gmail", tool: "search" },
154
- createdAt: `2026-06-24T00:02:${String(iteration).padStart(2, "0")}.000Z`,
155
- });
156
- const metadataAppendMs = performance.now() - started;
157
- return [
158
- { record: command, appendMs: commandAppendMs, text: commandText, stream: "combined", query: "alpha target", expectedOutputId: command.outputId },
159
- { record: observed, appendMs: observedAppendMs, text: observedText, stream: "raw", query: "beta target github", expectedOutputId: observed.outputId },
160
- { record: metadata, appendMs: metadataAppendMs, query: "gmail gamma target", expectedOutputId: metadata.outputId, forbiddenQuery: "PRIVATE_RAW_SECRET" },
161
- ];
140
+ const commandText = [
141
+ "setup noise ".repeat(100),
142
+ `alpha target evidence iteration ${iteration}`,
143
+ "tail noise ".repeat(100),
144
+ ].join("\n");
145
+ let started = performance.now();
146
+ const command = await storeCommandOutput(vault, {
147
+ sessionId: "vault-index-benchmark",
148
+ command: "npm test",
149
+ stdout: commandText,
150
+ stderr: "",
151
+ executionStatus: "success",
152
+ exitCode: 0,
153
+ createdAt: `2026-06-24T00:00:${String(iteration).padStart(2, "0")}.000Z`,
154
+ });
155
+ const commandAppendMs = performance.now() - started;
156
+ const observedText = [
157
+ "github issue search",
158
+ `beta target issue https://github.com/acme/freeflow/issues/${iteration + 1}`,
159
+ "body noise ".repeat(160),
160
+ ].join("\n");
161
+ started = performance.now();
162
+ const observed = await storeTextOutput(vault, {
163
+ sessionId: "vault-index-benchmark",
164
+ sourceKind: "mcp",
165
+ raw: observedText,
166
+ producer: { kind: "mcp", server: "github", tool: "search_issues" },
167
+ createdAt: `2026-06-24T00:01:${String(iteration).padStart(2, "0")}.000Z`,
168
+ });
169
+ const observedAppendMs = performance.now() - started;
170
+ started = performance.now();
171
+ const metadata = await storeMetadataOutput(vault, {
172
+ sessionId: "vault-index-benchmark",
173
+ sourceKind: "mcp",
174
+ rawLineCount: 3,
175
+ rawByteCount: 4096,
176
+ rawSha256: "b".repeat(64),
177
+ metadata: { producer: { kind: "mcp", server: "gmail", tool: "search" }, note: "gamma target metadata only" },
178
+ producer: { kind: "mcp", server: "gmail", tool: "search" },
179
+ createdAt: `2026-06-24T00:02:${String(iteration).padStart(2, "0")}.000Z`,
180
+ });
181
+ const metadataAppendMs = performance.now() - started;
182
+ return [
183
+ {
184
+ record: command,
185
+ appendMs: commandAppendMs,
186
+ text: commandText,
187
+ stream: "combined",
188
+ query: "alpha target",
189
+ expectedOutputId: command.outputId,
190
+ },
191
+ {
192
+ record: observed,
193
+ appendMs: observedAppendMs,
194
+ text: observedText,
195
+ stream: "raw",
196
+ query: "beta target github",
197
+ expectedOutputId: observed.outputId,
198
+ },
199
+ {
200
+ record: metadata,
201
+ appendMs: metadataAppendMs,
202
+ query: "gmail gamma target",
203
+ expectedOutputId: metadata.outputId,
204
+ forbiddenQuery: "PRIVATE_RAW_SECRET",
205
+ },
206
+ ];
162
207
  }
163
208
  async function scanVaultFixture(vault, fixture) {
164
- const text = await textForScan(vault, fixture.record, fixture.stream);
165
- return fixture.query.toLowerCase().split(/\s+/).every((term) => text.toLowerCase().includes(term));
209
+ const text = await textForScan(vault, fixture.record, fixture.stream);
210
+ return fixture.query
211
+ .toLowerCase()
212
+ .split(/\s+/)
213
+ .every((term) => text.toLowerCase().includes(term));
166
214
  }
167
215
  async function textForScan(vault, record, stream) {
168
- if (record.kind === "command") {
169
- return readOutputText(vault, "vault-index-benchmark", record.outputId, stream ?? "combined");
170
- }
171
- if (record.kind === "text") {
172
- return readOutputText(vault, "vault-index-benchmark", record.outputId, "raw");
173
- }
174
- return JSON.stringify(record);
216
+ if (record.kind === "command") {
217
+ return readOutputText(vault, "vault-index-benchmark", record.outputId, stream ?? "combined");
218
+ }
219
+ if (record.kind === "text") {
220
+ return readOutputText(vault, "vault-index-benchmark", record.outputId, "raw");
221
+ }
222
+ return JSON.stringify(record);
175
223
  }
176
224
  export function renderVaultIndexStorageBenchmarkReport(report) {
177
- const date = report.generatedAt.slice(0, 10);
178
- const lines = [
179
- "# Vault Index Storage Spike Report - Iteration 1",
180
- "",
181
- `Date: ${date}`,
182
- "",
183
- "## Scope",
184
- "",
185
- "Slice 11/12 storage-interface and write-path spike for the Freeflow vault index. The benchmark checks that a fresh index starts empty, persisted appends become queryable incrementally through automatic write-through indexing, metadata-only records do not index raw content, and the selected storage engine stays hidden behind the vault-index interface.",
186
- "",
187
- "## Command",
188
- "",
189
- "```sh",
190
- "npm run bench:router:vault-index",
191
- "```",
192
- "",
193
- "## Summary",
194
- "",
195
- `- Fixtures per iteration: ${report.summary.fixtures}`,
196
- `- Iterations: ${report.iterations}`,
197
- `- Selected engine: ${report.summary.selectedEngine}`,
198
- `- Selected reason: ${report.summary.selectedReason}`,
199
- `- Local sidecar passed: ${report.summary.localSidecarPassed ? "yes" : "no"}`,
200
- `- Scanner baseline passed: ${report.summary.scannerBaselinePassed ? "yes" : "no"}`,
201
- `- SQLite FTS status: ${report.summary.sqliteFtsStatus}`,
202
- `- Local append p50/p95: ${report.summary.localAppendMs.p50.toFixed(2)}/${report.summary.localAppendMs.p95.toFixed(2)} ms`,
203
- `- Local query p50/p95: ${report.summary.localQueryMs.p50.toFixed(2)}/${report.summary.localQueryMs.p95.toFixed(2)} ms`,
204
- `- Scan query p50/p95: ${report.summary.scanQueryMs.p50.toFixed(2)}/${report.summary.scanQueryMs.p95.toFixed(2)} ms`,
205
- `- Local query result reduction: ${formatPercent(report.summary.localQueryReductionPercent)}`,
206
- "",
207
- "## Candidates",
208
- "",
209
- "| candidate | status | adopted | append p50/p95 ms | query p50/p95 ms | raw/result bytes | reduction | checks | notes |",
210
- "| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |",
211
- ];
212
- for (const candidate of report.candidates) {
213
- lines.push([
214
- escapeTable(candidate.candidate),
215
- candidate.status,
216
- candidate.adopted ? "yes" : "no",
217
- candidate.appendMs ? `${candidate.appendMs.p50.toFixed(2)}/${candidate.appendMs.p95.toFixed(2)}` : "-",
218
- candidate.queryMs ? `${candidate.queryMs.p50.toFixed(2)}/${candidate.queryMs.p95.toFixed(2)}` : "-",
219
- candidate.rawBytes !== undefined && candidate.resultBytes !== undefined ? `${candidate.rawBytes}/${candidate.resultBytes}` : "-",
220
- candidate.reductionPercent !== undefined ? formatPercent(candidate.reductionPercent) : "-",
221
- escapeTable(candidate.checks.join("; ")),
222
- escapeTable(candidate.notes.join("; ")),
223
- ].join(" | ").replace(/^/, "| ").replace(/$/, " |"));
224
- }
225
- lines.push("", "## Decision", "", "Use the deterministic local JSON sidecar behind the vault-index interface for the next slices. Do not adopt SQLite/FTS in this slice because introducing a native dependency or relying on experimental runtime SQLite needs separate owner approval. The vault remains source truth; the index is a rebuildable sidecar.", "");
226
- return lines.join("\n");
225
+ const date = report.generatedAt.slice(0, 10);
226
+ const lines = [
227
+ "# Vault Index Storage Spike Report - Iteration 1",
228
+ "",
229
+ `Date: ${date}`,
230
+ "",
231
+ "## Scope",
232
+ "",
233
+ "Slice 11/12 storage-interface and write-path spike for the Freeflow vault index. The benchmark checks that a fresh index starts empty, persisted appends become queryable incrementally through automatic write-through indexing, metadata-only records do not index raw content, and the selected storage engine stays hidden behind the vault-index interface.",
234
+ "",
235
+ "## Command",
236
+ "",
237
+ "```sh",
238
+ "npm run bench:router:vault-index",
239
+ "```",
240
+ "",
241
+ "## Summary",
242
+ "",
243
+ `- Fixtures per iteration: ${report.summary.fixtures}`,
244
+ `- Iterations: ${report.iterations}`,
245
+ `- Selected engine: ${report.summary.selectedEngine}`,
246
+ `- Selected reason: ${report.summary.selectedReason}`,
247
+ `- Local sidecar passed: ${report.summary.localSidecarPassed ? "yes" : "no"}`,
248
+ `- Scanner baseline passed: ${report.summary.scannerBaselinePassed ? "yes" : "no"}`,
249
+ `- SQLite FTS status: ${report.summary.sqliteFtsStatus}`,
250
+ `- Local append p50/p95: ${report.summary.localAppendMs.p50.toFixed(2)}/${report.summary.localAppendMs.p95.toFixed(2)} ms`,
251
+ `- Local query p50/p95: ${report.summary.localQueryMs.p50.toFixed(2)}/${report.summary.localQueryMs.p95.toFixed(2)} ms`,
252
+ `- Scan query p50/p95: ${report.summary.scanQueryMs.p50.toFixed(2)}/${report.summary.scanQueryMs.p95.toFixed(2)} ms`,
253
+ `- Local query result reduction: ${formatPercent(report.summary.localQueryReductionPercent)}`,
254
+ "",
255
+ "## Candidates",
256
+ "",
257
+ "| candidate | status | adopted | append p50/p95 ms | query p50/p95 ms | raw/result bytes | reduction | checks | notes |",
258
+ "| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |",
259
+ ];
260
+ for (const candidate of report.candidates) {
261
+ lines.push(
262
+ [
263
+ escapeTable(candidate.candidate),
264
+ candidate.status,
265
+ candidate.adopted ? "yes" : "no",
266
+ candidate.appendMs ? `${candidate.appendMs.p50.toFixed(2)}/${candidate.appendMs.p95.toFixed(2)}` : "-",
267
+ candidate.queryMs ? `${candidate.queryMs.p50.toFixed(2)}/${candidate.queryMs.p95.toFixed(2)}` : "-",
268
+ candidate.rawBytes !== undefined && candidate.resultBytes !== undefined
269
+ ? `${candidate.rawBytes}/${candidate.resultBytes}`
270
+ : "-",
271
+ candidate.reductionPercent !== undefined ? formatPercent(candidate.reductionPercent) : "-",
272
+ escapeTable(candidate.checks.join("; ")),
273
+ escapeTable(candidate.notes.join("; ")),
274
+ ]
275
+ .join(" | ")
276
+ .replace(/^/, "| ")
277
+ .replace(/$/, " |"),
278
+ );
279
+ }
280
+ lines.push(
281
+ "",
282
+ "## Decision",
283
+ "",
284
+ "Use the deterministic local JSON sidecar behind the vault-index interface for the next slices. Do not adopt SQLite/FTS in this slice because introducing a native dependency or relying on experimental runtime SQLite needs separate owner approval. The vault remains source truth; the index is a rebuildable sidecar.",
285
+ "",
286
+ );
287
+ return lines.join("\n");
227
288
  }
228
289
  export async function writeVaultIndexStorageBenchmarkReport(report, reportPath) {
229
- await mkdir(dirname(reportPath), { recursive: true });
230
- await writeFile(reportPath, renderVaultIndexStorageBenchmarkReport(report), "utf8");
290
+ await mkdir(dirname(reportPath), { recursive: true });
291
+ await writeFile(reportPath, renderVaultIndexStorageBenchmarkReport(report), "utf8");
231
292
  }
232
293
  export async function writeVaultIndexStorageBenchmarkReports(report, markdownReportPath, options = {}) {
233
- return writeBenchmarkReportPair({
234
- report,
235
- markdownReportPath,
236
- jsonReportPath: options.jsonReportPath,
237
- renderMarkdown: renderVaultIndexStorageBenchmarkReport,
238
- });
294
+ return writeBenchmarkReportPair({
295
+ report,
296
+ markdownReportPath,
297
+ jsonReportPath: options.jsonReportPath,
298
+ renderMarkdown: renderVaultIndexStorageBenchmarkReport,
299
+ });
239
300
  }
240
301
  async function main() {
241
- const args = parseBenchmarkCliArgs(process.argv.slice(2), { reportPath: REPORT_PATH });
242
- const benchmarkOptions = args.iterations === undefined ? {} : { iterations: args.iterations };
243
- const report = await runVaultIndexStorageBenchmark(benchmarkOptions);
244
- const writeOptions = args.jsonReportPath === undefined ? {} : { jsonReportPath: args.jsonReportPath };
245
- const written = await writeVaultIndexStorageBenchmarkReports(report, resolve(process.cwd(), args.reportPath), writeOptions);
246
- console.log(`Wrote ${written.markdown}`);
247
- if (written.json) {
248
- console.log(`Wrote ${written.json}`);
249
- }
302
+ const args = parseBenchmarkCliArgs(process.argv.slice(2), { reportPath: REPORT_PATH });
303
+ const benchmarkOptions = args.iterations === undefined ? {} : { iterations: args.iterations };
304
+ const report = await runVaultIndexStorageBenchmark(benchmarkOptions);
305
+ const writeOptions = args.jsonReportPath === undefined ? {} : { jsonReportPath: args.jsonReportPath };
306
+ const written = await writeVaultIndexStorageBenchmarkReports(
307
+ report,
308
+ resolve(process.cwd(), args.reportPath),
309
+ writeOptions,
310
+ );
311
+ console.log(`Wrote ${written.markdown}`);
312
+ if (written.json) {
313
+ console.log(`Wrote ${written.json}`);
314
+ }
250
315
  }
251
316
  const isMain = process.argv[1] !== undefined && fileURLToPath(import.meta.url) === resolve(process.argv[1]);
252
317
  if (isMain) {
253
- main().catch((error) => {
254
- console.error(error);
255
- process.exitCode = 1;
256
- });
318
+ main().catch((error) => {
319
+ console.error(error);
320
+ process.exitCode = 1;
321
+ });
257
322
  }