@prestyj/cli 5.30.0 → 5.30.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (334) hide show
  1. package/README.md +54 -39
  2. package/assets/skills/bulletproof/SKILL.md +1 -1
  3. package/assets/skills/clarify/SKILL.md +1 -1
  4. package/assets/skills/code-review/SKILL.md +1 -1
  5. package/assets/skills/compliance-guard/SKILL.md +1 -1
  6. package/assets/skills/durable/SKILL.md +1 -1
  7. package/assets/skills/evidence-led-ui/SKILL.md +1 -1
  8. package/assets/skills/lean/SKILL.md +1 -1
  9. package/assets/skills/refactoring/SKILL.md +1 -1
  10. package/assets/skills/root-cause/SKILL.md +1 -1
  11. package/assets/skills/shared-language/SKILL.md +1 -1
  12. package/assets/skills/tdd/SKILL.md +1 -1
  13. package/dist/app-sidecar/app-settings.d.ts +63 -0
  14. package/dist/app-sidecar/app-settings.js +147 -0
  15. package/dist/app-sidecar/app-settings.js.map +1 -0
  16. package/dist/app-sidecar/attachments.d.ts +11 -0
  17. package/dist/app-sidecar/attachments.js +47 -0
  18. package/dist/app-sidecar/attachments.js.map +1 -0
  19. package/dist/app-sidecar/auth-routes.d.ts +3 -0
  20. package/dist/app-sidecar/auth-routes.js +293 -0
  21. package/dist/app-sidecar/auth-routes.js.map +1 -0
  22. package/dist/app-sidecar/autopilot-runner.d.ts +62 -0
  23. package/dist/app-sidecar/autopilot-runner.js +373 -0
  24. package/dist/app-sidecar/autopilot-runner.js.map +1 -0
  25. package/dist/app-sidecar/file-search.d.ts +15 -0
  26. package/dist/app-sidecar/file-search.js +103 -0
  27. package/dist/app-sidecar/file-search.js.map +1 -0
  28. package/dist/app-sidecar/hf-pull-manager.d.ts +20 -0
  29. package/dist/app-sidecar/hf-pull-manager.js +224 -0
  30. package/dist/app-sidecar/hf-pull-manager.js.map +1 -0
  31. package/dist/app-sidecar/history-routes.d.ts +3 -0
  32. package/dist/app-sidecar/history-routes.js +375 -0
  33. package/dist/app-sidecar/history-routes.js.map +1 -0
  34. package/dist/app-sidecar/history.d.ts +67 -0
  35. package/dist/app-sidecar/history.js +23 -0
  36. package/dist/app-sidecar/history.js.map +1 -0
  37. package/dist/app-sidecar/http.d.ts +5 -0
  38. package/dist/app-sidecar/http.js +27 -0
  39. package/dist/app-sidecar/http.js.map +1 -0
  40. package/dist/app-sidecar/integration-routes.d.ts +3 -0
  41. package/dist/app-sidecar/integration-routes.js +286 -0
  42. package/dist/app-sidecar/integration-routes.js.map +1 -0
  43. package/dist/app-sidecar/json-mode.d.ts +10 -0
  44. package/dist/app-sidecar/json-mode.js +73 -0
  45. package/dist/app-sidecar/json-mode.js.map +1 -0
  46. package/dist/app-sidecar/keep-awake.d.ts +4 -0
  47. package/dist/app-sidecar/keep-awake.js +5 -0
  48. package/dist/app-sidecar/keep-awake.js.map +1 -0
  49. package/dist/app-sidecar/mcp-routes.d.ts +3 -0
  50. package/dist/app-sidecar/mcp-routes.js +185 -0
  51. package/dist/app-sidecar/mcp-routes.js.map +1 -0
  52. package/dist/app-sidecar/mcp-rows.d.ts +31 -0
  53. package/dist/app-sidecar/mcp-rows.js +77 -0
  54. package/dist/app-sidecar/mcp-rows.js.map +1 -0
  55. package/dist/app-sidecar/nolan-context.d.ts +20 -0
  56. package/dist/app-sidecar/nolan-context.js +55 -0
  57. package/dist/app-sidecar/nolan-context.js.map +1 -0
  58. package/dist/app-sidecar/progress-manager.d.ts +10 -0
  59. package/dist/app-sidecar/progress-manager.js +116 -0
  60. package/dist/app-sidecar/progress-manager.js.map +1 -0
  61. package/dist/app-sidecar/prompt-routes.d.ts +3 -0
  62. package/dist/app-sidecar/prompt-routes.js +355 -0
  63. package/dist/app-sidecar/prompt-routes.js.map +1 -0
  64. package/dist/app-sidecar/repo-polls.d.ts +18 -0
  65. package/dist/app-sidecar/repo-polls.js +45 -0
  66. package/dist/app-sidecar/repo-polls.js.map +1 -0
  67. package/dist/app-sidecar/route-context.d.ts +147 -0
  68. package/dist/app-sidecar/route-context.js +2 -0
  69. package/dist/app-sidecar/route-context.js.map +1 -0
  70. package/dist/app-sidecar/run-control-routes.d.ts +3 -0
  71. package/dist/app-sidecar/run-control-routes.js +437 -0
  72. package/dist/app-sidecar/run-control-routes.js.map +1 -0
  73. package/dist/app-sidecar/session-routes.d.ts +3 -0
  74. package/dist/app-sidecar/session-routes.js +115 -0
  75. package/dist/app-sidecar/session-routes.js.map +1 -0
  76. package/dist/app-sidecar/session-types.d.ts +39 -0
  77. package/dist/app-sidecar/session-types.js +5 -0
  78. package/dist/app-sidecar/session-types.js.map +1 -0
  79. package/dist/app-sidecar/task-routes.d.ts +3 -0
  80. package/dist/app-sidecar/task-routes.js +232 -0
  81. package/dist/app-sidecar/task-routes.js.map +1 -0
  82. package/dist/app-sidecar/usage.d.ts +19 -0
  83. package/dist/app-sidecar/usage.js +153 -0
  84. package/dist/app-sidecar/usage.js.map +1 -0
  85. package/dist/app-sidecar/workspace-routes.d.ts +3 -0
  86. package/dist/app-sidecar/workspace-routes.js +374 -0
  87. package/dist/app-sidecar/workspace-routes.js.map +1 -0
  88. package/dist/app-sidecar.js +541 -3971
  89. package/dist/app-sidecar.js.map +1 -1
  90. package/dist/chat-agents/jiwa.js +2 -0
  91. package/dist/chat-agents/jiwa.js.map +1 -1
  92. package/dist/chat-agents/memory.js +2 -0
  93. package/dist/chat-agents/memory.js.map +1 -1
  94. package/dist/cli/auth.js +13 -9
  95. package/dist/cli/auth.js.map +1 -1
  96. package/dist/cli.js +10 -11
  97. package/dist/cli.js.map +1 -1
  98. package/dist/config.js +1 -1
  99. package/dist/core/acceptance-checks.js +3 -9
  100. package/dist/core/acceptance-checks.js.map +1 -1
  101. package/dist/core/agent-session/helpers.d.ts +26 -0
  102. package/dist/core/agent-session/helpers.js +47 -0
  103. package/dist/core/agent-session/helpers.js.map +1 -0
  104. package/dist/core/agent-session/mcp-tools.d.ts +68 -0
  105. package/dist/core/agent-session/mcp-tools.js +195 -0
  106. package/dist/core/agent-session/mcp-tools.js.map +1 -0
  107. package/dist/core/agent-session/run-loop-monitor.d.ts +63 -0
  108. package/dist/core/agent-session/run-loop-monitor.js +164 -0
  109. package/dist/core/agent-session/run-loop-monitor.js.map +1 -0
  110. package/dist/core/agent-session/transcript-markers.d.ts +57 -0
  111. package/dist/core/agent-session/transcript-markers.js +166 -0
  112. package/dist/core/agent-session/transcript-markers.js.map +1 -0
  113. package/dist/core/agent-session/verification-tracker.d.ts +50 -0
  114. package/dist/core/agent-session/verification-tracker.js +269 -0
  115. package/dist/core/agent-session/verification-tracker.js.map +1 -0
  116. package/dist/core/agent-session.d.ts +27 -87
  117. package/dist/core/agent-session.js +166 -713
  118. package/dist/core/agent-session.js.map +1 -1
  119. package/dist/core/auth-providers.js +3 -3
  120. package/dist/core/auth-providers.js.map +1 -1
  121. package/dist/core/autopilot-cycle.d.ts +8 -2
  122. package/dist/core/autopilot-cycle.js +22 -3
  123. package/dist/core/autopilot-cycle.js.map +1 -1
  124. package/dist/core/bundled-agents.js +6 -6
  125. package/dist/core/bundled-agents.js.map +1 -1
  126. package/dist/core/checklist-git.d.ts +19 -0
  127. package/dist/core/checklist-git.js +54 -0
  128. package/dist/core/checklist-git.js.map +1 -0
  129. package/dist/core/checklist-items.d.ts +10 -1
  130. package/dist/core/checklist-items.js +22 -0
  131. package/dist/core/checklist-items.js.map +1 -1
  132. package/dist/core/checklist-prompt.d.ts +14 -3
  133. package/dist/core/checklist-prompt.js +38 -8
  134. package/dist/core/checklist-prompt.js.map +1 -1
  135. package/dist/core/checklist-snapshot.d.ts +5 -1
  136. package/dist/core/checklist-snapshot.js +20 -3
  137. package/dist/core/checklist-snapshot.js.map +1 -1
  138. package/dist/core/checklist-store.d.ts +5 -1
  139. package/dist/core/checklist-store.js +35 -4
  140. package/dist/core/checklist-store.js.map +1 -1
  141. package/dist/core/context-limits.js +1 -1
  142. package/dist/core/context-limits.js.map +1 -1
  143. package/dist/core/event-bus.d.ts +1 -1
  144. package/dist/core/event-bus.js +39 -13
  145. package/dist/core/event-bus.js.map +1 -1
  146. package/dist/core/extensions/plugin-bundles.js +4 -2
  147. package/dist/core/extensions/plugin-bundles.js.map +1 -1
  148. package/dist/core/hashline.d.ts +10 -8
  149. package/dist/core/hashline.js.map +1 -1
  150. package/dist/core/mcp/client.js +2 -0
  151. package/dist/core/mcp/client.js.map +1 -1
  152. package/dist/core/mcp/loopback.d.ts +2 -2
  153. package/dist/core/mcp/loopback.js +2 -2
  154. package/dist/core/mcp/shared-pool.js +4 -3
  155. package/dist/core/mcp/shared-pool.js.map +1 -1
  156. package/dist/core/persistent-shell.js +2 -1
  157. package/dist/core/persistent-shell.js.map +1 -1
  158. package/dist/core/plan-handoff.d.ts +30 -0
  159. package/dist/core/plan-handoff.js +10 -0
  160. package/dist/core/plan-handoff.js.map +1 -0
  161. package/dist/core/plan-submission-gate.d.ts +28 -0
  162. package/dist/core/plan-submission-gate.js +27 -0
  163. package/dist/core/plan-submission-gate.js.map +1 -0
  164. package/dist/core/process-manager.js +9 -1
  165. package/dist/core/process-manager.js.map +1 -1
  166. package/dist/core/project-health-scan.d.ts +109 -0
  167. package/dist/core/project-health-scan.js +673 -0
  168. package/dist/core/project-health-scan.js.map +1 -0
  169. package/dist/core/project-health-score.d.ts +41 -0
  170. package/dist/core/project-health-score.js +257 -0
  171. package/dist/core/project-health-score.js.map +1 -0
  172. package/dist/core/prompt-commands.js +44 -6
  173. package/dist/core/prompt-commands.js.map +1 -1
  174. package/dist/core/session-manager.js.map +1 -1
  175. package/dist/core/session-storage.d.ts +1 -0
  176. package/dist/core/session-storage.js +1 -1
  177. package/dist/core/session-storage.js.map +1 -1
  178. package/dist/core/skills.js +6 -9
  179. package/dist/core/skills.js.map +1 -1
  180. package/dist/core/style-packs/index.js +2 -11
  181. package/dist/core/style-packs/index.js.map +1 -1
  182. package/dist/core/style-packs/packs.d.ts +3 -12
  183. package/dist/core/style-packs/packs.js +132 -173
  184. package/dist/core/style-packs/packs.js.map +1 -1
  185. package/dist/core/subagent-manager.js +11 -6
  186. package/dist/core/subagent-manager.js.map +1 -1
  187. package/dist/core/subagent-receipt.js +3 -4
  188. package/dist/core/subagent-receipt.js.map +1 -1
  189. package/dist/core/ui-adoption.js +2 -0
  190. package/dist/core/ui-adoption.js.map +1 -1
  191. package/dist/core/ui-registry.js +5 -4
  192. package/dist/core/ui-registry.js.map +1 -1
  193. package/dist/core/verification-evidence.d.ts +21 -2
  194. package/dist/core/verification-evidence.js +283 -8
  195. package/dist/core/verification-evidence.js.map +1 -1
  196. package/dist/core/verification-gate.js +4 -2
  197. package/dist/core/verification-gate.js.map +1 -1
  198. package/dist/core/verify-commands.js +2 -3
  199. package/dist/core/verify-commands.js.map +1 -1
  200. package/dist/modes/agent-home-mode.js +1 -4
  201. package/dist/modes/agent-home-mode.js.map +1 -1
  202. package/dist/modes/subagent-worker-mode.js +13 -8
  203. package/dist/modes/subagent-worker-mode.js.map +1 -1
  204. package/dist/system-prompt.js +70 -87
  205. package/dist/system-prompt.js.map +1 -1
  206. package/dist/tools/ask-user.js +13 -74
  207. package/dist/tools/ask-user.js.map +1 -1
  208. package/dist/tools/bash.d.ts +3 -1
  209. package/dist/tools/bash.js +28 -84
  210. package/dist/tools/bash.js.map +1 -1
  211. package/dist/tools/checklist.d.ts +7 -14
  212. package/dist/tools/checklist.js +22 -41
  213. package/dist/tools/checklist.js.map +1 -1
  214. package/dist/tools/code-nav.js +7 -29
  215. package/dist/tools/code-nav.js.map +1 -1
  216. package/dist/tools/edit.d.ts +4 -9
  217. package/dist/tools/edit.js +175 -65
  218. package/dist/tools/edit.js.map +1 -1
  219. package/dist/tools/enter-plan.js +2 -7
  220. package/dist/tools/enter-plan.js.map +1 -1
  221. package/dist/tools/exit-plan.js +2 -3
  222. package/dist/tools/exit-plan.js.map +1 -1
  223. package/dist/tools/find.js +3 -4
  224. package/dist/tools/find.js.map +1 -1
  225. package/dist/tools/grep.js +9 -20
  226. package/dist/tools/grep.js.map +1 -1
  227. package/dist/tools/index.js +10 -7
  228. package/dist/tools/index.js.map +1 -1
  229. package/dist/tools/ls.js +3 -3
  230. package/dist/tools/ls.js.map +1 -1
  231. package/dist/tools/prompt-hints.js +34 -47
  232. package/dist/tools/prompt-hints.js.map +1 -1
  233. package/dist/tools/read-only-bash.js +3 -0
  234. package/dist/tools/read-only-bash.js.map +1 -1
  235. package/dist/tools/read.js +8 -20
  236. package/dist/tools/read.js.map +1 -1
  237. package/dist/tools/search-code.js +4 -12
  238. package/dist/tools/search-code.js.map +1 -1
  239. package/dist/tools/skill.js +13 -7
  240. package/dist/tools/skill.js.map +1 -1
  241. package/dist/tools/subagent-control.js +7 -22
  242. package/dist/tools/subagent-control.js.map +1 -1
  243. package/dist/tools/subagent.js +1 -1
  244. package/dist/tools/subagent.js.map +1 -1
  245. package/dist/tools/task-output.js +4 -15
  246. package/dist/tools/task-output.js.map +1 -1
  247. package/dist/tools/task-send.js +5 -17
  248. package/dist/tools/task-send.js.map +1 -1
  249. package/dist/tools/task-stop.js +2 -2
  250. package/dist/tools/task-stop.js.map +1 -1
  251. package/dist/tools/tool-search.js +2 -7
  252. package/dist/tools/tool-search.js.map +1 -1
  253. package/dist/tools/tool-tiers.d.ts +21 -4
  254. package/dist/tools/tool-tiers.js +44 -17
  255. package/dist/tools/tool-tiers.js.map +1 -1
  256. package/dist/tools/web-fetch.js +7 -33
  257. package/dist/tools/web-fetch.js.map +1 -1
  258. package/dist/tools/write.js +3 -4
  259. package/dist/tools/write.js.map +1 -1
  260. package/dist/ui/App.js +20 -26
  261. package/dist/ui/App.js.map +1 -1
  262. package/dist/ui/components/Footer.js +1 -2
  263. package/dist/ui/components/Footer.js.map +1 -1
  264. package/dist/ui/components/InputArea.js +0 -2
  265. package/dist/ui/components/InputArea.js.map +1 -1
  266. package/dist/ui/components/ToolExecution.js +4 -3
  267. package/dist/ui/components/ToolExecution.js.map +1 -1
  268. package/dist/ui/hooks/useAgentLoop.js +7 -6
  269. package/dist/ui/hooks/useAgentLoop.js.map +1 -1
  270. package/dist/ui/sessions.js +1 -5
  271. package/dist/ui/sessions.js.map +1 -1
  272. package/dist/ui/utils/highlight.d.ts +0 -5
  273. package/dist/ui/utils/highlight.js +3 -49
  274. package/dist/ui/utils/highlight.js.map +1 -1
  275. package/dist/ui/utils/markdown-renderer.d.ts +6 -0
  276. package/dist/ui/utils/markdown-renderer.js +21 -5
  277. package/dist/ui/utils/markdown-renderer.js.map +1 -1
  278. package/dist/utils/process.js +3 -3
  279. package/dist/utils/process.js.map +1 -1
  280. package/dist/utils/shared-poll.d.ts +37 -0
  281. package/dist/utils/shared-poll.js +104 -0
  282. package/dist/utils/shared-poll.js.map +1 -0
  283. package/package.json +8 -8
  284. package/dist/core/api-benchmark.d.ts +0 -63
  285. package/dist/core/api-benchmark.js +0 -381
  286. package/dist/core/api-benchmark.js.map +0 -1
  287. package/dist/core/bash-spawn-benchmark.d.ts +0 -1
  288. package/dist/core/bash-spawn-benchmark.js +0 -139
  289. package/dist/core/bash-spawn-benchmark.js.map +0 -1
  290. package/dist/core/cache-warm-benchmark.d.ts +0 -21
  291. package/dist/core/cache-warm-benchmark.js +0 -170
  292. package/dist/core/cache-warm-benchmark.js.map +0 -1
  293. package/dist/core/fast-apply-benchmark.d.ts +0 -25
  294. package/dist/core/fast-apply-benchmark.js +0 -284
  295. package/dist/core/fast-apply-benchmark.js.map +0 -1
  296. package/dist/core/hashline-edit-benchmark.d.ts +0 -88
  297. package/dist/core/hashline-edit-benchmark.js +0 -518
  298. package/dist/core/hashline-edit-benchmark.js.map +0 -1
  299. package/dist/core/semantic-search-benchmark.d.ts +0 -36
  300. package/dist/core/semantic-search-benchmark.js +0 -211
  301. package/dist/core/semantic-search-benchmark.js.map +0 -1
  302. package/dist/core/speed-benchmark.d.ts +0 -132
  303. package/dist/core/speed-benchmark.js +0 -410
  304. package/dist/core/speed-benchmark.js.map +0 -1
  305. package/dist/ui/activity-phrases.d.ts +0 -10
  306. package/dist/ui/activity-phrases.js +0 -184
  307. package/dist/ui/activity-phrases.js.map +0 -1
  308. package/dist/ui/components/PlanApproval.d.ts +0 -7
  309. package/dist/ui/components/PlanApproval.js +0 -60
  310. package/dist/ui/components/PlanApproval.js.map +0 -1
  311. package/dist/ui/components/PlanBanner.d.ts +0 -5
  312. package/dist/ui/components/PlanBanner.js +0 -31
  313. package/dist/ui/components/PlanBanner.js.map +0 -1
  314. package/dist/ui/components/PlanProgress.d.ts +0 -6
  315. package/dist/ui/components/PlanProgress.js +0 -30
  316. package/dist/ui/components/PlanProgress.js.map +0 -1
  317. package/dist/ui/hooks/useMinDisplayTime.d.ts +0 -10
  318. package/dist/ui/hooks/useMinDisplayTime.js +0 -34
  319. package/dist/ui/hooks/useMinDisplayTime.js.map +0 -1
  320. package/dist/ui/hooks/useSessionManager.d.ts +0 -12
  321. package/dist/ui/hooks/useSessionManager.js +0 -43
  322. package/dist/ui/hooks/useSessionManager.js.map +0 -1
  323. package/dist/ui/hooks/useSlashCommands.d.ts +0 -6
  324. package/dist/ui/hooks/useSlashCommands.js +0 -11
  325. package/dist/ui/hooks/useSlashCommands.js.map +0 -1
  326. package/dist/ui/terminal-history-spacing.d.ts +0 -1
  327. package/dist/ui/terminal-history-spacing.js +0 -2
  328. package/dist/ui/terminal-history-spacing.js.map +0 -1
  329. package/dist/ui/utils/markdown-cache.d.ts +0 -10
  330. package/dist/ui/utils/markdown-cache.js +0 -58
  331. package/dist/ui/utils/markdown-cache.js.map +0 -1
  332. package/dist/ui/utils/token-to-ansi.d.ts +0 -8
  333. package/dist/ui/utils/token-to-ansi.js +0 -323
  334. package/dist/ui/utils/token-to-ansi.js.map +0 -1
@@ -1,139 +0,0 @@
1
- /**
2
- * Per-call spawn (what tools/bash.ts does today) vs a persistent shell session
3
- * (what oh-my-pi's embedded "brush" bash gives them) — local measurement, no
4
- * LLM calls needed.
5
- *
6
- * Strategy A — SPAWN (current): every command is `spawn("bash", ["-c", cmd])`,
7
- * a fresh process with fresh env/cwd. Cost = fork/exec + shell startup per call.
8
- *
9
- * Strategy B — PERSISTENT: one long-lived `bash` process; commands are written
10
- * to stdin and delimited with a sentinel echo. Cost ā‰ˆ pipe round-trip.
11
- * Bonus property (qualitative): cd / env / shell state survive across calls.
12
- *
13
- * We run the same command list through both and report total + per-call wall
14
- * time. Commands are trivial on purpose — we are measuring HARNESS overhead,
15
- * not the commands themselves.
16
- *
17
- * Usage:
18
- * npx tsx src/core/bash-spawn-benchmark.ts
19
- *
20
- * Env overrides:
21
- * EZ_BASH_BENCH_N (commands per strategy, default 30)
22
- */
23
- import { spawn } from "node:child_process";
24
- import { randomUUID } from "node:crypto";
25
- const COMMANDS = ["echo hello", "pwd", "ls / > /dev/null", "printf '%s' done", "true"];
26
- function buildCommandList(n) {
27
- const out = [];
28
- for (let i = 0; i < n; i++)
29
- out.push(COMMANDS[i % COMMANDS.length]);
30
- return out;
31
- }
32
- // ── Strategy A: fresh spawn per call (mirrors tools/bash.ts) ──
33
- function runSpawned(cmd) {
34
- return new Promise((resolve, reject) => {
35
- const start = performance.now();
36
- const child = spawn("bash", ["-c", cmd], { stdio: ["ignore", "pipe", "pipe"] });
37
- child.stdout.on("data", () => { });
38
- child.stderr.on("data", () => { });
39
- child.on("error", reject);
40
- child.on("close", () => resolve(performance.now() - start));
41
- });
42
- }
43
- // ── Strategy B: one persistent bash, sentinel-delimited ──
44
- class PersistentShell {
45
- child = spawn("bash", ["--norc", "--noprofile"], {
46
- stdio: ["pipe", "pipe", "pipe"],
47
- });
48
- buffer = "";
49
- waiter = null;
50
- constructor() {
51
- this.child.stdout.on("data", (d) => {
52
- this.buffer += d.toString();
53
- if (this.waiter && this.buffer.includes(this.waiter.sentinel)) {
54
- const w = this.waiter;
55
- this.waiter = null;
56
- this.buffer = "";
57
- w.resolve();
58
- }
59
- });
60
- this.child.stderr.on("data", () => { });
61
- }
62
- run(cmd) {
63
- const start = performance.now();
64
- const sentinel = `__EZ_DONE_${randomUUID()}__`;
65
- return new Promise((resolve) => {
66
- this.waiter = { sentinel, resolve: () => resolve(performance.now() - start) };
67
- this.child.stdin.write(`${cmd}\necho ${sentinel}\n`);
68
- });
69
- }
70
- /** Prove state persistence: cd + env var survive across run() calls. */
71
- async statePersists() {
72
- await this.run("cd /tmp && export EZ_BENCH_STATE=alive");
73
- const sentinel = `__EZ_STATE_${randomUUID()}__`;
74
- const out = await new Promise((resolve) => {
75
- let acc = "";
76
- const onData = (d) => {
77
- acc += d.toString();
78
- if (acc.includes(sentinel)) {
79
- this.child.stdout.off("data", onData);
80
- resolve(acc);
81
- }
82
- };
83
- this.child.stdout.on("data", onData);
84
- this.child.stdin.write(`pwd; echo "$EZ_BENCH_STATE"; echo ${sentinel}\n`);
85
- });
86
- return out.includes("/tmp") && out.includes("alive");
87
- }
88
- close() {
89
- this.child.stdin.end();
90
- this.child.kill();
91
- }
92
- }
93
- // ── Stats ──
94
- function stats(samples) {
95
- const sorted = [...samples].sort((a, b) => a - b);
96
- const total = samples.reduce((s, x) => s + x, 0);
97
- const at = (q) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))];
98
- return { total, mean: total / samples.length, p50: at(0.5), p95: at(0.95) };
99
- }
100
- function fmt(s) {
101
- return (`total ${s.total.toFixed(0)}ms | mean ${s.mean.toFixed(1)}ms | ` +
102
- `p50 ${s.p50.toFixed(1)}ms | p95 ${s.p95.toFixed(1)}ms`);
103
- }
104
- async function main() {
105
- const n = Math.max(5, parseInt(process.env.EZ_BASH_BENCH_N ?? "30", 10));
106
- const cmds = buildCommandList(n);
107
- console.log(`\n🐚 Bash harness overhead — ${n} commands per strategy\n`);
108
- // Warm both paths once so first-touch costs don't skew either side.
109
- await runSpawned("true");
110
- const shell = new PersistentShell();
111
- await shell.run("true");
112
- const spawnTimes = [];
113
- for (const c of cmds)
114
- spawnTimes.push(await runSpawned(c));
115
- const persistTimes = [];
116
- for (const c of cmds)
117
- persistTimes.push(await shell.run(c));
118
- const persists = await shell.statePersists();
119
- shell.close();
120
- const a = stats(spawnTimes);
121
- const b = stats(persistTimes);
122
- console.log(`SPAWN-PER-CALL (current): ${fmt(a)}`);
123
- console.log(`PERSISTENT SHELL (theirs): ${fmt(b)}`);
124
- const saved = a.mean - b.mean;
125
- console.log(`\nPer-call overhead saved: ${saved.toFixed(1)}ms ` +
126
- `(${((saved / a.mean) * 100).toFixed(0)}% of mean call cost)`);
127
- console.log(`State persistence across calls (cd/env): ${persists ? "YES" : "NO"}`);
128
- console.log(`\nVerdict: persistent shell wins if per-call saving is meaningful at agent scale ` +
129
- `(a long session runs hundreds of bash calls) AND state persistence is a UX win.\n`);
130
- }
131
- const isDirectRun = process.argv[1]?.endsWith("bash-spawn-benchmark.ts") ||
132
- process.argv[1]?.endsWith("bash-spawn-benchmark.js");
133
- if (isDirectRun) {
134
- main().catch((err) => {
135
- console.error("Benchmark failed:", err);
136
- process.exit(1);
137
- });
138
- }
139
- //# sourceMappingURL=bash-spawn-benchmark.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"bash-spawn-benchmark.js","sourceRoot":"","sources":["../../src/core/bash-spawn-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,QAAQ,GAAG,CAAC,YAAY,EAAE,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,CAAC,CAAC;AAEvF,SAAS,gBAAgB,CAAC,CAAS;IACjC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAE,CAAC,CAAC;IACrE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,iEAAiE;AAEjE,SAAS,UAAU,CAAC,GAAW;IAC7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAChF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAClC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAClC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,4DAA4D;AAE5D,MAAM,eAAe;IACX,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE;QACvD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;KAChC,CAAC,CAAC;IACK,MAAM,GAAG,EAAE,CAAC;IACZ,MAAM,GAAqD,IAAI,CAAC;IAExE;QACE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE;YACzC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;gBACtB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;gBACnB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;gBACjB,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,GAAG,CAAC,GAAW;QACb,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,aAAa,UAAU,EAAE,IAAI,CAAC;QAC/C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,IAAI,CAAC,MAAM,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC;YAC9E,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,QAAQ,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,aAAa;QACjB,MAAM,IAAI,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,cAAc,UAAU,EAAE,IAAI,CAAC;QAChD,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,EAAE;YAChD,IAAI,GAAG,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,CAAC,CAAS,EAAQ,EAAE;gBACjC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACpB,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;oBACtC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC;YACH,CAAC,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACrC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,qCAAqC,QAAQ,IAAI,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QACH,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACvD,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;CACF;AAED,cAAc;AAEd,SAAS,KAAK,CAAC,OAAiB;IAC9B,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,MAAM,EAAE,GAAG,CAAC,CAAS,EAAU,EAAE,CAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;IACtE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;AAC9E,CAAC;AAED,SAAS,GAAG,CAAC,CAA4D;IACvE,OAAO,CACL,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO;QAChE,OAAO,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACxD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAEjC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,0BAA0B,CAAC,CAAC;IAEzE,oEAAoE;IACpE,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;IACzB,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAExB,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,UAAU,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAE3D,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,YAAY,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;IAC7C,KAAK,CAAC,KAAK,EAAE,CAAC;IAEd,MAAM,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IAC5B,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;IAE9B,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;IAC9B,OAAO,CAAC,GAAG,CACT,8BAA8B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;QACjD,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAChE,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,4CAA4C,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACnF,OAAO,CAAC,GAAG,CACT,mFAAmF;QACjF,mFAAmF,CACtF,CAAC;AACJ,CAAC;AAED,MAAM,WAAW,GACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,yBAAyB,CAAC;IACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,yBAAyB,CAAC,CAAC;AAEvD,IAAI,WAAW,EAAE,CAAC;IAChB,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACnB,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
@@ -1,21 +0,0 @@
1
- /**
2
- * Prompt-cache warming micro-benchmark — isolates the cache effect.
3
- *
4
- * The existing api-benchmark grows the conversation each turn and uses a tiny
5
- * (~250 token) system prompt, which is BELOW Anthropic's minimum cacheable
6
- * prefix (1024 tok Sonnet/Opus, 2048 Haiku) — so caching never engages and the
7
- * A/B is dominated by turn-to-turn throughput variance.
8
- *
9
- * This test uses a large, FIXED prefix (well above the min) and a tiny variable
10
- * suffix, then measures, per arm:
11
- * COLD — fresh cache key, first hit (cache WRITE, no read).
12
- * WARM — prewarm the prefix, then measure the next call's cache_read + TTFT.
13
- *
14
- * It reports cache_read tokens and TTFT directly, so the cache effect is visible
15
- * regardless of throughput noise.
16
- *
17
- * Usage: npx tsx src/core/cache-warm-benchmark.ts
18
- * Env: EZ_CW_PROVIDER (default anthropic), EZ_CW_MODEL (default claude-haiku-4-5-20251001),
19
- * EZ_CW_REPS (default 4)
20
- */
21
- export {};
@@ -1,170 +0,0 @@
1
- /**
2
- * Prompt-cache warming micro-benchmark — isolates the cache effect.
3
- *
4
- * The existing api-benchmark grows the conversation each turn and uses a tiny
5
- * (~250 token) system prompt, which is BELOW Anthropic's minimum cacheable
6
- * prefix (1024 tok Sonnet/Opus, 2048 Haiku) — so caching never engages and the
7
- * A/B is dominated by turn-to-turn throughput variance.
8
- *
9
- * This test uses a large, FIXED prefix (well above the min) and a tiny variable
10
- * suffix, then measures, per arm:
11
- * COLD — fresh cache key, first hit (cache WRITE, no read).
12
- * WARM — prewarm the prefix, then measure the next call's cache_read + TTFT.
13
- *
14
- * It reports cache_read tokens and TTFT directly, so the cache effect is visible
15
- * regardless of throughput noise.
16
- *
17
- * Usage: npx tsx src/core/cache-warm-benchmark.ts
18
- * Env: EZ_CW_PROVIDER (default anthropic), EZ_CW_MODEL (default claude-haiku-4-5-20251001),
19
- * EZ_CW_REPS (default 4)
20
- */
21
- import { stream } from "@prestyj/ai";
22
- import { AuthStorage } from "./auth-storage.js";
23
- /** Build a large, realistic coding-agent system prompt (~well above 2048 tok). */
24
- function bigPrefix() {
25
- const toolDoc = (name, desc) => `### ${name}\n${desc}\nParameters are validated with a Zod schema and converted to JSON Schema at the provider boundary. ` +
26
- `Errors are returned as structured tool results so the agent can self-correct in the same turn. `;
27
- const tools = [
28
- toolDoc("read", "Read a file's contents with cat -n style numbered lines, truncated to 2000 lines or 50KB."),
29
- toolDoc("write", "Write content to a file, creating parent directories. Existing files must be read first."),
30
- toolDoc("edit", "Replace text via search/replace blocks applied sequentially with a fuzzy fallback ladder."),
31
- toolDoc("bash", "Execute a shell command in a non-interactive bash shell with combined stdout/stderr."),
32
- toolDoc("grep", "Search file contents using regex, returning filepath:line:content matches."),
33
- toolDoc("find", "Find files matching a glob pattern, respecting .gitignore."),
34
- toolDoc("ls", "List directory contents with file types and sizes."),
35
- toolDoc("web_fetch", "Fetch and read web page content as Markdown, extracting main content."),
36
- toolDoc("subagent", "Spawn an isolated sub-agent with its own context window for a focused task."),
37
- toolDoc("task_output", "Read new output from a background process by id."),
38
- ];
39
- // Repeat the tool block several times so the prefix clears Anthropic's
40
- // minimum cacheable size (1024 tok Sonnet/Opus, 2048 Haiku) with headroom.
41
- const bulkTools = [];
42
- for (let r = 0; r < 22; r++)
43
- bulkTools.push(...tools, `<!-- tool block repetition ${r} for prefix sizing -->`);
44
- return [
45
- "You are EZ Coder — a coding agent that works directly in the user's codebase.",
46
- "You explore, understand, change, and verify code end-to-end.",
47
- "",
48
- "## How to Talk",
49
- "Lead with the outcome. One idea per line. Bottom line first. No preamble or recap.",
50
- "Stay silent between tool calls unless you hit a decision, tradeoff, or question.",
51
- "",
52
- "## How to Work",
53
- "Read before edit/write; re-read after formatters, codegen, or any disk mutator.",
54
- "Match neighbouring code style. Keep edits small; plan multi-file work first.",
55
- "Choose targeted verification appropriate to the change; read and fix failures.",
56
- "Do not assume APIs, CLI flags, config schema, or error wording — verify against source.",
57
- "",
58
- "## Tools",
59
- ...bulkTools,
60
- "",
61
- "## Code Quality",
62
- "Intent-revealing names; reuse existing deps. Types first; handle I/O, input, and external API errors.",
63
- "No dead or commented-out code, placeholders, or unasked refactors.",
64
- "",
65
- "## Environment",
66
- "Working directory: /home/user/project. Platform: darwin.",
67
- ].join("\n");
68
- }
69
- async function call(cfg, system, user, promptCacheKey, cacheRetention) {
70
- const messages = [
71
- { role: "system", content: system },
72
- { role: "user", content: user },
73
- ];
74
- const start = Date.now();
75
- let ttftMs = 0;
76
- let first = true;
77
- const result = stream({
78
- provider: cfg.provider,
79
- model: cfg.model,
80
- messages,
81
- maxTokens: 16,
82
- apiKey: cfg.apiKey,
83
- baseUrl: cfg.baseUrl,
84
- accountId: cfg.accountId,
85
- promptCacheKey,
86
- cacheRetention,
87
- });
88
- for await (const event of result) {
89
- if (first && (event.type === "text_delta" || event.type === "thinking_delta")) {
90
- ttftMs = Date.now() - start;
91
- first = false;
92
- }
93
- }
94
- const response = await result.response;
95
- return {
96
- ttftMs: ttftMs || Date.now() - start,
97
- wallMs: Date.now() - start,
98
- inputTokens: response.usage.inputTokens,
99
- cacheRead: response.usage.cacheRead ?? 0,
100
- cacheWrite: response.usage.cacheWrite ?? 0,
101
- outputTokens: response.usage.outputTokens,
102
- };
103
- }
104
- function sleep(ms) {
105
- return new Promise((r) => setTimeout(r, ms));
106
- }
107
- async function main() {
108
- const provider = process.env.EZ_CW_PROVIDER ?? "anthropic";
109
- const model = process.env.EZ_CW_MODEL ?? "claude-haiku-4-5-20251001";
110
- const reps = parseInt(process.env.EZ_CW_REPS ?? "4", 10);
111
- const auth = new AuthStorage();
112
- await auth.load();
113
- const c = await auth.resolveCredentials(provider);
114
- const cfg = {
115
- provider,
116
- model,
117
- apiKey: c.accessToken,
118
- baseUrl: c.baseUrl,
119
- accountId: c.accountId,
120
- };
121
- const system = bigPrefix();
122
- // Rough token size of the prefix (chars/3.3) for context.
123
- console.log(`\nšŸ”„ Cache-warm micro-benchmark: ${provider}/${model}`);
124
- console.log(` Prefix ~${Math.round(system.length / 3.3)} tokens, ${reps} reps per arm\n`);
125
- // ── COLD arm: every rep uses a UNIQUE cache key → never reads cache ──
126
- console.log("ā–¶ COLD (no warming — fresh cache key each call):");
127
- // Each cold rep gets UNIQUE prefix content (Anthropic caches by content, not
128
- // by key) so every call is a true cache miss — otherwise rep 2+ would read
129
- // rep 1's cache and pollute the cold baseline.
130
- const cold = [];
131
- for (let i = 0; i < reps; i++) {
132
- const uniqueSystem = `<!-- cold-sample-${Date.now()}-${i} -->\n${system}`;
133
- const m = await call(cfg, uniqueSystem, `Question ${i}: name one tool.`, `cold-${Date.now()}-${i}`, "short");
134
- cold.push(m);
135
- console.log(` rep ${i + 1}: TTFT ${m.ttftMs}ms | in ${m.inputTokens} | cacheRead ${m.cacheRead} | cacheWrite ${m.cacheWrite}`);
136
- await sleep(1500);
137
- }
138
- await sleep(3000);
139
- // ── WARM arm: prewarm a SHARED key, then measure subsequent calls (cache reads) ──
140
- console.log("\nā–¶ WARM (prewarm shared prefix, then measure cache reads):");
141
- const warmKey = `warm-${Date.now()}`;
142
- // Prewarm: a throwaway call that writes the cache.
143
- const pw = await call(cfg, system, "warm.", warmKey, "long");
144
- console.log(` prewarm: cacheWrite ${pw.cacheWrite} (cache primed)`);
145
- await sleep(1500);
146
- const warm = [];
147
- for (let i = 0; i < reps; i++) {
148
- const m = await call(cfg, system, `Question ${i}: name one tool.`, warmKey, "long");
149
- warm.push(m);
150
- console.log(` rep ${i + 1}: TTFT ${m.ttftMs}ms | in ${m.inputTokens} | cacheRead ${m.cacheRead} | cacheWrite ${m.cacheWrite}`);
151
- await sleep(1500);
152
- }
153
- const avg = (xs, f) => xs.reduce((s, m) => s + f(m), 0) / xs.length;
154
- const coldTtft = avg(cold, (m) => m.ttftMs);
155
- const warmTtft = avg(warm, (m) => m.ttftMs);
156
- const coldRead = avg(cold, (m) => m.cacheRead);
157
- const warmRead = avg(warm, (m) => m.cacheRead);
158
- console.log("\n══════════════════════ RESULTS ══════════════════════");
159
- console.log(`Avg cacheRead tokens : cold ${coldRead.toFixed(0)} → warm ${warmRead.toFixed(0)}`);
160
- console.log(`Avg TTFT : cold ${coldTtft.toFixed(0)}ms → warm ${warmTtft.toFixed(0)}ms ` +
161
- `(${coldTtft > warmTtft ? "-" : "+"}${Math.abs(((coldTtft - warmTtft) / coldTtft) * 100).toFixed(0)}%)`);
162
- const warmHitRate = warm.filter((m) => m.cacheRead > 0).length / warm.length;
163
- console.log(`Warm cache hit rate : ${(warmHitRate * 100).toFixed(0)}%`);
164
- console.log(`Verdict: ${warmRead > 0 ? "caching ENGAGES when prefix exceeds the min — warming converts cold writes to reads" : "caching did NOT engage (prefix below provider minimum, or OAuth path strips cache_control)"}`);
165
- }
166
- main().catch((err) => {
167
- console.error("Benchmark failed:", err);
168
- process.exit(1);
169
- });
170
- //# sourceMappingURL=cache-warm-benchmark.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"cache-warm-benchmark.js","sourceRoot":"","sources":["../../src/core/cache-warm-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,MAAM,EAA8C,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,kFAAkF;AAClF,SAAS,SAAS;IAChB,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE,CAC7C,OAAO,IAAI,KAAK,IAAI,sGAAsG;QAC1H,iGAAiG,CAAC;IACpG,MAAM,KAAK,GAAG;QACZ,OAAO,CACL,MAAM,EACN,2FAA2F,CAC5F;QACD,OAAO,CACL,OAAO,EACP,0FAA0F,CAC3F;QACD,OAAO,CACL,MAAM,EACN,2FAA2F,CAC5F;QACD,OAAO,CACL,MAAM,EACN,sFAAsF,CACvF;QACD,OAAO,CAAC,MAAM,EAAE,4EAA4E,CAAC;QAC7F,OAAO,CAAC,MAAM,EAAE,4DAA4D,CAAC;QAC7E,OAAO,CAAC,IAAI,EAAE,oDAAoD,CAAC;QACnE,OAAO,CAAC,WAAW,EAAE,uEAAuE,CAAC;QAC7F,OAAO,CACL,UAAU,EACV,6EAA6E,CAC9E;QACD,OAAO,CAAC,aAAa,EAAE,kDAAkD,CAAC;KAC3E,CAAC;IACF,uEAAuE;IACvE,2EAA2E;IAC3E,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE;QACzB,SAAS,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,8BAA8B,CAAC,wBAAwB,CAAC,CAAC;IACpF,OAAO;QACL,+EAA+E;QAC/E,8DAA8D;QAC9D,EAAE;QACF,gBAAgB;QAChB,oFAAoF;QACpF,kFAAkF;QAClF,EAAE;QACF,gBAAgB;QAChB,iFAAiF;QACjF,8EAA8E;QAC9E,gFAAgF;QAChF,yFAAyF;QACzF,EAAE;QACF,UAAU;QACV,GAAG,SAAS;QACZ,EAAE;QACF,iBAAiB;QACjB,uGAAuG;QACvG,oEAAoE;QACpE,EAAE;QACF,gBAAgB;QAChB,0DAA0D;KAC3D,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAWD,KAAK,UAAU,IAAI,CACjB,GAMC,EACD,MAAc,EACd,IAAY,EACZ,cAAsB,EACtB,cAAgC;IAEhC,MAAM,QAAQ,GAAc;QAC1B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;QACnC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;KAChC,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,IAAI,CAAC;IACjB,MAAM,MAAM,GAAG,MAAM,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC,QAAiB;QAC/B,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ;QACR,SAAS,EAAE,EAAE;QACb,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc;QACd,cAAc;KACf,CAAC,CAAC;IACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAoC,EAAE,CAAC;QAC/D,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC,EAAE,CAAC;YAC9E,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YAC5B,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAA2D,MAAM,MAAM,CAAC,QAAQ,CAAC;IAC/F,OAAO;QACL,MAAM,EAAE,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;QACpC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;QAC1B,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,WAAW;QACvC,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC;QACxC,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC;QAC1C,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;KAC1C,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,WAAW,CAAC;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,2BAA2B,CAAC;IACrE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IAEzD,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;IAC/B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IAClB,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG;QACV,QAAQ;QACR,KAAK;QACL,MAAM,EAAE,CAAC,CAAC,WAAW;QACrB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,SAAS,EAAE,CAAC,CAAC,SAAS;KACvB,CAAC;IAEF,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,0DAA0D;IAC1D,OAAO,CAAC,GAAG,CAAC,oCAAoC,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,YAAY,IAAI,iBAAiB,CAAC,CAAC;IAE5F,wEAAwE;IACxE,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;IAChE,6EAA6E;IAC7E,2EAA2E;IAC3E,+CAA+C;IAC/C,MAAM,IAAI,GAAkB,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,oBAAoB,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,MAAM,EAAE,CAAC;QAC1E,MAAM,CAAC,GAAG,MAAM,IAAI,CAClB,GAAG,EACH,YAAY,EACZ,YAAY,CAAC,kBAAkB,EAC/B,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EACzB,OAAO,CACR,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,OAAO,CAAC,GAAG,CACT,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,WAAW,CAAC,CAAC,WAAW,gBAAgB,CAAC,CAAC,SAAS,iBAAiB,CAAC,CAAC,UAAU,EAAE,CACpH,CAAC;QACF,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IAElB,oFAAoF;IACpF,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACrC,mDAAmD;IACnD,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,CAAC,UAAU,iBAAiB,CAAC,CAAC;IACtE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IAClB,MAAM,IAAI,GAAkB,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,kBAAkB,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACpF,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,OAAO,CAAC,GAAG,CACT,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,WAAW,CAAC,CAAC,WAAW,gBAAgB,CAAC,CAAC,SAAS,iBAAiB,CAAC,CAAC,UAAU,EAAE,CACpH,CAAC;QACF,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,GAAG,GAAG,CAAC,EAAiB,EAAE,CAA6B,EAAE,EAAE,CAC/D,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IAC/C,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAE/C,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,+BAA+B,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClG,OAAO,CAAC,GAAG,CACT,+BAA+B,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;QACxF,IAAI,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAC1G,CAAC;IACF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACzE,OAAO,CAAC,GAAG,CACT,YAAY,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,qFAAqF,CAAC,CAAC,CAAC,4FAA4F,EAAE,CAClN,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
@@ -1,25 +0,0 @@
1
- /**
2
- * Fast-apply vs current-edit benchmark — real-API measurement.
3
- *
4
- * Two strategies for the same edit task, measured end-to-end against live models:
5
- *
6
- * BASELINE (what ezcoder does today): the frontier model emits full
7
- * search/replace edit blocks (old_text + new_text, verbatim context). Our
8
- * edit tool then applies them LOCALLY and instantly (no apply model).
9
- *
10
- * FAST-APPLY (the proposed feature): the frontier model emits a terse "lazy"
11
- * edit using `// ... existing code ...` markers (far fewer output tokens), and
12
- * a cheap/fast "apply" model merges that lazy edit into the full file —
13
- * regenerating the whole file as output.
14
- *
15
- * We measure, per task: frontier output tokens + time, apply-model output tokens
16
- * + time, total wall-clock, and an anchor-based correctness check.
17
- *
18
- * Usage:
19
- * npx tsx src/core/fast-apply-benchmark.ts
20
- *
21
- * Env overrides:
22
- * EZ_FA_FRONTIER_PROVIDER / EZ_FA_FRONTIER_MODEL (default openai / gpt-6.1-sol)
23
- * EZ_FA_APPLY_PROVIDER / EZ_FA_APPLY_MODEL (default gemini / gemini-3.1-flash-lite)
24
- */
25
- export {};