@prestyj/cli 5.30.0 ā 5.30.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -39
- package/assets/skills/bulletproof/SKILL.md +1 -1
- package/assets/skills/clarify/SKILL.md +1 -1
- package/assets/skills/code-review/SKILL.md +1 -1
- package/assets/skills/compliance-guard/SKILL.md +1 -1
- package/assets/skills/durable/SKILL.md +1 -1
- package/assets/skills/evidence-led-ui/SKILL.md +1 -1
- package/assets/skills/lean/SKILL.md +1 -1
- package/assets/skills/refactoring/SKILL.md +1 -1
- package/assets/skills/root-cause/SKILL.md +1 -1
- package/assets/skills/shared-language/SKILL.md +1 -1
- package/assets/skills/tdd/SKILL.md +1 -1
- package/dist/app-sidecar/app-settings.d.ts +63 -0
- package/dist/app-sidecar/app-settings.js +147 -0
- package/dist/app-sidecar/app-settings.js.map +1 -0
- package/dist/app-sidecar/attachments.d.ts +11 -0
- package/dist/app-sidecar/attachments.js +47 -0
- package/dist/app-sidecar/attachments.js.map +1 -0
- package/dist/app-sidecar/auth-routes.d.ts +3 -0
- package/dist/app-sidecar/auth-routes.js +293 -0
- package/dist/app-sidecar/auth-routes.js.map +1 -0
- package/dist/app-sidecar/autopilot-runner.d.ts +62 -0
- package/dist/app-sidecar/autopilot-runner.js +373 -0
- package/dist/app-sidecar/autopilot-runner.js.map +1 -0
- package/dist/app-sidecar/file-search.d.ts +15 -0
- package/dist/app-sidecar/file-search.js +103 -0
- package/dist/app-sidecar/file-search.js.map +1 -0
- package/dist/app-sidecar/hf-pull-manager.d.ts +20 -0
- package/dist/app-sidecar/hf-pull-manager.js +224 -0
- package/dist/app-sidecar/hf-pull-manager.js.map +1 -0
- package/dist/app-sidecar/history-routes.d.ts +3 -0
- package/dist/app-sidecar/history-routes.js +375 -0
- package/dist/app-sidecar/history-routes.js.map +1 -0
- package/dist/app-sidecar/history.d.ts +67 -0
- package/dist/app-sidecar/history.js +23 -0
- package/dist/app-sidecar/history.js.map +1 -0
- package/dist/app-sidecar/http.d.ts +5 -0
- package/dist/app-sidecar/http.js +27 -0
- package/dist/app-sidecar/http.js.map +1 -0
- package/dist/app-sidecar/integration-routes.d.ts +3 -0
- package/dist/app-sidecar/integration-routes.js +286 -0
- package/dist/app-sidecar/integration-routes.js.map +1 -0
- package/dist/app-sidecar/json-mode.d.ts +10 -0
- package/dist/app-sidecar/json-mode.js +73 -0
- package/dist/app-sidecar/json-mode.js.map +1 -0
- package/dist/app-sidecar/keep-awake.d.ts +4 -0
- package/dist/app-sidecar/keep-awake.js +5 -0
- package/dist/app-sidecar/keep-awake.js.map +1 -0
- package/dist/app-sidecar/mcp-routes.d.ts +3 -0
- package/dist/app-sidecar/mcp-routes.js +185 -0
- package/dist/app-sidecar/mcp-routes.js.map +1 -0
- package/dist/app-sidecar/mcp-rows.d.ts +31 -0
- package/dist/app-sidecar/mcp-rows.js +77 -0
- package/dist/app-sidecar/mcp-rows.js.map +1 -0
- package/dist/app-sidecar/nolan-context.d.ts +20 -0
- package/dist/app-sidecar/nolan-context.js +55 -0
- package/dist/app-sidecar/nolan-context.js.map +1 -0
- package/dist/app-sidecar/progress-manager.d.ts +10 -0
- package/dist/app-sidecar/progress-manager.js +116 -0
- package/dist/app-sidecar/progress-manager.js.map +1 -0
- package/dist/app-sidecar/prompt-routes.d.ts +3 -0
- package/dist/app-sidecar/prompt-routes.js +355 -0
- package/dist/app-sidecar/prompt-routes.js.map +1 -0
- package/dist/app-sidecar/repo-polls.d.ts +18 -0
- package/dist/app-sidecar/repo-polls.js +45 -0
- package/dist/app-sidecar/repo-polls.js.map +1 -0
- package/dist/app-sidecar/route-context.d.ts +147 -0
- package/dist/app-sidecar/route-context.js +2 -0
- package/dist/app-sidecar/route-context.js.map +1 -0
- package/dist/app-sidecar/run-control-routes.d.ts +3 -0
- package/dist/app-sidecar/run-control-routes.js +437 -0
- package/dist/app-sidecar/run-control-routes.js.map +1 -0
- package/dist/app-sidecar/session-routes.d.ts +3 -0
- package/dist/app-sidecar/session-routes.js +115 -0
- package/dist/app-sidecar/session-routes.js.map +1 -0
- package/dist/app-sidecar/session-types.d.ts +39 -0
- package/dist/app-sidecar/session-types.js +5 -0
- package/dist/app-sidecar/session-types.js.map +1 -0
- package/dist/app-sidecar/task-routes.d.ts +3 -0
- package/dist/app-sidecar/task-routes.js +232 -0
- package/dist/app-sidecar/task-routes.js.map +1 -0
- package/dist/app-sidecar/usage.d.ts +19 -0
- package/dist/app-sidecar/usage.js +153 -0
- package/dist/app-sidecar/usage.js.map +1 -0
- package/dist/app-sidecar/workspace-routes.d.ts +3 -0
- package/dist/app-sidecar/workspace-routes.js +374 -0
- package/dist/app-sidecar/workspace-routes.js.map +1 -0
- package/dist/app-sidecar.js +541 -3971
- package/dist/app-sidecar.js.map +1 -1
- package/dist/chat-agents/jiwa.js +2 -0
- package/dist/chat-agents/jiwa.js.map +1 -1
- package/dist/chat-agents/memory.js +2 -0
- package/dist/chat-agents/memory.js.map +1 -1
- package/dist/cli/auth.js +13 -9
- package/dist/cli/auth.js.map +1 -1
- package/dist/cli.js +10 -11
- package/dist/cli.js.map +1 -1
- package/dist/config.js +1 -1
- package/dist/core/acceptance-checks.js +3 -9
- package/dist/core/acceptance-checks.js.map +1 -1
- package/dist/core/agent-session/helpers.d.ts +26 -0
- package/dist/core/agent-session/helpers.js +47 -0
- package/dist/core/agent-session/helpers.js.map +1 -0
- package/dist/core/agent-session/mcp-tools.d.ts +68 -0
- package/dist/core/agent-session/mcp-tools.js +195 -0
- package/dist/core/agent-session/mcp-tools.js.map +1 -0
- package/dist/core/agent-session/run-loop-monitor.d.ts +63 -0
- package/dist/core/agent-session/run-loop-monitor.js +164 -0
- package/dist/core/agent-session/run-loop-monitor.js.map +1 -0
- package/dist/core/agent-session/transcript-markers.d.ts +57 -0
- package/dist/core/agent-session/transcript-markers.js +166 -0
- package/dist/core/agent-session/transcript-markers.js.map +1 -0
- package/dist/core/agent-session/verification-tracker.d.ts +50 -0
- package/dist/core/agent-session/verification-tracker.js +269 -0
- package/dist/core/agent-session/verification-tracker.js.map +1 -0
- package/dist/core/agent-session.d.ts +27 -87
- package/dist/core/agent-session.js +166 -713
- package/dist/core/agent-session.js.map +1 -1
- package/dist/core/auth-providers.js +3 -3
- package/dist/core/auth-providers.js.map +1 -1
- package/dist/core/autopilot-cycle.d.ts +8 -2
- package/dist/core/autopilot-cycle.js +22 -3
- package/dist/core/autopilot-cycle.js.map +1 -1
- package/dist/core/bundled-agents.js +6 -6
- package/dist/core/bundled-agents.js.map +1 -1
- package/dist/core/checklist-git.d.ts +19 -0
- package/dist/core/checklist-git.js +54 -0
- package/dist/core/checklist-git.js.map +1 -0
- package/dist/core/checklist-items.d.ts +10 -1
- package/dist/core/checklist-items.js +22 -0
- package/dist/core/checklist-items.js.map +1 -1
- package/dist/core/checklist-prompt.d.ts +14 -3
- package/dist/core/checklist-prompt.js +38 -8
- package/dist/core/checklist-prompt.js.map +1 -1
- package/dist/core/checklist-snapshot.d.ts +5 -1
- package/dist/core/checklist-snapshot.js +20 -3
- package/dist/core/checklist-snapshot.js.map +1 -1
- package/dist/core/checklist-store.d.ts +5 -1
- package/dist/core/checklist-store.js +35 -4
- package/dist/core/checklist-store.js.map +1 -1
- package/dist/core/context-limits.js +1 -1
- package/dist/core/context-limits.js.map +1 -1
- package/dist/core/event-bus.d.ts +1 -1
- package/dist/core/event-bus.js +39 -13
- package/dist/core/event-bus.js.map +1 -1
- package/dist/core/extensions/plugin-bundles.js +4 -2
- package/dist/core/extensions/plugin-bundles.js.map +1 -1
- package/dist/core/hashline.d.ts +10 -8
- package/dist/core/hashline.js.map +1 -1
- package/dist/core/mcp/client.js +2 -0
- package/dist/core/mcp/client.js.map +1 -1
- package/dist/core/mcp/loopback.d.ts +2 -2
- package/dist/core/mcp/loopback.js +2 -2
- package/dist/core/mcp/shared-pool.js +4 -3
- package/dist/core/mcp/shared-pool.js.map +1 -1
- package/dist/core/persistent-shell.js +2 -1
- package/dist/core/persistent-shell.js.map +1 -1
- package/dist/core/plan-handoff.d.ts +30 -0
- package/dist/core/plan-handoff.js +10 -0
- package/dist/core/plan-handoff.js.map +1 -0
- package/dist/core/plan-submission-gate.d.ts +28 -0
- package/dist/core/plan-submission-gate.js +27 -0
- package/dist/core/plan-submission-gate.js.map +1 -0
- package/dist/core/process-manager.js +9 -1
- package/dist/core/process-manager.js.map +1 -1
- package/dist/core/project-health-scan.d.ts +109 -0
- package/dist/core/project-health-scan.js +673 -0
- package/dist/core/project-health-scan.js.map +1 -0
- package/dist/core/project-health-score.d.ts +41 -0
- package/dist/core/project-health-score.js +257 -0
- package/dist/core/project-health-score.js.map +1 -0
- package/dist/core/prompt-commands.js +44 -6
- package/dist/core/prompt-commands.js.map +1 -1
- package/dist/core/session-manager.js.map +1 -1
- package/dist/core/session-storage.d.ts +1 -0
- package/dist/core/session-storage.js +1 -1
- package/dist/core/session-storage.js.map +1 -1
- package/dist/core/skills.js +6 -9
- package/dist/core/skills.js.map +1 -1
- package/dist/core/style-packs/index.js +2 -11
- package/dist/core/style-packs/index.js.map +1 -1
- package/dist/core/style-packs/packs.d.ts +3 -12
- package/dist/core/style-packs/packs.js +132 -173
- package/dist/core/style-packs/packs.js.map +1 -1
- package/dist/core/subagent-manager.js +11 -6
- package/dist/core/subagent-manager.js.map +1 -1
- package/dist/core/subagent-receipt.js +3 -4
- package/dist/core/subagent-receipt.js.map +1 -1
- package/dist/core/ui-adoption.js +2 -0
- package/dist/core/ui-adoption.js.map +1 -1
- package/dist/core/ui-registry.js +5 -4
- package/dist/core/ui-registry.js.map +1 -1
- package/dist/core/verification-evidence.d.ts +21 -2
- package/dist/core/verification-evidence.js +283 -8
- package/dist/core/verification-evidence.js.map +1 -1
- package/dist/core/verification-gate.js +4 -2
- package/dist/core/verification-gate.js.map +1 -1
- package/dist/core/verify-commands.js +2 -3
- package/dist/core/verify-commands.js.map +1 -1
- package/dist/modes/agent-home-mode.js +1 -4
- package/dist/modes/agent-home-mode.js.map +1 -1
- package/dist/modes/subagent-worker-mode.js +13 -8
- package/dist/modes/subagent-worker-mode.js.map +1 -1
- package/dist/system-prompt.js +70 -87
- package/dist/system-prompt.js.map +1 -1
- package/dist/tools/ask-user.js +13 -74
- package/dist/tools/ask-user.js.map +1 -1
- package/dist/tools/bash.d.ts +3 -1
- package/dist/tools/bash.js +28 -84
- package/dist/tools/bash.js.map +1 -1
- package/dist/tools/checklist.d.ts +7 -14
- package/dist/tools/checklist.js +22 -41
- package/dist/tools/checklist.js.map +1 -1
- package/dist/tools/code-nav.js +7 -29
- package/dist/tools/code-nav.js.map +1 -1
- package/dist/tools/edit.d.ts +4 -9
- package/dist/tools/edit.js +175 -65
- package/dist/tools/edit.js.map +1 -1
- package/dist/tools/enter-plan.js +2 -7
- package/dist/tools/enter-plan.js.map +1 -1
- package/dist/tools/exit-plan.js +2 -3
- package/dist/tools/exit-plan.js.map +1 -1
- package/dist/tools/find.js +3 -4
- package/dist/tools/find.js.map +1 -1
- package/dist/tools/grep.js +9 -20
- package/dist/tools/grep.js.map +1 -1
- package/dist/tools/index.js +10 -7
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/ls.js +3 -3
- package/dist/tools/ls.js.map +1 -1
- package/dist/tools/prompt-hints.js +34 -47
- package/dist/tools/prompt-hints.js.map +1 -1
- package/dist/tools/read-only-bash.js +3 -0
- package/dist/tools/read-only-bash.js.map +1 -1
- package/dist/tools/read.js +8 -20
- package/dist/tools/read.js.map +1 -1
- package/dist/tools/search-code.js +4 -12
- package/dist/tools/search-code.js.map +1 -1
- package/dist/tools/skill.js +13 -7
- package/dist/tools/skill.js.map +1 -1
- package/dist/tools/subagent-control.js +7 -22
- package/dist/tools/subagent-control.js.map +1 -1
- package/dist/tools/subagent.js +1 -1
- package/dist/tools/subagent.js.map +1 -1
- package/dist/tools/task-output.js +4 -15
- package/dist/tools/task-output.js.map +1 -1
- package/dist/tools/task-send.js +5 -17
- package/dist/tools/task-send.js.map +1 -1
- package/dist/tools/task-stop.js +2 -2
- package/dist/tools/task-stop.js.map +1 -1
- package/dist/tools/tool-search.js +2 -7
- package/dist/tools/tool-search.js.map +1 -1
- package/dist/tools/tool-tiers.d.ts +21 -4
- package/dist/tools/tool-tiers.js +44 -17
- package/dist/tools/tool-tiers.js.map +1 -1
- package/dist/tools/web-fetch.js +7 -33
- package/dist/tools/web-fetch.js.map +1 -1
- package/dist/tools/write.js +3 -4
- package/dist/tools/write.js.map +1 -1
- package/dist/ui/App.js +20 -26
- package/dist/ui/App.js.map +1 -1
- package/dist/ui/components/Footer.js +1 -2
- package/dist/ui/components/Footer.js.map +1 -1
- package/dist/ui/components/InputArea.js +0 -2
- package/dist/ui/components/InputArea.js.map +1 -1
- package/dist/ui/components/ToolExecution.js +4 -3
- package/dist/ui/components/ToolExecution.js.map +1 -1
- package/dist/ui/hooks/useAgentLoop.js +7 -6
- package/dist/ui/hooks/useAgentLoop.js.map +1 -1
- package/dist/ui/sessions.js +1 -5
- package/dist/ui/sessions.js.map +1 -1
- package/dist/ui/utils/highlight.d.ts +0 -5
- package/dist/ui/utils/highlight.js +3 -49
- package/dist/ui/utils/highlight.js.map +1 -1
- package/dist/ui/utils/markdown-renderer.d.ts +6 -0
- package/dist/ui/utils/markdown-renderer.js +21 -5
- package/dist/ui/utils/markdown-renderer.js.map +1 -1
- package/dist/utils/process.js +3 -3
- package/dist/utils/process.js.map +1 -1
- package/dist/utils/shared-poll.d.ts +37 -0
- package/dist/utils/shared-poll.js +104 -0
- package/dist/utils/shared-poll.js.map +1 -0
- package/package.json +8 -8
- package/dist/core/api-benchmark.d.ts +0 -63
- package/dist/core/api-benchmark.js +0 -381
- package/dist/core/api-benchmark.js.map +0 -1
- package/dist/core/bash-spawn-benchmark.d.ts +0 -1
- package/dist/core/bash-spawn-benchmark.js +0 -139
- package/dist/core/bash-spawn-benchmark.js.map +0 -1
- package/dist/core/cache-warm-benchmark.d.ts +0 -21
- package/dist/core/cache-warm-benchmark.js +0 -170
- package/dist/core/cache-warm-benchmark.js.map +0 -1
- package/dist/core/fast-apply-benchmark.d.ts +0 -25
- package/dist/core/fast-apply-benchmark.js +0 -284
- package/dist/core/fast-apply-benchmark.js.map +0 -1
- package/dist/core/hashline-edit-benchmark.d.ts +0 -88
- package/dist/core/hashline-edit-benchmark.js +0 -518
- package/dist/core/hashline-edit-benchmark.js.map +0 -1
- package/dist/core/semantic-search-benchmark.d.ts +0 -36
- package/dist/core/semantic-search-benchmark.js +0 -211
- package/dist/core/semantic-search-benchmark.js.map +0 -1
- package/dist/core/speed-benchmark.d.ts +0 -132
- package/dist/core/speed-benchmark.js +0 -410
- package/dist/core/speed-benchmark.js.map +0 -1
- package/dist/ui/activity-phrases.d.ts +0 -10
- package/dist/ui/activity-phrases.js +0 -184
- package/dist/ui/activity-phrases.js.map +0 -1
- package/dist/ui/components/PlanApproval.d.ts +0 -7
- package/dist/ui/components/PlanApproval.js +0 -60
- package/dist/ui/components/PlanApproval.js.map +0 -1
- package/dist/ui/components/PlanBanner.d.ts +0 -5
- package/dist/ui/components/PlanBanner.js +0 -31
- package/dist/ui/components/PlanBanner.js.map +0 -1
- package/dist/ui/components/PlanProgress.d.ts +0 -6
- package/dist/ui/components/PlanProgress.js +0 -30
- package/dist/ui/components/PlanProgress.js.map +0 -1
- package/dist/ui/hooks/useMinDisplayTime.d.ts +0 -10
- package/dist/ui/hooks/useMinDisplayTime.js +0 -34
- package/dist/ui/hooks/useMinDisplayTime.js.map +0 -1
- package/dist/ui/hooks/useSessionManager.d.ts +0 -12
- package/dist/ui/hooks/useSessionManager.js +0 -43
- package/dist/ui/hooks/useSessionManager.js.map +0 -1
- package/dist/ui/hooks/useSlashCommands.d.ts +0 -6
- package/dist/ui/hooks/useSlashCommands.js +0 -11
- package/dist/ui/hooks/useSlashCommands.js.map +0 -1
- package/dist/ui/terminal-history-spacing.d.ts +0 -1
- package/dist/ui/terminal-history-spacing.js +0 -2
- package/dist/ui/terminal-history-spacing.js.map +0 -1
- package/dist/ui/utils/markdown-cache.d.ts +0 -10
- package/dist/ui/utils/markdown-cache.js +0 -58
- package/dist/ui/utils/markdown-cache.js.map +0 -1
- package/dist/ui/utils/token-to-ansi.d.ts +0 -8
- package/dist/ui/utils/token-to-ansi.js +0 -323
- package/dist/ui/utils/token-to-ansi.js.map +0 -1
|
@@ -1,139 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Per-call spawn (what tools/bash.ts does today) vs a persistent shell session
|
|
3
|
-
* (what oh-my-pi's embedded "brush" bash gives them) ā local measurement, no
|
|
4
|
-
* LLM calls needed.
|
|
5
|
-
*
|
|
6
|
-
* Strategy A ā SPAWN (current): every command is `spawn("bash", ["-c", cmd])`,
|
|
7
|
-
* a fresh process with fresh env/cwd. Cost = fork/exec + shell startup per call.
|
|
8
|
-
*
|
|
9
|
-
* Strategy B ā PERSISTENT: one long-lived `bash` process; commands are written
|
|
10
|
-
* to stdin and delimited with a sentinel echo. Cost ā pipe round-trip.
|
|
11
|
-
* Bonus property (qualitative): cd / env / shell state survive across calls.
|
|
12
|
-
*
|
|
13
|
-
* We run the same command list through both and report total + per-call wall
|
|
14
|
-
* time. Commands are trivial on purpose ā we are measuring HARNESS overhead,
|
|
15
|
-
* not the commands themselves.
|
|
16
|
-
*
|
|
17
|
-
* Usage:
|
|
18
|
-
* npx tsx src/core/bash-spawn-benchmark.ts
|
|
19
|
-
*
|
|
20
|
-
* Env overrides:
|
|
21
|
-
* EZ_BASH_BENCH_N (commands per strategy, default 30)
|
|
22
|
-
*/
|
|
23
|
-
import { spawn } from "node:child_process";
|
|
24
|
-
import { randomUUID } from "node:crypto";
|
|
25
|
-
const COMMANDS = ["echo hello", "pwd", "ls / > /dev/null", "printf '%s' done", "true"];
|
|
26
|
-
function buildCommandList(n) {
|
|
27
|
-
const out = [];
|
|
28
|
-
for (let i = 0; i < n; i++)
|
|
29
|
-
out.push(COMMANDS[i % COMMANDS.length]);
|
|
30
|
-
return out;
|
|
31
|
-
}
|
|
32
|
-
// āā Strategy A: fresh spawn per call (mirrors tools/bash.ts) āā
|
|
33
|
-
function runSpawned(cmd) {
|
|
34
|
-
return new Promise((resolve, reject) => {
|
|
35
|
-
const start = performance.now();
|
|
36
|
-
const child = spawn("bash", ["-c", cmd], { stdio: ["ignore", "pipe", "pipe"] });
|
|
37
|
-
child.stdout.on("data", () => { });
|
|
38
|
-
child.stderr.on("data", () => { });
|
|
39
|
-
child.on("error", reject);
|
|
40
|
-
child.on("close", () => resolve(performance.now() - start));
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
// āā Strategy B: one persistent bash, sentinel-delimited āā
|
|
44
|
-
class PersistentShell {
|
|
45
|
-
child = spawn("bash", ["--norc", "--noprofile"], {
|
|
46
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
47
|
-
});
|
|
48
|
-
buffer = "";
|
|
49
|
-
waiter = null;
|
|
50
|
-
constructor() {
|
|
51
|
-
this.child.stdout.on("data", (d) => {
|
|
52
|
-
this.buffer += d.toString();
|
|
53
|
-
if (this.waiter && this.buffer.includes(this.waiter.sentinel)) {
|
|
54
|
-
const w = this.waiter;
|
|
55
|
-
this.waiter = null;
|
|
56
|
-
this.buffer = "";
|
|
57
|
-
w.resolve();
|
|
58
|
-
}
|
|
59
|
-
});
|
|
60
|
-
this.child.stderr.on("data", () => { });
|
|
61
|
-
}
|
|
62
|
-
run(cmd) {
|
|
63
|
-
const start = performance.now();
|
|
64
|
-
const sentinel = `__EZ_DONE_${randomUUID()}__`;
|
|
65
|
-
return new Promise((resolve) => {
|
|
66
|
-
this.waiter = { sentinel, resolve: () => resolve(performance.now() - start) };
|
|
67
|
-
this.child.stdin.write(`${cmd}\necho ${sentinel}\n`);
|
|
68
|
-
});
|
|
69
|
-
}
|
|
70
|
-
/** Prove state persistence: cd + env var survive across run() calls. */
|
|
71
|
-
async statePersists() {
|
|
72
|
-
await this.run("cd /tmp && export EZ_BENCH_STATE=alive");
|
|
73
|
-
const sentinel = `__EZ_STATE_${randomUUID()}__`;
|
|
74
|
-
const out = await new Promise((resolve) => {
|
|
75
|
-
let acc = "";
|
|
76
|
-
const onData = (d) => {
|
|
77
|
-
acc += d.toString();
|
|
78
|
-
if (acc.includes(sentinel)) {
|
|
79
|
-
this.child.stdout.off("data", onData);
|
|
80
|
-
resolve(acc);
|
|
81
|
-
}
|
|
82
|
-
};
|
|
83
|
-
this.child.stdout.on("data", onData);
|
|
84
|
-
this.child.stdin.write(`pwd; echo "$EZ_BENCH_STATE"; echo ${sentinel}\n`);
|
|
85
|
-
});
|
|
86
|
-
return out.includes("/tmp") && out.includes("alive");
|
|
87
|
-
}
|
|
88
|
-
close() {
|
|
89
|
-
this.child.stdin.end();
|
|
90
|
-
this.child.kill();
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
// āā Stats āā
|
|
94
|
-
function stats(samples) {
|
|
95
|
-
const sorted = [...samples].sort((a, b) => a - b);
|
|
96
|
-
const total = samples.reduce((s, x) => s + x, 0);
|
|
97
|
-
const at = (q) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))];
|
|
98
|
-
return { total, mean: total / samples.length, p50: at(0.5), p95: at(0.95) };
|
|
99
|
-
}
|
|
100
|
-
function fmt(s) {
|
|
101
|
-
return (`total ${s.total.toFixed(0)}ms | mean ${s.mean.toFixed(1)}ms | ` +
|
|
102
|
-
`p50 ${s.p50.toFixed(1)}ms | p95 ${s.p95.toFixed(1)}ms`);
|
|
103
|
-
}
|
|
104
|
-
async function main() {
|
|
105
|
-
const n = Math.max(5, parseInt(process.env.EZ_BASH_BENCH_N ?? "30", 10));
|
|
106
|
-
const cmds = buildCommandList(n);
|
|
107
|
-
console.log(`\nš Bash harness overhead ā ${n} commands per strategy\n`);
|
|
108
|
-
// Warm both paths once so first-touch costs don't skew either side.
|
|
109
|
-
await runSpawned("true");
|
|
110
|
-
const shell = new PersistentShell();
|
|
111
|
-
await shell.run("true");
|
|
112
|
-
const spawnTimes = [];
|
|
113
|
-
for (const c of cmds)
|
|
114
|
-
spawnTimes.push(await runSpawned(c));
|
|
115
|
-
const persistTimes = [];
|
|
116
|
-
for (const c of cmds)
|
|
117
|
-
persistTimes.push(await shell.run(c));
|
|
118
|
-
const persists = await shell.statePersists();
|
|
119
|
-
shell.close();
|
|
120
|
-
const a = stats(spawnTimes);
|
|
121
|
-
const b = stats(persistTimes);
|
|
122
|
-
console.log(`SPAWN-PER-CALL (current): ${fmt(a)}`);
|
|
123
|
-
console.log(`PERSISTENT SHELL (theirs): ${fmt(b)}`);
|
|
124
|
-
const saved = a.mean - b.mean;
|
|
125
|
-
console.log(`\nPer-call overhead saved: ${saved.toFixed(1)}ms ` +
|
|
126
|
-
`(${((saved / a.mean) * 100).toFixed(0)}% of mean call cost)`);
|
|
127
|
-
console.log(`State persistence across calls (cd/env): ${persists ? "YES" : "NO"}`);
|
|
128
|
-
console.log(`\nVerdict: persistent shell wins if per-call saving is meaningful at agent scale ` +
|
|
129
|
-
`(a long session runs hundreds of bash calls) AND state persistence is a UX win.\n`);
|
|
130
|
-
}
|
|
131
|
-
const isDirectRun = process.argv[1]?.endsWith("bash-spawn-benchmark.ts") ||
|
|
132
|
-
process.argv[1]?.endsWith("bash-spawn-benchmark.js");
|
|
133
|
-
if (isDirectRun) {
|
|
134
|
-
main().catch((err) => {
|
|
135
|
-
console.error("Benchmark failed:", err);
|
|
136
|
-
process.exit(1);
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
|
-
//# sourceMappingURL=bash-spawn-benchmark.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"bash-spawn-benchmark.js","sourceRoot":"","sources":["../../src/core/bash-spawn-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,QAAQ,GAAG,CAAC,YAAY,EAAE,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,CAAC,CAAC;AAEvF,SAAS,gBAAgB,CAAC,CAAS;IACjC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAE,CAAC,CAAC;IACrE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,iEAAiE;AAEjE,SAAS,UAAU,CAAC,GAAW;IAC7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAChF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAClC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAClC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,4DAA4D;AAE5D,MAAM,eAAe;IACX,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE;QACvD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;KAChC,CAAC,CAAC;IACK,MAAM,GAAG,EAAE,CAAC;IACZ,MAAM,GAAqD,IAAI,CAAC;IAExE;QACE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE;YACzC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;gBACtB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;gBACnB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;gBACjB,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,GAAG,CAAC,GAAW;QACb,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,aAAa,UAAU,EAAE,IAAI,CAAC;QAC/C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,IAAI,CAAC,MAAM,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC;YAC9E,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,QAAQ,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,aAAa;QACjB,MAAM,IAAI,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,cAAc,UAAU,EAAE,IAAI,CAAC;QAChD,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,EAAE;YAChD,IAAI,GAAG,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,CAAC,CAAS,EAAQ,EAAE;gBACjC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACpB,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;oBACtC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC;YACH,CAAC,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACrC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,qCAAqC,QAAQ,IAAI,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QACH,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACvD,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;CACF;AAED,cAAc;AAEd,SAAS,KAAK,CAAC,OAAiB;IAC9B,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,MAAM,EAAE,GAAG,CAAC,CAAS,EAAU,EAAE,CAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;IACtE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;AAC9E,CAAC;AAED,SAAS,GAAG,CAAC,CAA4D;IACvE,OAAO,CACL,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO;QAChE,OAAO,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACxD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAEjC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,0BAA0B,CAAC,CAAC;IAEzE,oEAAoE;IACpE,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;IACzB,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAExB,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,UAAU,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAE3D,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,YAAY,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;IAC7C,KAAK,CAAC,KAAK,EAAE,CAAC;IAEd,MAAM,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IAC5B,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;IAE9B,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;IAC9B,OAAO,CAAC,GAAG,CACT,8BAA8B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;QACjD,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAChE,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,4CAA4C,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACnF,OAAO,CAAC,GAAG,CACT,mFAAmF;QACjF,mFAAmF,CACtF,CAAC;AACJ,CAAC;AAED,MAAM,WAAW,GACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,yBAAyB,CAAC;IACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,yBAAyB,CAAC,CAAC;AAEvD,IAAI,WAAW,EAAE,CAAC;IAChB,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACnB,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Prompt-cache warming micro-benchmark ā isolates the cache effect.
|
|
3
|
-
*
|
|
4
|
-
* The existing api-benchmark grows the conversation each turn and uses a tiny
|
|
5
|
-
* (~250 token) system prompt, which is BELOW Anthropic's minimum cacheable
|
|
6
|
-
* prefix (1024 tok Sonnet/Opus, 2048 Haiku) ā so caching never engages and the
|
|
7
|
-
* A/B is dominated by turn-to-turn throughput variance.
|
|
8
|
-
*
|
|
9
|
-
* This test uses a large, FIXED prefix (well above the min) and a tiny variable
|
|
10
|
-
* suffix, then measures, per arm:
|
|
11
|
-
* COLD ā fresh cache key, first hit (cache WRITE, no read).
|
|
12
|
-
* WARM ā prewarm the prefix, then measure the next call's cache_read + TTFT.
|
|
13
|
-
*
|
|
14
|
-
* It reports cache_read tokens and TTFT directly, so the cache effect is visible
|
|
15
|
-
* regardless of throughput noise.
|
|
16
|
-
*
|
|
17
|
-
* Usage: npx tsx src/core/cache-warm-benchmark.ts
|
|
18
|
-
* Env: EZ_CW_PROVIDER (default anthropic), EZ_CW_MODEL (default claude-haiku-4-5-20251001),
|
|
19
|
-
* EZ_CW_REPS (default 4)
|
|
20
|
-
*/
|
|
21
|
-
export {};
|
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Prompt-cache warming micro-benchmark ā isolates the cache effect.
|
|
3
|
-
*
|
|
4
|
-
* The existing api-benchmark grows the conversation each turn and uses a tiny
|
|
5
|
-
* (~250 token) system prompt, which is BELOW Anthropic's minimum cacheable
|
|
6
|
-
* prefix (1024 tok Sonnet/Opus, 2048 Haiku) ā so caching never engages and the
|
|
7
|
-
* A/B is dominated by turn-to-turn throughput variance.
|
|
8
|
-
*
|
|
9
|
-
* This test uses a large, FIXED prefix (well above the min) and a tiny variable
|
|
10
|
-
* suffix, then measures, per arm:
|
|
11
|
-
* COLD ā fresh cache key, first hit (cache WRITE, no read).
|
|
12
|
-
* WARM ā prewarm the prefix, then measure the next call's cache_read + TTFT.
|
|
13
|
-
*
|
|
14
|
-
* It reports cache_read tokens and TTFT directly, so the cache effect is visible
|
|
15
|
-
* regardless of throughput noise.
|
|
16
|
-
*
|
|
17
|
-
* Usage: npx tsx src/core/cache-warm-benchmark.ts
|
|
18
|
-
* Env: EZ_CW_PROVIDER (default anthropic), EZ_CW_MODEL (default claude-haiku-4-5-20251001),
|
|
19
|
-
* EZ_CW_REPS (default 4)
|
|
20
|
-
*/
|
|
21
|
-
import { stream } from "@prestyj/ai";
|
|
22
|
-
import { AuthStorage } from "./auth-storage.js";
|
|
23
|
-
/** Build a large, realistic coding-agent system prompt (~well above 2048 tok). */
|
|
24
|
-
function bigPrefix() {
|
|
25
|
-
const toolDoc = (name, desc) => `### ${name}\n${desc}\nParameters are validated with a Zod schema and converted to JSON Schema at the provider boundary. ` +
|
|
26
|
-
`Errors are returned as structured tool results so the agent can self-correct in the same turn. `;
|
|
27
|
-
const tools = [
|
|
28
|
-
toolDoc("read", "Read a file's contents with cat -n style numbered lines, truncated to 2000 lines or 50KB."),
|
|
29
|
-
toolDoc("write", "Write content to a file, creating parent directories. Existing files must be read first."),
|
|
30
|
-
toolDoc("edit", "Replace text via search/replace blocks applied sequentially with a fuzzy fallback ladder."),
|
|
31
|
-
toolDoc("bash", "Execute a shell command in a non-interactive bash shell with combined stdout/stderr."),
|
|
32
|
-
toolDoc("grep", "Search file contents using regex, returning filepath:line:content matches."),
|
|
33
|
-
toolDoc("find", "Find files matching a glob pattern, respecting .gitignore."),
|
|
34
|
-
toolDoc("ls", "List directory contents with file types and sizes."),
|
|
35
|
-
toolDoc("web_fetch", "Fetch and read web page content as Markdown, extracting main content."),
|
|
36
|
-
toolDoc("subagent", "Spawn an isolated sub-agent with its own context window for a focused task."),
|
|
37
|
-
toolDoc("task_output", "Read new output from a background process by id."),
|
|
38
|
-
];
|
|
39
|
-
// Repeat the tool block several times so the prefix clears Anthropic's
|
|
40
|
-
// minimum cacheable size (1024 tok Sonnet/Opus, 2048 Haiku) with headroom.
|
|
41
|
-
const bulkTools = [];
|
|
42
|
-
for (let r = 0; r < 22; r++)
|
|
43
|
-
bulkTools.push(...tools, `<!-- tool block repetition ${r} for prefix sizing -->`);
|
|
44
|
-
return [
|
|
45
|
-
"You are EZ Coder ā a coding agent that works directly in the user's codebase.",
|
|
46
|
-
"You explore, understand, change, and verify code end-to-end.",
|
|
47
|
-
"",
|
|
48
|
-
"## How to Talk",
|
|
49
|
-
"Lead with the outcome. One idea per line. Bottom line first. No preamble or recap.",
|
|
50
|
-
"Stay silent between tool calls unless you hit a decision, tradeoff, or question.",
|
|
51
|
-
"",
|
|
52
|
-
"## How to Work",
|
|
53
|
-
"Read before edit/write; re-read after formatters, codegen, or any disk mutator.",
|
|
54
|
-
"Match neighbouring code style. Keep edits small; plan multi-file work first.",
|
|
55
|
-
"Choose targeted verification appropriate to the change; read and fix failures.",
|
|
56
|
-
"Do not assume APIs, CLI flags, config schema, or error wording ā verify against source.",
|
|
57
|
-
"",
|
|
58
|
-
"## Tools",
|
|
59
|
-
...bulkTools,
|
|
60
|
-
"",
|
|
61
|
-
"## Code Quality",
|
|
62
|
-
"Intent-revealing names; reuse existing deps. Types first; handle I/O, input, and external API errors.",
|
|
63
|
-
"No dead or commented-out code, placeholders, or unasked refactors.",
|
|
64
|
-
"",
|
|
65
|
-
"## Environment",
|
|
66
|
-
"Working directory: /home/user/project. Platform: darwin.",
|
|
67
|
-
].join("\n");
|
|
68
|
-
}
|
|
69
|
-
async function call(cfg, system, user, promptCacheKey, cacheRetention) {
|
|
70
|
-
const messages = [
|
|
71
|
-
{ role: "system", content: system },
|
|
72
|
-
{ role: "user", content: user },
|
|
73
|
-
];
|
|
74
|
-
const start = Date.now();
|
|
75
|
-
let ttftMs = 0;
|
|
76
|
-
let first = true;
|
|
77
|
-
const result = stream({
|
|
78
|
-
provider: cfg.provider,
|
|
79
|
-
model: cfg.model,
|
|
80
|
-
messages,
|
|
81
|
-
maxTokens: 16,
|
|
82
|
-
apiKey: cfg.apiKey,
|
|
83
|
-
baseUrl: cfg.baseUrl,
|
|
84
|
-
accountId: cfg.accountId,
|
|
85
|
-
promptCacheKey,
|
|
86
|
-
cacheRetention,
|
|
87
|
-
});
|
|
88
|
-
for await (const event of result) {
|
|
89
|
-
if (first && (event.type === "text_delta" || event.type === "thinking_delta")) {
|
|
90
|
-
ttftMs = Date.now() - start;
|
|
91
|
-
first = false;
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
const response = await result.response;
|
|
95
|
-
return {
|
|
96
|
-
ttftMs: ttftMs || Date.now() - start,
|
|
97
|
-
wallMs: Date.now() - start,
|
|
98
|
-
inputTokens: response.usage.inputTokens,
|
|
99
|
-
cacheRead: response.usage.cacheRead ?? 0,
|
|
100
|
-
cacheWrite: response.usage.cacheWrite ?? 0,
|
|
101
|
-
outputTokens: response.usage.outputTokens,
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
function sleep(ms) {
|
|
105
|
-
return new Promise((r) => setTimeout(r, ms));
|
|
106
|
-
}
|
|
107
|
-
async function main() {
|
|
108
|
-
const provider = process.env.EZ_CW_PROVIDER ?? "anthropic";
|
|
109
|
-
const model = process.env.EZ_CW_MODEL ?? "claude-haiku-4-5-20251001";
|
|
110
|
-
const reps = parseInt(process.env.EZ_CW_REPS ?? "4", 10);
|
|
111
|
-
const auth = new AuthStorage();
|
|
112
|
-
await auth.load();
|
|
113
|
-
const c = await auth.resolveCredentials(provider);
|
|
114
|
-
const cfg = {
|
|
115
|
-
provider,
|
|
116
|
-
model,
|
|
117
|
-
apiKey: c.accessToken,
|
|
118
|
-
baseUrl: c.baseUrl,
|
|
119
|
-
accountId: c.accountId,
|
|
120
|
-
};
|
|
121
|
-
const system = bigPrefix();
|
|
122
|
-
// Rough token size of the prefix (chars/3.3) for context.
|
|
123
|
-
console.log(`\nš„ Cache-warm micro-benchmark: ${provider}/${model}`);
|
|
124
|
-
console.log(` Prefix ~${Math.round(system.length / 3.3)} tokens, ${reps} reps per arm\n`);
|
|
125
|
-
// āā COLD arm: every rep uses a UNIQUE cache key ā never reads cache āā
|
|
126
|
-
console.log("ā¶ COLD (no warming ā fresh cache key each call):");
|
|
127
|
-
// Each cold rep gets UNIQUE prefix content (Anthropic caches by content, not
|
|
128
|
-
// by key) so every call is a true cache miss ā otherwise rep 2+ would read
|
|
129
|
-
// rep 1's cache and pollute the cold baseline.
|
|
130
|
-
const cold = [];
|
|
131
|
-
for (let i = 0; i < reps; i++) {
|
|
132
|
-
const uniqueSystem = `<!-- cold-sample-${Date.now()}-${i} -->\n${system}`;
|
|
133
|
-
const m = await call(cfg, uniqueSystem, `Question ${i}: name one tool.`, `cold-${Date.now()}-${i}`, "short");
|
|
134
|
-
cold.push(m);
|
|
135
|
-
console.log(` rep ${i + 1}: TTFT ${m.ttftMs}ms | in ${m.inputTokens} | cacheRead ${m.cacheRead} | cacheWrite ${m.cacheWrite}`);
|
|
136
|
-
await sleep(1500);
|
|
137
|
-
}
|
|
138
|
-
await sleep(3000);
|
|
139
|
-
// āā WARM arm: prewarm a SHARED key, then measure subsequent calls (cache reads) āā
|
|
140
|
-
console.log("\nā¶ WARM (prewarm shared prefix, then measure cache reads):");
|
|
141
|
-
const warmKey = `warm-${Date.now()}`;
|
|
142
|
-
// Prewarm: a throwaway call that writes the cache.
|
|
143
|
-
const pw = await call(cfg, system, "warm.", warmKey, "long");
|
|
144
|
-
console.log(` prewarm: cacheWrite ${pw.cacheWrite} (cache primed)`);
|
|
145
|
-
await sleep(1500);
|
|
146
|
-
const warm = [];
|
|
147
|
-
for (let i = 0; i < reps; i++) {
|
|
148
|
-
const m = await call(cfg, system, `Question ${i}: name one tool.`, warmKey, "long");
|
|
149
|
-
warm.push(m);
|
|
150
|
-
console.log(` rep ${i + 1}: TTFT ${m.ttftMs}ms | in ${m.inputTokens} | cacheRead ${m.cacheRead} | cacheWrite ${m.cacheWrite}`);
|
|
151
|
-
await sleep(1500);
|
|
152
|
-
}
|
|
153
|
-
const avg = (xs, f) => xs.reduce((s, m) => s + f(m), 0) / xs.length;
|
|
154
|
-
const coldTtft = avg(cold, (m) => m.ttftMs);
|
|
155
|
-
const warmTtft = avg(warm, (m) => m.ttftMs);
|
|
156
|
-
const coldRead = avg(cold, (m) => m.cacheRead);
|
|
157
|
-
const warmRead = avg(warm, (m) => m.cacheRead);
|
|
158
|
-
console.log("\nāāāāāāāāāāāāāāāāāāāāāā RESULTS āāāāāāāāāāāāāāāāāāāāāā");
|
|
159
|
-
console.log(`Avg cacheRead tokens : cold ${coldRead.toFixed(0)} ā warm ${warmRead.toFixed(0)}`);
|
|
160
|
-
console.log(`Avg TTFT : cold ${coldTtft.toFixed(0)}ms ā warm ${warmTtft.toFixed(0)}ms ` +
|
|
161
|
-
`(${coldTtft > warmTtft ? "-" : "+"}${Math.abs(((coldTtft - warmTtft) / coldTtft) * 100).toFixed(0)}%)`);
|
|
162
|
-
const warmHitRate = warm.filter((m) => m.cacheRead > 0).length / warm.length;
|
|
163
|
-
console.log(`Warm cache hit rate : ${(warmHitRate * 100).toFixed(0)}%`);
|
|
164
|
-
console.log(`Verdict: ${warmRead > 0 ? "caching ENGAGES when prefix exceeds the min ā warming converts cold writes to reads" : "caching did NOT engage (prefix below provider minimum, or OAuth path strips cache_control)"}`);
|
|
165
|
-
}
|
|
166
|
-
main().catch((err) => {
|
|
167
|
-
console.error("Benchmark failed:", err);
|
|
168
|
-
process.exit(1);
|
|
169
|
-
});
|
|
170
|
-
//# sourceMappingURL=cache-warm-benchmark.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cache-warm-benchmark.js","sourceRoot":"","sources":["../../src/core/cache-warm-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,MAAM,EAA8C,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,kFAAkF;AAClF,SAAS,SAAS;IAChB,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE,CAC7C,OAAO,IAAI,KAAK,IAAI,sGAAsG;QAC1H,iGAAiG,CAAC;IACpG,MAAM,KAAK,GAAG;QACZ,OAAO,CACL,MAAM,EACN,2FAA2F,CAC5F;QACD,OAAO,CACL,OAAO,EACP,0FAA0F,CAC3F;QACD,OAAO,CACL,MAAM,EACN,2FAA2F,CAC5F;QACD,OAAO,CACL,MAAM,EACN,sFAAsF,CACvF;QACD,OAAO,CAAC,MAAM,EAAE,4EAA4E,CAAC;QAC7F,OAAO,CAAC,MAAM,EAAE,4DAA4D,CAAC;QAC7E,OAAO,CAAC,IAAI,EAAE,oDAAoD,CAAC;QACnE,OAAO,CAAC,WAAW,EAAE,uEAAuE,CAAC;QAC7F,OAAO,CACL,UAAU,EACV,6EAA6E,CAC9E;QACD,OAAO,CAAC,aAAa,EAAE,kDAAkD,CAAC;KAC3E,CAAC;IACF,uEAAuE;IACvE,2EAA2E;IAC3E,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE;QACzB,SAAS,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,8BAA8B,CAAC,wBAAwB,CAAC,CAAC;IACpF,OAAO;QACL,+EAA+E;QAC/E,8DAA8D;QAC9D,EAAE;QACF,gBAAgB;QAChB,oFAAoF;QACpF,kFAAkF;QAClF,EAAE;QACF,gBAAgB;QAChB,iFAAiF;QACjF,8EAA8E;QAC9E,gFAAgF;QAChF,yFAAyF;QACzF,EAAE;QACF,UAAU;QACV,GAAG,SAAS;QACZ,EAAE;QACF,iBAAiB;QACjB,uGAAuG;QACvG,oEAAoE;QACpE,EAAE;QACF,gBAAgB;QAChB,0DAA0D;KAC3D,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAWD,KAAK,UAAU,IAAI,CACjB,GAMC,EACD,MAAc,EACd,IAAY,EACZ,cAAsB,EACtB,cAAgC;IAEhC,MAAM,QAAQ,GAAc;QAC1B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;QACnC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;KAChC,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,IAAI,CAAC;IACjB,MAAM,MAAM,GAAG,MAAM,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC,QAAiB;QAC/B,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ;QACR,SAAS,EAAE,EAAE;QACb,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc;QACd,cAAc;KACf,CAAC,CAAC;IACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAoC,EAAE,CAAC;QAC/D,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC,EAAE,CAAC;YAC9E,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YAC5B,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAA2D,MAAM,MAAM,CAAC,QAAQ,CAAC;IAC/F,OAAO;QACL,MAAM,EAAE,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;QACpC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;QAC1B,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,WAAW;QACvC,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC;QACxC,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC;QAC1C,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;KAC1C,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,WAAW,CAAC;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,2BAA2B,CAAC;IACrE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IAEzD,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;IAC/B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IAClB,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG;QACV,QAAQ;QACR,KAAK;QACL,MAAM,EAAE,CAAC,CAAC,WAAW;QACrB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,SAAS,EAAE,CAAC,CAAC,SAAS;KACvB,CAAC;IAEF,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,0DAA0D;IAC1D,OAAO,CAAC,GAAG,CAAC,oCAAoC,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,YAAY,IAAI,iBAAiB,CAAC,CAAC;IAE5F,wEAAwE;IACxE,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;IAChE,6EAA6E;IAC7E,2EAA2E;IAC3E,+CAA+C;IAC/C,MAAM,IAAI,GAAkB,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,oBAAoB,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,MAAM,EAAE,CAAC;QAC1E,MAAM,CAAC,GAAG,MAAM,IAAI,CAClB,GAAG,EACH,YAAY,EACZ,YAAY,CAAC,kBAAkB,EAC/B,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EACzB,OAAO,CACR,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,OAAO,CAAC,GAAG,CACT,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,WAAW,CAAC,CAAC,WAAW,gBAAgB,CAAC,CAAC,SAAS,iBAAiB,CAAC,CAAC,UAAU,EAAE,CACpH,CAAC;QACF,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IAElB,oFAAoF;IACpF,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACrC,mDAAmD;IACnD,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,CAAC,UAAU,iBAAiB,CAAC,CAAC;IACtE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IAClB,MAAM,IAAI,GAAkB,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,kBAAkB,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACpF,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,OAAO,CAAC,GAAG,CACT,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,WAAW,CAAC,CAAC,WAAW,gBAAgB,CAAC,CAAC,SAAS,iBAAiB,CAAC,CAAC,UAAU,EAAE,CACpH,CAAC;QACF,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,GAAG,GAAG,CAAC,EAAiB,EAAE,CAA6B,EAAE,EAAE,CAC/D,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IAC/C,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAE/C,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,+BAA+B,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClG,OAAO,CAAC,GAAG,CACT,+BAA+B,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;QACxF,IAAI,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAC1G,CAAC;IACF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACzE,OAAO,CAAC,GAAG,CACT,YAAY,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,qFAAqF,CAAC,CAAC,CAAC,4FAA4F,EAAE,CAClN,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Fast-apply vs current-edit benchmark ā real-API measurement.
|
|
3
|
-
*
|
|
4
|
-
* Two strategies for the same edit task, measured end-to-end against live models:
|
|
5
|
-
*
|
|
6
|
-
* BASELINE (what ezcoder does today): the frontier model emits full
|
|
7
|
-
* search/replace edit blocks (old_text + new_text, verbatim context). Our
|
|
8
|
-
* edit tool then applies them LOCALLY and instantly (no apply model).
|
|
9
|
-
*
|
|
10
|
-
* FAST-APPLY (the proposed feature): the frontier model emits a terse "lazy"
|
|
11
|
-
* edit using `// ... existing code ...` markers (far fewer output tokens), and
|
|
12
|
-
* a cheap/fast "apply" model merges that lazy edit into the full file ā
|
|
13
|
-
* regenerating the whole file as output.
|
|
14
|
-
*
|
|
15
|
-
* We measure, per task: frontier output tokens + time, apply-model output tokens
|
|
16
|
-
* + time, total wall-clock, and an anchor-based correctness check.
|
|
17
|
-
*
|
|
18
|
-
* Usage:
|
|
19
|
-
* npx tsx src/core/fast-apply-benchmark.ts
|
|
20
|
-
*
|
|
21
|
-
* Env overrides:
|
|
22
|
-
* EZ_FA_FRONTIER_PROVIDER / EZ_FA_FRONTIER_MODEL (default openai / gpt-6.1-sol)
|
|
23
|
-
* EZ_FA_APPLY_PROVIDER / EZ_FA_APPLY_MODEL (default gemini / gemini-3.1-flash-lite)
|
|
24
|
-
*/
|
|
25
|
-
export {};
|