opencode-agent-harness 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/LICENSE +21 -0
- package/README.md +590 -2
- package/bin/agent-harness.js +3059 -0
- package/contrib/global-plugins/README.md +49 -0
- package/contrib/global-plugins/harness-freshness.ts +355 -0
- package/package.json +48 -4
- package/templates/.agents/AGENTS.md +204 -0
- package/templates/.agents/features/INDEX.md +10 -0
- package/templates/.agents/handoffs/README.md +87 -0
- package/templates/.agents/memory/dev-loop.md +59 -0
- package/templates/.agents/memory/domain-map.md +29 -0
- package/templates/.agents/memory/environments.md +35 -0
- package/templates/.agents/memory/flow-map.md +23 -0
- package/templates/.agents/memory/history.md +7 -0
- package/templates/.agents/memory/locations.md +14 -0
- package/templates/.agents/memory/model-routing.md +261 -0
- package/templates/.agents/memory/orchestration.md +183 -0
- package/templates/.agents/memory/owned-files.md +19 -0
- package/templates/.agents/memory/product-principles.md +37 -0
- package/templates/.agents/memory/quality-parameters.md +37 -0
- package/templates/.agents/memory/stack-versions.md +35 -0
- package/templates/.agents/memory/test-design-parameters.md +36 -0
- package/templates/.agents/nightly.json +17 -0
- package/templates/.agents/qa-plans/README.md +70 -0
- package/templates/.agents/qa-runs/README.md +21 -0
- package/templates/.agents/reviews/README.md +47 -0
- package/templates/.agents/rules/00-operating.md +199 -0
- package/templates/.agents/skills/data-engineer/SKILL.md +31 -0
- package/templates/.agents/skills/devops-engineer/SKILL.md +50 -0
- package/templates/.agents/skills/diagnostics-expert/SKILL.md +44 -0
- package/templates/.agents/skills/docs/SKILL.md +140 -0
- package/templates/.agents/skills/features/SKILL.md +187 -0
- package/templates/.agents/skills/frontend-engineer/SKILL.md +81 -0
- package/templates/.agents/skills/handoff/SKILL.md +85 -0
- package/templates/.agents/skills/mobile-engineer/SKILL.md +101 -0
- package/templates/.agents/skills/planner/SKILL.md +112 -0
- package/templates/.agents/skills/product-manager/SKILL.md +115 -0
- package/templates/.agents/skills/qa-architect/SKILL.md +224 -0
- package/templates/.agents/skills/qa-runner/SKILL.md +64 -0
- package/templates/.agents/skills/security-engineer/SKILL.md +53 -0
- package/templates/.agents/skills/system-architect/SKILL.md +42 -0
- package/templates/.agents/skills/ui-designer/SKILL.md +71 -0
- package/templates/.github/workflows/ci.yml +148 -0
- package/templates/.github/workflows/deploy-uat.yml +95 -0
- package/templates/.github/workflows/nightly.yml +83 -0
- package/templates/.github/workflows/uat-health.yml +41 -0
- package/templates/.husky/pre-push +23 -0
- package/templates/.ignore +28 -0
- package/templates/.opencode/agents/data-engineer.md +37 -0
- package/templates/.opencode/agents/devops-engineer.md +37 -0
- package/templates/.opencode/agents/diagnostics-expert.md +37 -0
- package/templates/.opencode/agents/docs.md +37 -0
- package/templates/.opencode/agents/features.md +21 -0
- package/templates/.opencode/agents/frontend-engineer.md +37 -0
- package/templates/.opencode/agents/handoff.md +21 -0
- package/templates/.opencode/agents/mobile-engineer.md +37 -0
- package/templates/.opencode/agents/planner.md +34 -0
- package/templates/.opencode/agents/product-manager.md +29 -0
- package/templates/.opencode/agents/qa-architect.md +27 -0
- package/templates/.opencode/agents/qa-runner.md +17 -0
- package/templates/.opencode/agents/security-engineer.md +37 -0
- package/templates/.opencode/agents/system-architect.md +37 -0
- package/templates/.opencode/agents/ui-designer.md +37 -0
- package/templates/.opencode/commands/board.md +15 -0
- package/templates/.opencode/commands/checkpoint.md +25 -0
- package/templates/.opencode/commands/feature.md +56 -0
- package/templates/.opencode/commands/finish.md +36 -0
- package/templates/.opencode/commands/next.md +49 -0
- package/templates/.opencode/commands/report.md +55 -0
- package/templates/.opencode/commands/resume.md +39 -0
- package/templates/.opencode/commands/switch-model.md +34 -0
- package/templates/.opencode/commands/tidy.md +49 -0
- package/templates/.opencode/plugins/backlog-notify.ts +57 -0
- package/templates/.opencode/plugins/backlog.ts +272 -0
- package/templates/.opencode/plugins/checkpoint.ts +160 -0
- package/templates/.opencode/plugins/context-guard.ts +1164 -0
- package/templates/.opencode/plugins/diff.ts +85 -0
- package/templates/.opencode/plugins/gates.ts +71 -0
- package/templates/.opencode/plugins/job-notify.ts +241 -0
- package/templates/.opencode/plugins/orient.ts +67 -0
- package/templates/.opencode/plugins/session.ts +89 -0
- package/templates/.opencode/plugins/wait.ts +101 -0
- package/templates/.opencode/plugins/where.ts +70 -0
- package/templates/.opencode/skills/bounded-explore/SKILL.md +38 -0
- package/templates/.opencode/skills/dev-loop/SKILL.md +87 -0
- package/templates/.vercelignore +16 -0
- package/templates/AGENTS.md +27 -0
- package/templates/BACKLOG.md +95 -0
- package/templates/docs/ci-cd-pipeline.md +81 -0
- package/templates/docs/gates.md +128 -0
- package/templates/docs/mcp-integrations.md +97 -0
- package/templates/docs/mobile-testing.md +555 -0
- package/templates/docs/nightly-testing.md +106 -0
- package/templates/docs/wsl-optimizations.md +290 -0
- package/templates/opencode.json +188 -0
- package/templates/scripts/agent-report.js +541 -0
- package/templates/scripts/agent-search.sh +117 -0
- package/templates/scripts/backlog.js +1864 -0
- package/templates/scripts/board-service.sh +207 -0
- package/templates/scripts/checkpoint.js +290 -0
- package/templates/scripts/finalize.js +277 -0
- package/templates/scripts/nightly.js +913 -0
- package/templates/scripts/orient.js +514 -0
- package/templates/scripts/qa/board-client.js +260 -0
- package/templates/scripts/qa/board-lib.js +96 -0
- package/templates/scripts/qa/board-sync.js +469 -0
- package/templates/scripts/qa/capacity-lib.js +916 -0
- package/templates/scripts/qa/check-backlog.js +427 -0
- package/templates/scripts/qa/check-capacity.js +85 -0
- package/templates/scripts/qa/check-ci-policy.js +164 -0
- package/templates/scripts/qa/check-concurrency.js +344 -0
- package/templates/scripts/qa/check-dispatch-config.js +296 -0
- package/templates/scripts/qa/check-docs.js +284 -0
- package/templates/scripts/qa/check-integrate.js +98 -0
- package/templates/scripts/qa/check-linger.js +160 -0
- package/templates/scripts/qa/check-prefix-stability.js +183 -0
- package/templates/scripts/qa/check-product.js +177 -0
- package/templates/scripts/qa/check-qa-plan.js +378 -0
- package/templates/scripts/qa/check-qa-scripts.js +85 -0
- package/templates/scripts/qa/check-quality.js +249 -0
- package/templates/scripts/qa/check-test-wiring.js +253 -0
- package/templates/scripts/qa/ci-triage.js +346 -0
- package/templates/scripts/qa/ci-usage-lib.js +185 -0
- package/templates/scripts/qa/ci-wait-lib.js +294 -0
- package/templates/scripts/qa/command-classifier.js +552 -0
- package/templates/scripts/qa/context-guard-core.js +1618 -0
- package/templates/scripts/qa/diff.js +174 -0
- package/templates/scripts/qa/feature-scaffold.js +224 -0
- package/templates/scripts/qa/fixtures/backlog.md +104 -0
- package/templates/scripts/qa/governance.js +174 -0
- package/templates/scripts/qa/hash-file.js +32 -0
- package/templates/scripts/qa/lease-lib.js +263 -0
- package/templates/scripts/qa/merge-backlog.js +324 -0
- package/templates/scripts/qa/mobile-e2e/analyze-timings.js +300 -0
- package/templates/scripts/qa/mobile-e2e/apk-cache.js +420 -0
- package/templates/scripts/qa/mobile-e2e/backend-build-cache.js +203 -0
- package/templates/scripts/qa/mobile-e2e/boot-emulator.sh +432 -0
- package/templates/scripts/qa/mobile-e2e/build-apk.sh +210 -0
- package/templates/scripts/qa/mobile-e2e/check-flows.js +453 -0
- package/templates/scripts/qa/mobile-e2e/config.yaml +18 -0
- package/templates/scripts/qa/mobile-e2e/device-down.sh +88 -0
- package/templates/scripts/qa/mobile-e2e/device-up.sh +258 -0
- package/templates/scripts/qa/mobile-e2e/ios-simulator-down.sh +72 -0
- package/templates/scripts/qa/mobile-e2e/ios-simulator-up.sh +153 -0
- package/templates/scripts/qa/mobile-e2e/job-events.js +181 -0
- package/templates/scripts/qa/mobile-e2e/prewarm.sh +56 -0
- package/templates/scripts/qa/mobile-e2e/run-detached.sh +222 -0
- package/templates/scripts/qa/mobile-e2e/run-maestro.js +1543 -0
- package/templates/scripts/qa/mobile-e2e/ui-dump.js +327 -0
- package/templates/scripts/qa/models.allowlist.txt +32 -0
- package/templates/scripts/qa/provider-stats.js +878 -0
- package/templates/scripts/qa/run-clean.js +83 -0
- package/templates/scripts/qa/run-gates.js +212 -0
- package/templates/scripts/qa/run-loop.js +220 -0
- package/templates/scripts/qa/session-recon.js +512 -0
- package/templates/scripts/qa/store-lib.js +324 -0
- package/templates/scripts/qa/tool-lib.js +134 -0
- package/templates/scripts/qa/where.js +214 -0
- package/templates/scripts/qa/wslconfig-merge.js +157 -0
- package/templates/scripts/serve.js +1328 -0
- package/templates/scripts/setup-nightly.sh +185 -0
- package/templates/scripts/setup-tmpfs.sh +374 -0
- package/templates/scripts/ship.js +586 -0
- package/templates/scripts/switch-reasoning-model.ps1 +101 -0
- package/templates/scripts/switch-reasoning-model.sh +104 -0
- package/templates/scripts/wait-for.js +342 -0
- package/templates/scripts/wsl-gpu.sh +180 -0
- package/templates/scripts/wsl-tune.sh +261 -0
|
@@ -0,0 +1,878 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/*
|
|
4
|
+
* provider-stats — deterministic cost/token/turn telemetry from the local
|
|
5
|
+
* opencode store (Phase B of the cost-optimization plan).
|
|
6
|
+
*
|
|
7
|
+
* Reads observed provider data for one project root — including every git
|
|
8
|
+
* worktree registered to it — through `store-lib.js`, which resolves the store
|
|
9
|
+
* and the authoritative table generation. OpenCode V2 removed the
|
|
10
|
+
* `opencode db` subcommand AND moved sessions to `session_v2` +
|
|
11
|
+
* `session_message`; the V1 path (`session`/`message`/`part` via `opencode db`)
|
|
12
|
+
* is kept for a V1 runtime, and both are driven by the same code (see
|
|
13
|
+
* store-lib.js). Aggregates the primary orchestrator session plus its subagent
|
|
14
|
+
* child sessions into one "feature run" group, so `cost per shipped feature`
|
|
15
|
+
* has a real join key (the root session id recorded in the handoff `session:`
|
|
16
|
+
* metadata).
|
|
17
|
+
*
|
|
18
|
+
* Nothing here is inferred from character counts: every number is the
|
|
19
|
+
* provider's own recorded value. When the store is unreachable, the script
|
|
20
|
+
* fails loudly instead of printing fabricated totals.
|
|
21
|
+
*
|
|
22
|
+
* Usage:
|
|
23
|
+
* node scripts/qa/provider-stats.js [--since=14d] [--json]
|
|
24
|
+
* node scripts/qa/provider-stats.js --session <ses_...> [--json]
|
|
25
|
+
* node scripts/qa/provider-stats.js --export <file> # write a fixture
|
|
26
|
+
* node scripts/qa/provider-stats.js --fixture <file> # read a fixture
|
|
27
|
+
* node scripts/qa/provider-stats.js --root <dir> [--opencode <exe>]
|
|
28
|
+
*
|
|
29
|
+
* Fixture schema (stable): { schema, meta, sessions } — the same document the
|
|
30
|
+
* CLI emits on --json, minus aggregates (always recomputed from sessions).
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
const fs = require("fs");
|
|
34
|
+
const os = require("os");
|
|
35
|
+
const path = require("path");
|
|
36
|
+
const { execFileSync, spawnSync } = require("child_process");
|
|
37
|
+
const CG = require("./context-guard-core.js");
|
|
38
|
+
const storeLib = require("./store-lib.js");
|
|
39
|
+
|
|
40
|
+
const SCHEMA = "provider-stats/1";
|
|
41
|
+
const TURN_UNITS = { h: 3600000, d: 86400000, w: 604800000, m: 2592000000 };
|
|
42
|
+
|
|
43
|
+
// ---------------------------------------------------------------- pure core
|
|
44
|
+
|
|
45
|
+
/* Parse `--since`: relative (`14d`, `36h`, `2w`, `1m` = 30d) or an ISO date.
|
|
46
|
+
* Returns an epoch-ms lower bound, or null for "all time". Throws on garbage so
|
|
47
|
+
* a typo can never silently widen the window. */
|
|
48
|
+
function parseSince(text, now = Date.now()) {
|
|
49
|
+
if (text == null || String(text).trim() === "" || /^all$/i.test(String(text).trim())) return null;
|
|
50
|
+
const s = String(text).trim();
|
|
51
|
+
const rel = s.match(/^(\d+)\s*([hdwm])$/i);
|
|
52
|
+
if (rel) {
|
|
53
|
+
const unit = TURN_UNITS[rel[2].toLowerCase()];
|
|
54
|
+
return now - Number(rel[1]) * unit;
|
|
55
|
+
}
|
|
56
|
+
const t = Date.parse(s);
|
|
57
|
+
if (Number.isNaN(t)) {
|
|
58
|
+
throw new Error(`invalid since value "${text}" — use 14d / 36h / 2w / 1m or an ISO date`);
|
|
59
|
+
}
|
|
60
|
+
return t;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/* opencode stores `session.model` as JSON (or a bare string in older rows). */
|
|
64
|
+
function parseModelCell(value) {
|
|
65
|
+
const out = { model: "?", provider: "?" };
|
|
66
|
+
if (value == null || value === "") return out;
|
|
67
|
+
const s = String(value).trim();
|
|
68
|
+
if (s.startsWith("{")) {
|
|
69
|
+
try {
|
|
70
|
+
const parsed = JSON.parse(s);
|
|
71
|
+
if (parsed && parsed.id) out.model = String(parsed.id);
|
|
72
|
+
if (parsed && parsed.providerID) out.provider = String(parsed.providerID);
|
|
73
|
+
return out;
|
|
74
|
+
} catch (_) {
|
|
75
|
+
/* fall through to the bare string */
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
out.model = s;
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/* Nearest-rank percentile over the finite values (no interpolation). */
|
|
83
|
+
function percentile(values, pct) {
|
|
84
|
+
const arr = values.filter((v) => Number.isFinite(v)).sort((a, b) => a - b);
|
|
85
|
+
if (arr.length === 0) return null;
|
|
86
|
+
const idx = Math.min(arr.length - 1, Math.max(0, Math.ceil((pct / 100) * arr.length) - 1));
|
|
87
|
+
return arr[idx];
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function num(v) {
|
|
91
|
+
const n = Number(v);
|
|
92
|
+
return Number.isFinite(n) ? n : 0;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function commas(n) {
|
|
96
|
+
return String(Math.round(num(n))).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function money(n) {
|
|
100
|
+
return "$" + num(n).toFixed(4);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/* Normalize to the forward-slash form the opencode store records. */
|
|
104
|
+
function normalizeRoot(p) {
|
|
105
|
+
return String(p || "").replace(/\\/g, "/").replace(/\/+$/, "");
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function sqlString(value) {
|
|
109
|
+
return "'" + String(value).replace(/'/g, "''") + "'";
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// ---------------------------------------------------------------- collections
|
|
113
|
+
|
|
114
|
+
/* Resolve the real opencode executable. On Windows the PATH entry is usually a
|
|
115
|
+
* `.cmd`/`.ps1` shim that Node cannot execFileSync directly, so prefer the real
|
|
116
|
+
* `.exe` next to it (or the npm-global module path); on POSIX the shim runs. */
|
|
117
|
+
function resolveOpencode(opts = {}) {
|
|
118
|
+
const platform = opts.platform || process.platform;
|
|
119
|
+
const win32Paths = platform === "win32";
|
|
120
|
+
const p = win32Paths ? path.win32 : path;
|
|
121
|
+
const env = opts.env || process.env;
|
|
122
|
+
const exists = opts.exists || fs.existsSync;
|
|
123
|
+
const exec = opts.execFileSync || execFileSync;
|
|
124
|
+
if (env.OPENCODE_BIN && exists(env.OPENCODE_BIN)) return env.OPENCODE_BIN;
|
|
125
|
+
let found = null;
|
|
126
|
+
try {
|
|
127
|
+
const cmd = platform === "win32" ? "where.exe" : "which";
|
|
128
|
+
const out = exec(cmd, ["opencode"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
129
|
+
found = String(out).split(/\r?\n/).map((l) => l.trim()).find(Boolean) || null;
|
|
130
|
+
} catch (_) {
|
|
131
|
+
found = null;
|
|
132
|
+
}
|
|
133
|
+
if (!found) return null;
|
|
134
|
+
if (win32Paths && !/\.exe$/i.test(found)) {
|
|
135
|
+
// `where.exe` often lists the extensionless POSIX shim first; CreateProcess
|
|
136
|
+
// cannot run it. Prefer the real exe beside it, then the npm-global module.
|
|
137
|
+
// Node's host `path` is POSIX when the harness runs under WSL/Linux, so the
|
|
138
|
+
// win32 branch must use `path.win32` (a Windows path split by the host
|
|
139
|
+
// module yields "." — F-191-adjacent WSL portability).
|
|
140
|
+
const exe = /\.(cmd|ps1)$/i.test(found) ? found.replace(/\.(cmd|ps1)$/i, ".exe") : null;
|
|
141
|
+
for (const cand of [found + ".exe", exe]) {
|
|
142
|
+
if (cand && exists(cand)) return cand;
|
|
143
|
+
}
|
|
144
|
+
const npmExe = p.join(p.dirname(found), "node_modules", "opencode-ai", "bin", "opencode.exe");
|
|
145
|
+
if (exists(npmExe)) return npmExe;
|
|
146
|
+
return null;
|
|
147
|
+
}
|
|
148
|
+
return found;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/* Synchronous sleep — `runDbJson` is called from sync CLI paths, so the retry
|
|
152
|
+
* backoff must block the thread. */
|
|
153
|
+
function sleepSync(ms) {
|
|
154
|
+
try {
|
|
155
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, Math.min(2000, ms)));
|
|
156
|
+
} catch (_) {
|
|
157
|
+
/* Atomics.wait unavailable — skip the backoff rather than fail the read */
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/* Run one SQL statement against the store, with a bounded retry.
|
|
162
|
+
*
|
|
163
|
+
* V1 runtime: `opencode db --format json` (the CLI truncates a large dump to a
|
|
164
|
+
* PIPE, so store-lib writes to a real fd). V2: the subcommand is gone and the
|
|
165
|
+
* tables moved — store-lib falls through to `node:sqlite` / the `sqlite3` CLI
|
|
166
|
+
* against the resolved db file, which is also why this no longer needs the temp
|
|
167
|
+
* file dance for the common case. A live store mid-checkpoint can still yield a
|
|
168
|
+
* torn read, so the whole attempt is retried before giving up (never a silent
|
|
169
|
+
* empty aggregate). */
|
|
170
|
+
function runDbJson(opencode, sql, opts = {}) {
|
|
171
|
+
const attempts = Number.isFinite(opts.retries) ? opts.retries : 3;
|
|
172
|
+
const dbPath = opts.dbPath || storeLib.resolveDbPath({ ...opts, opencodeBin: opencode });
|
|
173
|
+
const query = opts.query || storeLib.makeQuery(dbPath, {
|
|
174
|
+
...opts,
|
|
175
|
+
opencodeBin: opencode,
|
|
176
|
+
sqlite3: opts.sqlite3,
|
|
177
|
+
// An explicit `opencode` (or a discovered one) may still serve `db`; keep
|
|
178
|
+
// the preference so a V1 runtime behaves exactly as before.
|
|
179
|
+
opencodeDb: opts.opencodeDb !== false,
|
|
180
|
+
});
|
|
181
|
+
let lastErr = null;
|
|
182
|
+
for (let i = 0; i <= attempts; i++) {
|
|
183
|
+
try {
|
|
184
|
+
return query(sql);
|
|
185
|
+
} catch (e) {
|
|
186
|
+
lastErr = e;
|
|
187
|
+
if (i < attempts) sleepSync(150 * (i + 1));
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
throw lastErr;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/* Project ids registered for this root: the main worktree plus every git
|
|
194
|
+
* worktree opencode knows about (`project_directory.strategy = git_worktree`).
|
|
195
|
+
* This is what makes a session started inside a feature worktree match. */
|
|
196
|
+
function projectIdsSql(root) {
|
|
197
|
+
const wtBase = normalizeRoot(path.posix.join(root, "..", ".worktrees", path.posix.basename(root)));
|
|
198
|
+
return (
|
|
199
|
+
"SELECT DISTINCT project_id AS id FROM project_directory " +
|
|
200
|
+
`WHERE lower(directory) = lower(${sqlString(root)}) ` +
|
|
201
|
+
`OR lower(directory) LIKE lower(${sqlString(wtBase + "/")}) || '%' ` +
|
|
202
|
+
"UNION " +
|
|
203
|
+
`SELECT DISTINCT id FROM project WHERE lower(worktree) = lower(${sqlString(root)})`
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function sessionsSql(projectIds, sinceMs, sessionsTable = "session") {
|
|
208
|
+
const ids = projectIds.map(sqlString).join(", ");
|
|
209
|
+
const since = sinceMs ? ` AND s.time_created >= ${Number(sinceMs)}` : "";
|
|
210
|
+
return (
|
|
211
|
+
"SELECT s.id, s.parent_id, s.title, s.directory, s.agent, s.model, s.cost, " +
|
|
212
|
+
"s.tokens_input, s.tokens_output, s.tokens_reasoning, " +
|
|
213
|
+
"s.tokens_cache_read, s.tokens_cache_write, s.time_created, s.time_updated " +
|
|
214
|
+
`FROM ${sessionsTable} s WHERE s.project_id IN (${ids})${since} ORDER BY s.time_created`
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/* Assistant-message count == orchestrator turns.
|
|
219
|
+
* v1: one row per message in `message`, role in the JSON payload.
|
|
220
|
+
* v2: one row per message in `session_message`, role in a real column. */
|
|
221
|
+
function turnsSql(sessionIds, flavour = "v1") {
|
|
222
|
+
const ids = sessionIds.map(sqlString).join(", ");
|
|
223
|
+
if (flavour === "v2") {
|
|
224
|
+
return (
|
|
225
|
+
"SELECT session_id, COUNT(*) AS turns FROM session_message " +
|
|
226
|
+
`WHERE session_id IN (${ids}) AND type = 'assistant' GROUP BY session_id`
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
return (
|
|
230
|
+
"SELECT session_id, COUNT(*) AS turns FROM message " +
|
|
231
|
+
`WHERE session_id IN (${ids}) AND json_valid(data) ` +
|
|
232
|
+
"AND json_extract(data, '$.role') = 'assistant' GROUP BY session_id"
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/* Tool-call parts, for the loop detector AND the recon's tool-habit metrics.
|
|
237
|
+
* v1: `part` rows, one per part, tool in `$.tool`.
|
|
238
|
+
* v2: parts are embedded in `session_message.data.content[]` — walk the array
|
|
239
|
+
* and read `$.name` (V2) falling back to `$.tool`.
|
|
240
|
+
* Returns `status` / `outputLen` / `time_created` too, so `session-recon.js` can
|
|
241
|
+
* share this one flavour-aware query instead of hardcoding `FROM part` (F-15). */
|
|
242
|
+
function toolPartsSql(sessionIds, flavour = "v1") {
|
|
243
|
+
const ids = sessionIds.map(sqlString).join(", ");
|
|
244
|
+
if (flavour === "v2") {
|
|
245
|
+
return (
|
|
246
|
+
"SELECT m.session_id AS session_id, " +
|
|
247
|
+
"COALESCE(json_extract(je.value, '$.name'), json_extract(je.value, '$.tool')) AS tool, " +
|
|
248
|
+
"json_extract(je.value, '$.state.input') AS input, " +
|
|
249
|
+
"json_extract(je.value, '$.state.status') AS status, " +
|
|
250
|
+
"length(json_extract(je.value, '$.state.output')) AS outputLen, " +
|
|
251
|
+
"m.time_created AS time_created " +
|
|
252
|
+
"FROM session_message m, json_each(m.data, '$.content') je " +
|
|
253
|
+
`WHERE m.session_id IN (${ids}) AND m.type = 'assistant' ` +
|
|
254
|
+
"AND json_type(m.data, '$.content') = 'array' " +
|
|
255
|
+
"AND json_extract(je.value, '$.type') = 'tool'"
|
|
256
|
+
);
|
|
257
|
+
}
|
|
258
|
+
return (
|
|
259
|
+
"SELECT session_id, json_extract(data, '$.tool') AS tool, " +
|
|
260
|
+
"json_extract(data, '$.state.input') AS input, " +
|
|
261
|
+
"json_extract(data, '$.state.status') AS status, " +
|
|
262
|
+
"length(json_extract(data, '$.state.output')) AS outputLen, " +
|
|
263
|
+
"time_created AS time_created FROM part " +
|
|
264
|
+
`WHERE session_id IN (${ids}) AND json_valid(data) ` +
|
|
265
|
+
"AND json_extract(data, '$.type') = 'tool'"
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/* Run one IN-list query in chunks so a 700-session project never overflows the
|
|
270
|
+
* command line / SQL parser. */
|
|
271
|
+
function queryChunked(runDb, ids, buildSql, chunk = 50) {
|
|
272
|
+
const rows = [];
|
|
273
|
+
for (let i = 0; i < ids.length; i += chunk) rows.push(...runDb(buildSql(ids.slice(i, i + chunk))));
|
|
274
|
+
return rows;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function mapSessionRow(row) {
|
|
278
|
+
const model = parseModelCell(row.model);
|
|
279
|
+
return {
|
|
280
|
+
id: row.id,
|
|
281
|
+
parentId: row.parent_id || null,
|
|
282
|
+
title: row.title || "",
|
|
283
|
+
directory: row.directory || "",
|
|
284
|
+
agent: row.agent || null,
|
|
285
|
+
model: model.model,
|
|
286
|
+
provider: model.provider,
|
|
287
|
+
cost: num(row.cost),
|
|
288
|
+
tokens: {
|
|
289
|
+
input: num(row.tokens_input),
|
|
290
|
+
output: num(row.tokens_output),
|
|
291
|
+
reasoning: num(row.tokens_reasoning),
|
|
292
|
+
cacheRead: num(row.tokens_cache_read),
|
|
293
|
+
cacheWrite: num(row.tokens_cache_write),
|
|
294
|
+
},
|
|
295
|
+
turns: 0,
|
|
296
|
+
toolCalls: 0,
|
|
297
|
+
maxIdentical: 0,
|
|
298
|
+
loops: [],
|
|
299
|
+
timeCreated: row.time_created == null ? null : Number(row.time_created),
|
|
300
|
+
timeUpdated: row.time_updated == null ? null : Number(row.time_updated),
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/* Keep only `sessionId` and its descendants (subagent sessions). A no-match
|
|
305
|
+
* yields [] so the caller can report "session not found" without guessing. */
|
|
306
|
+
function filterDescendants(sessions, sessionId) {
|
|
307
|
+
const byId = new Map(sessions.map((s) => [s.id, s]));
|
|
308
|
+
if (!byId.has(sessionId)) return [];
|
|
309
|
+
const children = new Map();
|
|
310
|
+
for (const s of sessions) {
|
|
311
|
+
if (!s.parentId) continue;
|
|
312
|
+
if (!children.has(s.parentId)) children.set(s.parentId, []);
|
|
313
|
+
children.get(s.parentId).push(s.id);
|
|
314
|
+
}
|
|
315
|
+
const out = [];
|
|
316
|
+
const queue = [sessionId];
|
|
317
|
+
const seen = new Set();
|
|
318
|
+
while (queue.length) {
|
|
319
|
+
const id = queue.shift();
|
|
320
|
+
if (seen.has(id)) continue;
|
|
321
|
+
seen.add(id);
|
|
322
|
+
out.push(byId.get(id));
|
|
323
|
+
for (const child of children.get(id) || []) queue.push(child);
|
|
324
|
+
}
|
|
325
|
+
return out;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/* Walk parent_id to the topmost selected ancestor (cycle-safe). */
|
|
329
|
+
function rootIdOf(session, byId) {
|
|
330
|
+
let cur = session;
|
|
331
|
+
let guard = 0;
|
|
332
|
+
while (cur.parentId && byId.has(cur.parentId) && guard++ < 1000) cur = byId.get(cur.parentId);
|
|
333
|
+
return cur.id;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function enrichTurns(sessions, turnRows) {
|
|
337
|
+
const byId = new Map(sessions.map((s) => [s.id, s]));
|
|
338
|
+
for (const row of turnRows) {
|
|
339
|
+
const s = byId.get(row.session_id);
|
|
340
|
+
if (s) s.turns = num(row.turns);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/* Cumulative identical tool+args detection, same canonical key + exemption
|
|
345
|
+
* rules as the live context-guard (`.agents/memory/orchestration.md`
|
|
346
|
+
* `identical_tool_calls`). Only the hash is retained — args can carry secrets. */
|
|
347
|
+
function enrichLoops(sessions, partRows, stuck) {
|
|
348
|
+
const threshold = Number.isFinite(stuck?.identical_tool_calls) ? stuck.identical_tool_calls : 3;
|
|
349
|
+
const counts = new Map(sessions.map((s) => [s.id, new Map()]));
|
|
350
|
+
const tools = new Map(sessions.map((s) => [s.id, new Map()]));
|
|
351
|
+
for (const row of partRows) {
|
|
352
|
+
const sessionCounts = counts.get(row.session_id);
|
|
353
|
+
if (!sessionCounts) continue;
|
|
354
|
+
const sessionTools = tools.get(row.session_id);
|
|
355
|
+
sessionTools.set(row.tool, (sessionTools.get(row.tool) || 0) + 1);
|
|
356
|
+
let input = null;
|
|
357
|
+
if (row.input != null) {
|
|
358
|
+
try {
|
|
359
|
+
input = JSON.parse(row.input);
|
|
360
|
+
} catch (_) {
|
|
361
|
+
input = { raw: String(row.input) };
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
if (CG.isLoopExempt(row.tool, input)) continue;
|
|
365
|
+
const key = CG.loopKey(row.tool, input);
|
|
366
|
+
sessionCounts.set(key, (sessionCounts.get(key) || 0) + 1);
|
|
367
|
+
}
|
|
368
|
+
for (const s of sessions) {
|
|
369
|
+
const sessionCounts = counts.get(s.id);
|
|
370
|
+
let toolCalls = 0;
|
|
371
|
+
for (const c of tools.get(s.id).values()) toolCalls += c;
|
|
372
|
+
s.toolCalls = toolCalls;
|
|
373
|
+
let max = 0;
|
|
374
|
+
const loops = [];
|
|
375
|
+
for (const [key, count] of sessionCounts) {
|
|
376
|
+
if (count > max) max = count;
|
|
377
|
+
if (count >= threshold) {
|
|
378
|
+
const [tool] = key.split(":");
|
|
379
|
+
loops.push({ tool, hash: CG.shortHash(key), count });
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
loops.sort((a, b) => b.count - a.count || a.hash.localeCompare(b.hash));
|
|
383
|
+
s.maxIdentical = max;
|
|
384
|
+
s.loops = loops.slice(0, 10);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// ---------------------------------------------------------------- aggregate
|
|
389
|
+
|
|
390
|
+
/* Machine-readable model-tier targets from `.agents/memory/model-routing.md`
|
|
391
|
+
* (`opportunistic_models:` / `escalation_models:` lines). These classify the
|
|
392
|
+
* observed model mix; anything unlisted is the standard execution tier. */
|
|
393
|
+
function parseModelTiers(text) {
|
|
394
|
+
const out = { opportunistic: [], escalation: [] };
|
|
395
|
+
for (const m of String(text || "").matchAll(/^(opportunistic|escalation)_models:\s*(.+)$/gim)) {
|
|
396
|
+
out[m[1].toLowerCase()] = m[2]
|
|
397
|
+
.split(/[,\s]+/)
|
|
398
|
+
.map((s) => s.trim())
|
|
399
|
+
.filter(Boolean);
|
|
400
|
+
}
|
|
401
|
+
return out;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function readModelTiers(root) {
|
|
405
|
+
let text = "";
|
|
406
|
+
try {
|
|
407
|
+
text = fs.readFileSync(path.join(root, ".agents", "memory", "model-routing.md"), "utf8");
|
|
408
|
+
} catch (_) {
|
|
409
|
+
/* missing file -> no declared tiers (everything is standard) */
|
|
410
|
+
}
|
|
411
|
+
return parseModelTiers(text);
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
function tierOf(model, opportunistic, escalation) {
|
|
415
|
+
if (escalation.has(model)) return "escalation";
|
|
416
|
+
if (opportunistic.has(model)) return "opportunistic";
|
|
417
|
+
return "standard";
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/* Aggregate observed sessions into feature-run groups (a primary orchestrator
|
|
421
|
+
* session + every subagent child session) plus project-wide totals. Pure: all
|
|
422
|
+
* inputs come from `collectStats` (or a fixture), nothing is fabricated. */
|
|
423
|
+
function aggregateSessions(sessions, opts = {}) {
|
|
424
|
+
const stuck = opts.stuck || CG.DEFAULT_STUCK;
|
|
425
|
+
const softTurnBudget = Number.isFinite(opts.softTurnBudget)
|
|
426
|
+
? opts.softTurnBudget
|
|
427
|
+
: CG.DEFAULT_TURNS.soft_turn_budget;
|
|
428
|
+
const escalationModels = new Set(opts.escalationModels || []);
|
|
429
|
+
const opportunisticModels = new Set(opts.opportunisticModels || []);
|
|
430
|
+
const byId = new Map(sessions.map((s) => [s.id, s]));
|
|
431
|
+
|
|
432
|
+
const totals = {
|
|
433
|
+
sessions: sessions.length,
|
|
434
|
+
primarySessions: 0,
|
|
435
|
+
subagentSessions: 0,
|
|
436
|
+
cost: 0,
|
|
437
|
+
tokens: { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0 },
|
|
438
|
+
turns: 0,
|
|
439
|
+
toolCalls: 0,
|
|
440
|
+
};
|
|
441
|
+
const groups = new Map();
|
|
442
|
+
for (const s of sessions) {
|
|
443
|
+
const rootId = rootIdOf(s, byId);
|
|
444
|
+
if (!groups.has(rootId)) {
|
|
445
|
+
const root = byId.get(rootId);
|
|
446
|
+
groups.set(rootId, {
|
|
447
|
+
rootId,
|
|
448
|
+
title: root ? root.title : rootId,
|
|
449
|
+
rootModel: root ? root.model : "?",
|
|
450
|
+
cost: 0,
|
|
451
|
+
turns: 0,
|
|
452
|
+
totalTurns: 0,
|
|
453
|
+
tokens: { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0 },
|
|
454
|
+
sessions: 0,
|
|
455
|
+
subagentSessions: 0,
|
|
456
|
+
toolCalls: 0,
|
|
457
|
+
maxIdentical: 0,
|
|
458
|
+
models: new Set(),
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
const g = groups.get(rootId);
|
|
462
|
+
g.cost += s.cost;
|
|
463
|
+
g.totalTurns += s.turns;
|
|
464
|
+
if (s.id === rootId) g.turns += s.turns;
|
|
465
|
+
else g.subagentSessions += 1;
|
|
466
|
+
g.sessions += 1;
|
|
467
|
+
g.toolCalls += s.toolCalls;
|
|
468
|
+
if (s.maxIdentical > g.maxIdentical) g.maxIdentical = s.maxIdentical;
|
|
469
|
+
g.models.add(s.model);
|
|
470
|
+
for (const k of Object.keys(totals.tokens)) {
|
|
471
|
+
g.tokens[k] += s.tokens[k];
|
|
472
|
+
totals.tokens[k] += s.tokens[k];
|
|
473
|
+
}
|
|
474
|
+
totals.cost += s.cost;
|
|
475
|
+
totals.turns += s.turns;
|
|
476
|
+
totals.toolCalls += s.toolCalls;
|
|
477
|
+
if (s.id === rootId) totals.primarySessions += 1;
|
|
478
|
+
else totals.subagentSessions += 1;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
const inputDenominator =
|
|
482
|
+
totals.tokens.input + totals.tokens.cacheRead + totals.tokens.cacheWrite;
|
|
483
|
+
totals.cacheReadShare = inputDenominator > 0 ? totals.tokens.cacheRead / inputDenominator : null;
|
|
484
|
+
|
|
485
|
+
const groupList = [...groups.values()]
|
|
486
|
+
.map((g) => ({ ...g, models: [...g.models].sort() }))
|
|
487
|
+
.sort((a, b) => b.cost - a.cost || a.rootId.localeCompare(b.rootId));
|
|
488
|
+
|
|
489
|
+
const modelMap = new Map();
|
|
490
|
+
for (const s of sessions) {
|
|
491
|
+
const key = s.model + "|" + s.provider;
|
|
492
|
+
if (!modelMap.has(key)) {
|
|
493
|
+
modelMap.set(key, {
|
|
494
|
+
model: s.model,
|
|
495
|
+
provider: s.provider,
|
|
496
|
+
tier: tierOf(s.model, opportunisticModels, escalationModels),
|
|
497
|
+
sessions: 0,
|
|
498
|
+
cost: 0,
|
|
499
|
+
tokens: { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0 },
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
const m = modelMap.get(key);
|
|
503
|
+
m.sessions += 1;
|
|
504
|
+
m.cost += s.cost;
|
|
505
|
+
for (const k of Object.keys(m.tokens)) m.tokens[k] += s.tokens[k];
|
|
506
|
+
}
|
|
507
|
+
const modelMix = [...modelMap.values()].sort((a, b) => b.cost - a.cost || a.model.localeCompare(b.model));
|
|
508
|
+
|
|
509
|
+
/* Tier mix: cost / sessions / tokens rolled up by tier (standard vs declared
|
|
510
|
+
* opportunistic / escalation targets). Never inferred — classification only. */
|
|
511
|
+
const tierMap = new Map();
|
|
512
|
+
for (const s of sessions) {
|
|
513
|
+
const tier = tierOf(s.model, opportunisticModels, escalationModels);
|
|
514
|
+
if (!tierMap.has(tier)) tierMap.set(tier, { tier, sessions: 0, cost: 0, tokens: { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0 } });
|
|
515
|
+
const t = tierMap.get(tier);
|
|
516
|
+
t.sessions += 1;
|
|
517
|
+
t.cost += s.cost;
|
|
518
|
+
for (const k of Object.keys(t.tokens)) t.tokens[k] += s.tokens[k];
|
|
519
|
+
}
|
|
520
|
+
const tierOrder = { escalation: 0, standard: 1, opportunistic: 2 };
|
|
521
|
+
const tierMix = [...tierMap.values()].sort(
|
|
522
|
+
(a, b) => (tierOrder[a.tier] ?? 9) - (tierOrder[b.tier] ?? 9) || b.cost - a.cost
|
|
523
|
+
);
|
|
524
|
+
|
|
525
|
+
/* Escalation evidence surface: primary (root) sessions observed on a declared
|
|
526
|
+
* escalation model. A pro model on a subagent is a legitimate pinned persona;
|
|
527
|
+
* the primary orchestrator running on one is the thing that needs a recorded
|
|
528
|
+
* reason (agent-report joins that to the handoff). */
|
|
529
|
+
const escalationGroups = groupList.filter((g) => escalationModels.has(g.rootModel));
|
|
530
|
+
const escalation = {
|
|
531
|
+
models: [...escalationModels].sort(),
|
|
532
|
+
count: escalationGroups.length,
|
|
533
|
+
cost: escalationGroups.reduce((a, g) => a + g.cost, 0),
|
|
534
|
+
sessions: escalationGroups.map((g) => ({ id: g.rootId, title: g.title, model: g.rootModel, cost: g.cost })),
|
|
535
|
+
rootIds: escalationGroups.map((g) => g.rootId),
|
|
536
|
+
};
|
|
537
|
+
const opportunistic = {
|
|
538
|
+
models: [...opportunisticModels].sort(),
|
|
539
|
+
sessions: sessions.filter((s) => opportunisticModels.has(s.model)).length,
|
|
540
|
+
cost: sessions.filter((s) => opportunisticModels.has(s.model)).reduce((a, s) => a + s.cost, 0),
|
|
541
|
+
};
|
|
542
|
+
|
|
543
|
+
/* Mid-cycle context resets: a primary session whose title says it resumed a
|
|
544
|
+
* checkpoint. This is the E2E proxy for "the feature needed more than one
|
|
545
|
+
* bounded context" — the metric the auto-resume runner should trend to zero
|
|
546
|
+
* (or absorb automatically). Observed provider data (the session title), never
|
|
547
|
+
* inferred. */
|
|
548
|
+
const resumeList = groupList.filter((g) => /resum|checkpoint/i.test(g.title || ""));
|
|
549
|
+
|
|
550
|
+
const loopCandidates = sessions
|
|
551
|
+
.filter((s) => s.maxIdentical >= (Number.isFinite(stuck.identical_tool_calls) ? stuck.identical_tool_calls : 3))
|
|
552
|
+
.map((s) => ({
|
|
553
|
+
id: s.id,
|
|
554
|
+
title: s.title,
|
|
555
|
+
parentId: s.parentId,
|
|
556
|
+
tool: s.loops[0] ? s.loops[0].tool : null,
|
|
557
|
+
hash: s.loops[0] ? s.loops[0].hash : null,
|
|
558
|
+
count: s.maxIdentical,
|
|
559
|
+
}))
|
|
560
|
+
.sort((a, b) => b.count - a.count || a.id.localeCompare(b.id));
|
|
561
|
+
|
|
562
|
+
return {
|
|
563
|
+
totals,
|
|
564
|
+
percentiles: {
|
|
565
|
+
primaryCost: {
|
|
566
|
+
p50: percentile(groupList.map((g) => g.cost), 50),
|
|
567
|
+
p90: percentile(groupList.map((g) => g.cost), 90),
|
|
568
|
+
},
|
|
569
|
+
primaryTurns: {
|
|
570
|
+
p50: percentile(groupList.map((g) => g.turns), 50),
|
|
571
|
+
p90: percentile(groupList.map((g) => g.turns), 90),
|
|
572
|
+
},
|
|
573
|
+
},
|
|
574
|
+
modelMix,
|
|
575
|
+
tierMix,
|
|
576
|
+
escalation,
|
|
577
|
+
opportunistic,
|
|
578
|
+
topSessions: groupList.slice(0, 5).map((g) => ({
|
|
579
|
+
id: g.rootId,
|
|
580
|
+
title: g.title,
|
|
581
|
+
cost: g.cost,
|
|
582
|
+
turns: g.turns,
|
|
583
|
+
tokens: g.tokens,
|
|
584
|
+
sessions: g.sessions,
|
|
585
|
+
models: g.models,
|
|
586
|
+
})),
|
|
587
|
+
sessionsOverTurnBudget: {
|
|
588
|
+
threshold: softTurnBudget,
|
|
589
|
+
count: groupList.filter((g) => g.turns > softTurnBudget).length,
|
|
590
|
+
sessions: groupList.filter((g) => g.turns > softTurnBudget).slice(0, 10).map((g) => ({ id: g.rootId, title: g.title, turns: g.turns, cost: g.cost })),
|
|
591
|
+
},
|
|
592
|
+
loopCandidates: {
|
|
593
|
+
threshold: Number.isFinite(stuck.identical_tool_calls) ? stuck.identical_tool_calls : 3,
|
|
594
|
+
count: loopCandidates.length,
|
|
595
|
+
sessions: loopCandidates.slice(0, 10),
|
|
596
|
+
},
|
|
597
|
+
resumes: {
|
|
598
|
+
count: resumeList.length,
|
|
599
|
+
sessions: resumeList.slice(0, 10).map((g) => ({
|
|
600
|
+
id: g.rootId,
|
|
601
|
+
title: g.title,
|
|
602
|
+
cost: g.cost,
|
|
603
|
+
turns: g.turns,
|
|
604
|
+
})),
|
|
605
|
+
},
|
|
606
|
+
groups: Object.fromEntries(groupList.map((g) => [g.rootId, { cost: g.cost, turns: g.turns, totalTurns: g.totalTurns, sessions: g.sessions, tokens: g.tokens, rootModel: g.rootModel, models: g.models }])),
|
|
607
|
+
};
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
// ---------------------------------------------------------------- collect
|
|
611
|
+
|
|
612
|
+
function readThresholds(root) {
|
|
613
|
+
let text = "";
|
|
614
|
+
try {
|
|
615
|
+
text = fs.readFileSync(path.join(root, ".agents", "memory", "orchestration.md"), "utf8");
|
|
616
|
+
} catch (_) {
|
|
617
|
+
/* missing file -> documented defaults */
|
|
618
|
+
}
|
|
619
|
+
return CG.parseThresholds(text);
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/* The script runs from `<project>/scripts/qa/`, so the project root is two
|
|
623
|
+
* levels up. In the harness repo itself (templates/scripts/qa) pass `--root`
|
|
624
|
+
* explicitly; the harness copy is validated through deployed-project tests. */
|
|
625
|
+
function defaultRoot() {
|
|
626
|
+
return normalizeRoot(path.resolve(__dirname, "..", ".."));
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/* Collect observed sessions for a project root. Injectable `runDb` keeps this
|
|
630
|
+
* unit-testable without a live opencode install; `opencode` overrides the
|
|
631
|
+
* executable path; `now` fixes relative `--since` values. */
|
|
632
|
+
function collectStats(opts = {}) {
|
|
633
|
+
const root = normalizeRoot(opts.root || defaultRoot());
|
|
634
|
+
const since = parseSince(opts.since, opts.now);
|
|
635
|
+
const thresholds = readThresholds(root);
|
|
636
|
+
const stuck = opts.stuck || thresholds.stuck;
|
|
637
|
+
const softTurnBudget = Number.isFinite(opts.softTurnBudget) ? opts.softTurnBudget : thresholds.turns.soft_turn_budget;
|
|
638
|
+
// The opencode CLI is only needed when the store must be READ through it. A
|
|
639
|
+
// caller that injects its own `runDb`/`query` (fixtures, unit tests) does not
|
|
640
|
+
// need opencode installed — demanding it turned a store-only test into a false
|
|
641
|
+
// CI failure on every runner without the CLI (opencode is not installed in CI).
|
|
642
|
+
const external = Boolean(opts.runDb || opts.query);
|
|
643
|
+
const opencode = external ? opts.opencode || null : opts.opencode || resolveOpencode();
|
|
644
|
+
if (!external && !opencode) {
|
|
645
|
+
throw new Error("opencode CLI not found — install it, or set OPENCODE_BIN / pass --opencode <path>");
|
|
646
|
+
}
|
|
647
|
+
// Table generation + engine. An explicit `store` (fixtures / advanced callers)
|
|
648
|
+
// wins; otherwise DETECT the generation through whatever `runDb` we have. A
|
|
649
|
+
// runtime caller that builds its own `runDb` (session-recon) must still see the
|
|
650
|
+
// live V2 tables — the old "any runDb implies V1" shortcut made the recon read
|
|
651
|
+
// the frozen `part` mirror on a V2 store (F-15). A test stub that does not
|
|
652
|
+
// answer the detection query falls back to V1, preserving the injected-shape
|
|
653
|
+
// contract (tests assert the V1 SQL).
|
|
654
|
+
const store = opts.store
|
|
655
|
+
? opts.store
|
|
656
|
+
: opts.runDb
|
|
657
|
+
? storeLib.tableSet(opts.runDb, opts.flavour)
|
|
658
|
+
: storeLib.openStore({ ...opts, opencodeBin: opencode, query: opts.query });
|
|
659
|
+
const runDb = opts.runDb || (opts.query ? (sql) => opts.query(sql) : (sql) => runDbJson(opencode, sql, { ...opts, dbPath: store.dbPath }));
|
|
660
|
+
|
|
661
|
+
const projectIds = runDb(projectIdsSql(root)).map((r) => r.id).filter(Boolean);
|
|
662
|
+
// `--root` may be a linked feature worktree that no `project`/`project_directory`
|
|
663
|
+
// row matches, so the requested session (which belongs to the MAIN project) is
|
|
664
|
+
// invisible and the tool reports "store not reachable from this machine". The
|
|
665
|
+
// session id is the global join key: widen the scan to the session's own
|
|
666
|
+
// project id so `--session` works from any linked worktree.
|
|
667
|
+
if (opts.session) {
|
|
668
|
+
const owner = runDb(`SELECT project_id AS id FROM ${store.sessions} WHERE id = ${sqlString(opts.session)}`)
|
|
669
|
+
.map((r) => r.id)
|
|
670
|
+
.filter(Boolean);
|
|
671
|
+
for (const id of owner) if (!projectIds.includes(id)) projectIds.push(id);
|
|
672
|
+
}
|
|
673
|
+
const meta = {
|
|
674
|
+
schema: SCHEMA,
|
|
675
|
+
generatedAt: new Date(opts.now || Date.now()).toISOString(),
|
|
676
|
+
root,
|
|
677
|
+
since: since ? { arg: String(opts.since), fromMs: since } : null,
|
|
678
|
+
projectIds,
|
|
679
|
+
scope: null,
|
|
680
|
+
stuck,
|
|
681
|
+
softTurnBudget,
|
|
682
|
+
store: { flavour: store.flavour, sessions: store.sessions, db: opts.runDb ? null : store.dbPath },
|
|
683
|
+
modelTiers: readModelTiers(root),
|
|
684
|
+
};
|
|
685
|
+
if (projectIds.length === 0) return { meta: { ...meta, note: "no opencode project matches this root" }, sessions: [] };
|
|
686
|
+
|
|
687
|
+
let sessions = runDb(sessionsSql(projectIds, since, store.sessions)).map(mapSessionRow);
|
|
688
|
+
if (opts.session) {
|
|
689
|
+
const allById = new Map(sessions.map((s) => [s.id, s]));
|
|
690
|
+
const requested = allById.get(opts.session);
|
|
691
|
+
if (!requested) {
|
|
692
|
+
return { meta: { ...meta, scope: { session: opts.session, rootId: null } }, sessions: [] };
|
|
693
|
+
}
|
|
694
|
+
const rootId = rootIdOf(requested, allById);
|
|
695
|
+
sessions = filterDescendants(sessions, rootId);
|
|
696
|
+
meta.scope = { session: opts.session, rootId };
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
const ids = sessions.map((s) => s.id);
|
|
700
|
+
enrichTurns(sessions, ids.length ? queryChunked(runDb, ids, (chunk) => turnsSql(chunk, store.flavour)) : []);
|
|
701
|
+
enrichLoops(sessions, ids.length ? queryChunked(runDb, ids, (chunk) => toolPartsSql(chunk, store.flavour)) : [], stuck);
|
|
702
|
+
return { meta, sessions };
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
function readFixture(file) {
|
|
706
|
+
const doc = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
707
|
+
if (!doc || doc.schema !== SCHEMA || !Array.isArray(doc.sessions)) {
|
|
708
|
+
throw new Error(`${file} is not a ${SCHEMA} fixture`);
|
|
709
|
+
}
|
|
710
|
+
return { schema: SCHEMA, meta: { ...(doc.meta || {}), schema: SCHEMA }, sessions: doc.sessions };
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
function writeFixture(file, stats) {
|
|
714
|
+
fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true });
|
|
715
|
+
fs.writeFileSync(file, JSON.stringify({ schema: SCHEMA, meta: stats.meta, sessions: stats.sessions }, null, 2) + "\n", "utf8");
|
|
716
|
+
return path.resolve(file);
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
// ---------------------------------------------------------------- CLI
|
|
720
|
+
|
|
721
|
+
function parseArgs(argv) {
|
|
722
|
+
const out = { json: false, since: null, session: null, exportFile: null, fixture: null, opencode: null, root: null, help: false };
|
|
723
|
+
for (let i = 0; i < argv.length; i++) {
|
|
724
|
+
const a = argv[i];
|
|
725
|
+
if (a === "--json") out.json = true;
|
|
726
|
+
else if (a === "--help" || a === "-h") out.help = true;
|
|
727
|
+
else if (a === "--since") out.since = argv[++i];
|
|
728
|
+
else if (a.startsWith("--since=")) out.since = a.slice("--since=".length);
|
|
729
|
+
else if (a === "--session") out.session = argv[++i];
|
|
730
|
+
else if (a === "--export") out.exportFile = argv[++i];
|
|
731
|
+
else if (a === "--fixture") out.fixture = argv[++i];
|
|
732
|
+
else if (a === "--opencode") out.opencode = argv[++i];
|
|
733
|
+
else if (a === "--root") out.root = argv[++i];
|
|
734
|
+
}
|
|
735
|
+
return out;
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
function printHuman(stats, agg) {
|
|
739
|
+
const { meta } = stats;
|
|
740
|
+
const t = agg.totals;
|
|
741
|
+
const since = meta.since ? meta.since.arg : "all time";
|
|
742
|
+
console.log(`Provider telemetry — ${meta.root} (${since})\n`);
|
|
743
|
+
if (t.sessions === 0) {
|
|
744
|
+
console.log(meta.note ? `No sessions: ${meta.note}` : "No sessions in this window.");
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
if (meta.scope && meta.scope.session) {
|
|
748
|
+
console.log(`Scope: session ${meta.scope.session} (primary ${meta.scope.rootId || "not found"})`);
|
|
749
|
+
}
|
|
750
|
+
console.log(
|
|
751
|
+
`Sessions: ${t.sessions} (${t.primarySessions} primary + ${t.subagentSessions} subagent) · ` +
|
|
752
|
+
`Cost: ${money(t.cost)} · Turns: ${commas(t.turns)}`
|
|
753
|
+
);
|
|
754
|
+
console.log(
|
|
755
|
+
`Tokens: input ${commas(t.tokens.input)} · cache-read ${commas(t.tokens.cacheRead)}` +
|
|
756
|
+
(t.cacheReadShare == null ? "" : ` (${(t.cacheReadShare * 100).toFixed(1)}% of input)`) +
|
|
757
|
+
` · cache-write ${commas(t.tokens.cacheWrite)} · output ${commas(t.tokens.output)} · reasoning ${commas(t.tokens.reasoning)}`
|
|
758
|
+
);
|
|
759
|
+
console.log(
|
|
760
|
+
`Per primary session: cost P50 ${money(agg.percentiles.primaryCost.p50)} · P90 ${money(agg.percentiles.primaryCost.p90)}` +
|
|
761
|
+
` · turns P50 ${agg.percentiles.primaryTurns.p50 == null ? "n/a" : agg.percentiles.primaryTurns.p50}` +
|
|
762
|
+
` · P90 ${agg.percentiles.primaryTurns.p90 == null ? "n/a" : agg.percentiles.primaryTurns.p90}`
|
|
763
|
+
);
|
|
764
|
+
console.log(
|
|
765
|
+
`Primary sessions over ${agg.sessionsOverTurnBudget.threshold} turns: ${agg.sessionsOverTurnBudget.count}` +
|
|
766
|
+
` · identical-call loops (>= ${agg.loopCandidates.threshold}x): ${agg.loopCandidates.count}`
|
|
767
|
+
);
|
|
768
|
+
|
|
769
|
+
console.log("\nModel mix (all sessions):");
|
|
770
|
+
for (const m of agg.modelMix) {
|
|
771
|
+
console.log(
|
|
772
|
+
` ${(m.provider + "/" + m.model).padEnd(38)} ${String(m.sessions).padStart(4)} sessions ${money(m.cost).padStart(10)} ` +
|
|
773
|
+
`in ${commas(m.tokens.input)} · cache ${commas(m.tokens.cacheRead)} · out ${commas(m.tokens.output)}`
|
|
774
|
+
);
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
const tiers = agg.tierMix.filter((t) => t.sessions > 0);
|
|
778
|
+
if (tiers.length) {
|
|
779
|
+
console.log("\nModel mix by tier:");
|
|
780
|
+
for (const t of tiers) {
|
|
781
|
+
console.log(` ${t.tier.padEnd(14)} ${String(t.sessions).padStart(4)} sessions ${money(t.cost).padStart(10)}`);
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
if (agg.escalation.count) {
|
|
785
|
+
console.log(
|
|
786
|
+
`\nEscalation tier: ${agg.escalation.count} primary session(s) on ${agg.escalation.models.join(", ")} — ` +
|
|
787
|
+
`${money(agg.escalation.cost)} (record the reason in the handoff; agent-report flags the missing ones)`
|
|
788
|
+
);
|
|
789
|
+
}
|
|
790
|
+
if (agg.opportunistic.sessions) {
|
|
791
|
+
console.log(
|
|
792
|
+
`Opportunistic tier: ${agg.opportunistic.sessions} session(s) (${money(agg.opportunistic.cost)}) on ${agg.opportunistic.models.join(", ")}`
|
|
793
|
+
);
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
if (agg.topSessions.length) {
|
|
797
|
+
console.log("\nTop primary sessions by cost:");
|
|
798
|
+
for (const s of agg.topSessions) {
|
|
799
|
+
console.log(` ${money(s.cost).padStart(10)} ${String(s.turns).padStart(4)} turns ${String(s.sessions).padStart(3)} session(s) ${s.id} ${s.title.slice(0, 60)}`);
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
if (agg.loopCandidates.sessions.length) {
|
|
804
|
+
console.log(`\nLoop candidates (cumulative identical tool+args >= ${agg.loopCandidates.threshold}):`);
|
|
805
|
+
for (const l of agg.loopCandidates.sessions) {
|
|
806
|
+
console.log(` ${String(l.count).padStart(3)}x ${l.tool || "?"} #${l.hash || "?"} ${l.id} ${l.title.slice(0, 60)}`);
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
console.log("\nAll values are observed provider data (opencode store). Nothing is inferred from text length.");
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
function main() {
|
|
813
|
+
const args = parseArgs(process.argv.slice(2));
|
|
814
|
+
if (args.help) {
|
|
815
|
+
console.log(
|
|
816
|
+
"Usage: node scripts/qa/provider-stats.js [--since=14d] [--session <id>] [--json]\n" +
|
|
817
|
+
" [--export <file>] [--fixture <file>] [--root <dir>] [--opencode <path>]"
|
|
818
|
+
);
|
|
819
|
+
return;
|
|
820
|
+
}
|
|
821
|
+
const stats = args.fixture
|
|
822
|
+
? readFixture(args.fixture)
|
|
823
|
+
: collectStats({ root: args.root || defaultRoot(), since: args.since, session: args.session, opencode: args.opencode });
|
|
824
|
+
const thresholds = readThresholds(stats.meta.root || defaultRoot());
|
|
825
|
+
const tiers = stats.meta.modelTiers || readModelTiers(stats.meta.root || defaultRoot());
|
|
826
|
+
const agg = aggregateSessions(stats.sessions, {
|
|
827
|
+
stuck: stats.meta.stuck || thresholds.stuck,
|
|
828
|
+
softTurnBudget: stats.meta.softTurnBudget,
|
|
829
|
+
opportunisticModels: tiers.opportunistic,
|
|
830
|
+
escalationModels: tiers.escalation,
|
|
831
|
+
});
|
|
832
|
+
if (args.exportFile) {
|
|
833
|
+
const file = writeFixture(args.exportFile, stats);
|
|
834
|
+
console.error(`provider-stats: wrote ${file}`);
|
|
835
|
+
}
|
|
836
|
+
if (args.json) console.log(JSON.stringify({ schema: SCHEMA, ...stats, aggregates: agg }, null, 2));
|
|
837
|
+
else printHuman(stats, agg);
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
if (require.main === module) {
|
|
841
|
+
try {
|
|
842
|
+
main();
|
|
843
|
+
} catch (e) {
|
|
844
|
+
console.error("provider-stats: " + (e && e.message ? e.message : e));
|
|
845
|
+
process.exit(1);
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
module.exports = {
|
|
850
|
+
SCHEMA,
|
|
851
|
+
parseSince,
|
|
852
|
+
parseModelCell,
|
|
853
|
+
percentile,
|
|
854
|
+
normalizeRoot,
|
|
855
|
+
sqlString,
|
|
856
|
+
resolveOpencode,
|
|
857
|
+
runDbJson,
|
|
858
|
+
projectIdsSql,
|
|
859
|
+
sessionsSql,
|
|
860
|
+
turnsSql,
|
|
861
|
+
toolPartsSql,
|
|
862
|
+
queryChunked,
|
|
863
|
+
mapSessionRow,
|
|
864
|
+
filterDescendants,
|
|
865
|
+
rootIdOf,
|
|
866
|
+
enrichTurns,
|
|
867
|
+
enrichLoops,
|
|
868
|
+
aggregateSessions,
|
|
869
|
+
parseModelTiers,
|
|
870
|
+
readModelTiers,
|
|
871
|
+
readThresholds,
|
|
872
|
+
defaultRoot,
|
|
873
|
+
collectStats,
|
|
874
|
+
readFixture,
|
|
875
|
+
writeFixture,
|
|
876
|
+
commas,
|
|
877
|
+
money,
|
|
878
|
+
};
|