@xhulz/nina 0.28.21 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +1 -1
- package/releases/0.29.0/core/GAPS.md +125 -0
- package/releases/0.29.0/core/answered.json +29 -0
- package/releases/0.29.0/core/defaults/tree/.claude/agents/architect.md +1 -0
- package/releases/0.29.0/core/defaults/tree/.claude/agents/dba.md +3 -0
- package/releases/0.29.0/core/defaults/tree/.claude/agents/devops.md +1 -0
- package/releases/0.29.0/core/defaults/tree/.claude/agents/implementer.md +1 -0
- package/releases/0.29.0/core/defaults/tree/.claude/agents/integration-tester.md +3 -0
- package/releases/0.29.0/core/defaults/tree/.claude/agents/qa.md +2 -0
- package/releases/0.29.0/core/defaults/tree/.claude/agents/reviewer.md +3 -0
- package/releases/0.29.0/core/defaults/tree/.claude/agents/secops.md +3 -0
- package/releases/0.29.0/core/defaults/tree/.claude/agents/solidity-auditor.md +3 -0
- package/releases/0.29.0/core/defaults/tree/.claude/agents/solidity-dev.md +3 -0
- package/releases/0.29.0/core/tree/.claude/agents/architect.md +125 -0
- package/releases/0.29.0/core/tree/.claude/agents/dba.md +107 -0
- package/releases/0.29.0/core/tree/.claude/agents/devops.md +113 -0
- package/releases/0.29.0/core/tree/.claude/agents/implementer.md +155 -0
- package/releases/0.29.0/core/tree/.claude/agents/integration-tester.md +160 -0
- package/releases/0.29.0/core/tree/.claude/agents/planner.md +132 -0
- package/releases/0.29.0/core/tree/.claude/agents/qa.md +146 -0
- package/releases/0.29.0/core/tree/.claude/agents/reviewer.md +180 -0
- package/releases/0.29.0/core/tree/.claude/agents/secops.md +121 -0
- package/releases/0.29.0/core/tree/.claude/agents/solidity-auditor.md +126 -0
- package/releases/0.29.0/core/tree/.claude/agents/solidity-dev.md +123 -0
- package/releases/0.29.0/core/tree/.claude/agents-overview.md +64 -0
- package/releases/0.29.0/core/tree/.claude/graph.md +73 -0
- package/releases/0.29.0/core/tree/.claude/patterns.md +212 -0
- package/releases/0.29.0/core/tree/.claude/pills/README.md +145 -0
- package/releases/0.29.0/core/tree/.claude/pipeline.md +102 -0
- package/releases/0.29.0/core/tree/.claude/retrieval.md +104 -0
- package/releases/0.29.0/core/tree/.claude/router.md +201 -0
- package/releases/0.29.0/core/tree/.claude/templates/integration.md +45 -0
- package/releases/0.29.0/core/tree/CLAUDE.md +155 -0
- package/releases/0.29.0/core/tree/scripts/edit-guard.mjs +16 -0
- package/releases/0.29.0/core/tree/scripts/harness-check.mjs +71 -0
- package/releases/0.29.0/core/tree/scripts/loop-gate.mjs +19 -0
- package/releases/0.29.0/core/vocabulary.json +12 -0
- package/releases/0.29.0/surfaces/blockchain/tree/.claude/agents/architect.md +5 -0
- package/releases/0.29.0/surfaces/blockchain/tree/.claude/agents/devops.md +5 -0
- package/releases/0.29.0/surfaces/blockchain/tree/.claude/agents/reviewer.md +5 -0
- package/releases/0.29.0/surfaces/blockchain/tree/.claude/agents/solidity-auditor.md +18 -0
- package/releases/0.29.0/surfaces/blockchain/tree/.claude/agents/solidity-dev.md +16 -0
- package/releases/0.29.0/surfaces/blockchain/tree/.claude/agents-overview.md +6 -0
- package/releases/0.29.0/surfaces/blockchain/tree/.claude/graph.md +11 -0
- package/releases/0.29.0/surfaces/blockchain/tree/.claude/router.md +9 -0
- package/releases/0.29.0/surfaces/blockchain/tree/CLAUDE.md +12 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/agents/architect.md +8 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/agents/dba.md +84 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/agents/devops.md +14 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/agents/implementer.md +5 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/agents/integration-tester.md +13 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/agents/planner.md +2 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/agents/reviewer.md +25 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/agents/secops.md +2 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/agents-overview.md +5 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/graph.md +8 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/patterns.md +29 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/pipeline.md +13 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/retrieval.md +5 -0
- package/releases/0.29.0/surfaces/db/tree/.claude/router.md +10 -0
- package/releases/0.29.0/surfaces/db/tree/CLAUDE.md +23 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/agents/architect.md +14 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/agents/dba.md +2 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/agents/devops.md +31 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/agents/implementer.md +14 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/agents/integration-tester.md +11 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/agents/qa.md +3 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/agents/reviewer.md +8 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/agents/secops.md +8 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/patterns.md +17 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/retrieval.md +8 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/.claude/router.md +21 -0
- package/releases/0.29.0/surfaces/edge-cf/tree/CLAUDE.md +27 -0
- package/releases/0.29.0/surfaces/frontend/tree/.claude/agents/architect.md +20 -0
- package/releases/0.29.0/surfaces/frontend/tree/.claude/agents/devops.md +28 -0
- package/releases/0.29.0/surfaces/frontend/tree/.claude/agents/implementer.md +25 -0
- package/releases/0.29.0/surfaces/frontend/tree/.claude/agents/qa.md +17 -0
- package/releases/0.29.0/surfaces/frontend/tree/.claude/agents/reviewer.md +62 -0
- package/releases/0.29.0/surfaces/frontend/tree/.claude/patterns.md +15 -0
- package/releases/0.29.0/surfaces/frontend/tree/.claude/retrieval.md +5 -0
- package/releases/0.29.0/surfaces/frontend/tree/.claude/router.md +17 -0
- package/releases/0.29.0/surfaces/frontend/tree/CLAUDE.md +5 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/agents/architect.md +16 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/agents/implementer.md +12 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/agents/integration-tester.md +20 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/agents/planner.md +2 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/agents/reviewer.md +21 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/agents/secops.md +2 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/agents-overview.md +8 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/graph.md +9 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/patterns.md +38 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/pipeline.md +16 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/retrieval.md +16 -0
- package/releases/0.29.0/surfaces/integrations/tree/.claude/router.md +6 -0
- package/releases/0.29.0/surfaces/integrations/tree/CLAUDE.md +17 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/agents/architect.md +12 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/agents/dba.md +14 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/agents/implementer.md +11 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/agents/integration-tester.md +3 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/agents/reviewer.md +23 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/agents/secops.md +5 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/agents/solidity-auditor.md +4 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/agents/solidity-dev.md +4 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/graph.md +2 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/patterns.md +49 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/pipeline.md +2 -0
- package/releases/0.29.0/surfaces/money/tree/.claude/retrieval.md +2 -0
- package/releases/0.29.0/surfaces/money/tree/CLAUDE.md +11 -0
- package/releases/0.29.0/surfaces/pii/tree/.claude/agents/dba.md +2 -0
- package/releases/0.29.0/surfaces/pii/tree/.claude/agents/devops.md +8 -0
- package/releases/0.29.0/surfaces/pii/tree/.claude/agents/reviewer.md +8 -0
- package/releases/0.29.0/surfaces/pii/tree/.claude/agents/secops.md +2 -0
- package/releases/0.29.0/surfaces/pii/tree/.claude/agents/solidity-auditor.md +5 -0
- package/releases/0.29.0/surfaces/pii/tree/CLAUDE.md +5 -0
- package/src/agentrun.mjs +12 -4
- package/src/commands/compose.mjs +45 -1
- package/src/commands/export.mjs +15 -4
- package/src/commands/gate.mjs +91 -8
- package/src/commands/init.mjs +34 -15
- package/src/commands/learn.mjs +22 -9
- package/src/commands/pipeline.mjs +17 -42
- package/src/commands/snapshot.mjs +35 -4
- package/src/commands/stats.mjs +96 -53
- package/src/commands/upgrade.mjs +3 -0
- package/src/deep.mjs +4 -31
- package/src/detectors.mjs +33 -4
- package/src/gate.mjs +1 -1
- package/src/langfuse.mjs +2 -0
- package/src/transcripts.mjs +232 -81
package/src/commands/stats.mjs
CHANGED
|
@@ -18,7 +18,7 @@ import { createReadStream, existsSync, readFileSync, realpathSync, statSync } fr
|
|
|
18
18
|
import { createInterface } from 'node:readline';
|
|
19
19
|
import { homedir } from 'node:os';
|
|
20
20
|
import { basename, join } from 'node:path';
|
|
21
|
-
import { isLoopBack } from '../transcripts.mjs';
|
|
21
|
+
import { ROLE_TOKENS, isLoopBack, runOf } from '../transcripts.mjs';
|
|
22
22
|
import { frontmatter, pillFiles } from './pills.mjs';
|
|
23
23
|
import { HARNESS, snapshotsDir } from '../paths.mjs';
|
|
24
24
|
import { defaultVocabulary } from '../vocabulary.mjs';
|
|
@@ -26,20 +26,17 @@ import { layerRootFor } from './compose.mjs';
|
|
|
26
26
|
import { PRICES_AS_OF, costOf } from '../prices.mjs';
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
|
-
* The
|
|
30
|
-
*
|
|
31
|
-
* of the report by default — capture stays wide because losing a transcript is
|
|
32
|
-
* irreversible, but measuring a pipeline against projects that have none is noise.
|
|
33
|
-
*/
|
|
34
|
-
/**
|
|
35
|
-
* The roles whose job is to approve or reject.
|
|
29
|
+
* The roles whose job is to approve or reject: every stage whose verdicts include no `-READY` token, the one
|
|
30
|
+
* a stage gives when it has produced something rather than ruled on it.
|
|
36
31
|
*
|
|
37
32
|
* Only these can be judged by loop-back rate. A planner, architect or implementer produces
|
|
38
33
|
* work rather than ruling on it, so its rate is near zero by nature and says nothing — the
|
|
39
34
|
* first version of this check used a sample size instead of a role list and therefore asked
|
|
40
|
-
* the question of everyone, which is why it had to be silenced by a single loop-back.
|
|
35
|
+
* the question of everyone, which is why it had to be silenced by a single loop-back. Read from
|
|
36
|
+
* `ROLE_TOKENS` rather than listed here: the list here was written before the blockchain surface and never
|
|
37
|
+
* learned its auditor, so a blockchain project's gate was never judged as one.
|
|
41
38
|
*/
|
|
42
|
-
const GATES = new Set([
|
|
39
|
+
const GATES = new Set(Object.entries(ROLE_TOKENS).filter(([, tokens]) => !tokens.some((t) => t.endsWith('-READY'))).map(([role]) => role));
|
|
43
40
|
|
|
44
41
|
/** Below this rate a gate is worth a second look. The gates that do stop things sit near 20%. */
|
|
45
42
|
const GATE_FLOOR = 0.05;
|
|
@@ -47,17 +44,13 @@ const GATE_FLOOR = 0.05;
|
|
|
47
44
|
/** Fewer readable verdicts than this and the rate is noise, whatever it says. */
|
|
48
45
|
const GATE_SAMPLE = 20;
|
|
49
46
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
'qa',
|
|
58
|
-
'secops',
|
|
59
|
-
'devops',
|
|
60
|
-
]);
|
|
47
|
+
/**
|
|
48
|
+
* The pipeline roles: every stage a release composes, as `ROLE_TOKENS` names them. A project that
|
|
49
|
+
* dispatches only generic agents (`general-purpose`, `Explore`, `Plan`) is not running the harness, so it
|
|
50
|
+
* is left out of the report by default — capture stays wide because losing a transcript is irreversible,
|
|
51
|
+
* but measuring a pipeline against projects that have none is noise.
|
|
52
|
+
*/
|
|
53
|
+
const PIPELINE_ROLES = new Set(Object.keys(ROLE_TOKENS));
|
|
61
54
|
|
|
62
55
|
/**
|
|
63
56
|
* Loads snapshot records, optionally filtered.
|
|
@@ -157,6 +150,32 @@ export async function harvest(projects) {
|
|
|
157
150
|
/** Formats a percentage, or a dash when there is nothing to divide by. */
|
|
158
151
|
const pct = (n, d) => (d > 0 ? `${Math.round((n / d) * 100)}%` : '—');
|
|
159
152
|
|
|
153
|
+
/**
|
|
154
|
+
* Records grouped by the run they are rounds of, each run's rounds in the order given.
|
|
155
|
+
*
|
|
156
|
+
* @param {object[]} records - Snapshot records, one per round.
|
|
157
|
+
* @returns {Map<string, object[]>}
|
|
158
|
+
*/
|
|
159
|
+
function byRun(records) {
|
|
160
|
+
const runs = new Map();
|
|
161
|
+
for (const r of records) runs.set(runOf(r), [...(runs.get(runOf(r)) ?? []), r]);
|
|
162
|
+
return runs;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** How long a round may go without a report and still be running. The longest measured took 68 minutes. */
|
|
166
|
+
const RUNNING_HOURS = 3;
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Whether a round that has no report is still running, rather than one that never reported or whose report
|
|
170
|
+
* the snapshot could not find. The two looked the same — neither forward, nor sent back, nor unreadable — so
|
|
171
|
+
* a change in Claude Code that hid every report would have left `stats` saying every verdict could be read.
|
|
172
|
+
*
|
|
173
|
+
* @param {object} r - A record.
|
|
174
|
+
* @param {number} now - Milliseconds since the epoch.
|
|
175
|
+
* @returns {boolean}
|
|
176
|
+
*/
|
|
177
|
+
const running = (r, now) => !r.verdict && now - Date.parse(r.ts) <= RUNNING_HOURS * 3_600_000;
|
|
178
|
+
|
|
160
179
|
/**
|
|
161
180
|
* What each stage cost, at API list prices, over the runs whose tokens were recorded. A gate that
|
|
162
181
|
* rarely sends work back is a question of what it costs as much as of what it catches, and until the
|
|
@@ -167,16 +186,19 @@ const pct = (n, d) => (d > 0 ? `${Math.round((n / d) * 100)}%` : '—');
|
|
|
167
186
|
function costReport(records) {
|
|
168
187
|
const measured = records.filter((r) => r.tokens && typeof r.tokens === 'object');
|
|
169
188
|
if (measured.length === 0) return;
|
|
189
|
+
// A run's cost is its rounds', so a reviewer resumed twice is one run at the price of three rounds.
|
|
170
190
|
/** @type {Map<string, number[]>} */
|
|
171
191
|
const byRole = new Map();
|
|
172
192
|
let unpriced = 0;
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
193
|
+
const runs = byRun(measured);
|
|
194
|
+
for (const rounds of runs.values()) {
|
|
195
|
+
const costs = rounds.map((r) => costOf(r.tokens, r.usage_model)).filter((c) => c !== null);
|
|
196
|
+
if (costs.length === 0) {
|
|
176
197
|
unpriced += 1;
|
|
177
198
|
continue;
|
|
178
199
|
}
|
|
179
|
-
|
|
200
|
+
const role = rounds[0].role;
|
|
201
|
+
byRole.set(role, [...(byRole.get(role) ?? []), costs.reduce((a, b) => a + b, 0)]);
|
|
180
202
|
}
|
|
181
203
|
const total = [...byRole.values()].flat().reduce((a, b) => a + b, 0);
|
|
182
204
|
if (total === 0) return;
|
|
@@ -186,7 +208,7 @@ function costReport(records) {
|
|
|
186
208
|
console.log(
|
|
187
209
|
heading(
|
|
188
210
|
'cost',
|
|
189
|
-
`at API list prices of ${PRICES_AS_OF}; ${
|
|
211
|
+
`at API list prices of ${PRICES_AS_OF}; ${runs.size} of ${byRun(records).size} runs have a token record` +
|
|
190
212
|
(unpriced > 0 ? `, ${unpriced} on a model the price table does not know` : ''),
|
|
191
213
|
),
|
|
192
214
|
);
|
|
@@ -269,13 +291,17 @@ function modelReport(records) {
|
|
|
269
291
|
const lines = [];
|
|
270
292
|
for (const [role, models] of changed) {
|
|
271
293
|
const ordered = [...models].sort(([, a], [, b]) => String(a[0].ts).localeCompare(String(b[0].ts)));
|
|
272
|
-
for (const [i, [model,
|
|
273
|
-
const clear =
|
|
294
|
+
for (const [i, [model, rounds]] of ordered.entries()) {
|
|
295
|
+
const clear = rounds.filter((r) => r.verdict && r.verdict !== 'UNCLEAR' && r.verdict !== 'NONE');
|
|
274
296
|
const loops = clear.filter((r) => isLoopBack(r.verdict)).length;
|
|
275
|
-
const
|
|
276
|
-
const
|
|
297
|
+
const runs = byRun(rounds);
|
|
298
|
+
const costs = [...runs.values()]
|
|
299
|
+
.map((rs) => rs.map((r) => (r.tokens ? costOf(r.tokens, r.usage_model) : null)).filter((c) => c !== null))
|
|
300
|
+
.filter((cs) => cs.length > 0)
|
|
301
|
+
.map((cs) => cs.reduce((a, b) => a + b, 0));
|
|
302
|
+
const dates = rounds.map((r) => String(r.ts).slice(0, 10)).sort();
|
|
277
303
|
lines.push(
|
|
278
|
-
` ${i === 0 ? `${pink(role)}${' '.repeat(Math.max(20 - role.length, 1))}` : ''.padEnd(20)}${model.padEnd(26)}${String(runs.
|
|
304
|
+
` ${i === 0 ? `${pink(role)}${' '.repeat(Math.max(20 - role.length, 1))}` : ''.padEnd(20)}${model.padEnd(26)}${String(runs.size).padStart(5)} run(s) ${dim(`${dates[0]} → ${dates.at(-1)}`)}` +
|
|
279
305
|
(clear.length ? ` loop-back ${pct(loops, clear.length)} of ${clear.length}` : ' no readable verdict') +
|
|
280
306
|
(costs.length ? ` median ${money(median(costs))}` : ''),
|
|
281
307
|
);
|
|
@@ -301,7 +327,7 @@ function modelReport(records) {
|
|
|
301
327
|
if (other.length === 0) continue;
|
|
302
328
|
const used = [...new Set(other.map((r) => r.usage_model))].join(', ');
|
|
303
329
|
drift.push(
|
|
304
|
-
` ${basename(dir)}: ${role} declares ${declared}, and ${other.length} of ${since.length}
|
|
330
|
+
` ${basename(dir)}: ${role} declares ${declared}, and ${other.length} of ${since.length} round(s) since that line last changed ran ${used} — an override no spec shows`,
|
|
305
331
|
);
|
|
306
332
|
}
|
|
307
333
|
}
|
|
@@ -422,7 +448,7 @@ function proportionReport(records, stepLimit = () => null) {
|
|
|
422
448
|
const read = typeof largest.tokens?.read === 'number' ? `, ${Math.round(largest.tokens.read / 1e6)}M tokens read from cache` : '';
|
|
423
449
|
console.log(
|
|
424
450
|
note(
|
|
425
|
-
`${over.length} implementer
|
|
451
|
+
`${over.length} implementer round(s) wrote more files than one step may (${stepLimit(largest.project)}); the largest wrote ${largest.files_touched}${read}. The architect splits such a spec into steps.`,
|
|
426
452
|
'warn',
|
|
427
453
|
),
|
|
428
454
|
);
|
|
@@ -537,18 +563,23 @@ export async function stats(argv, ctx) {
|
|
|
537
563
|
const skipped = new Set(all.map((r) => r.project)).size - new Set(records.map((r) => r.project)).size;
|
|
538
564
|
|
|
539
565
|
/**
|
|
540
|
-
*
|
|
541
|
-
*
|
|
566
|
+
* Per stage, counted in rounds: the verdicts are the rounds', and the loop gate counts rounds.
|
|
567
|
+
*
|
|
568
|
+
* @type {Map<string, {n:number, runs:Set<string>, done:number, lost:number, clear:number, loop:number,
|
|
569
|
+
* unclear:number, declared:number, durations:number[]}>}
|
|
542
570
|
*/
|
|
543
571
|
const byRole = new Map();
|
|
544
572
|
// A dispatch a hook denied never ran, so it is neither a run nor a missing verdict.
|
|
545
573
|
const held = records.filter((r) => r.status === 'denied').length;
|
|
574
|
+
const now = Date.now();
|
|
546
575
|
for (const r of records.filter((r) => r.status !== 'denied')) {
|
|
547
576
|
if (!byRole.has(r.role)) {
|
|
548
|
-
byRole.set(r.role, { n: 0, done: 0, clear: 0, loop: 0, unclear: 0, declared: 0, durations: [] });
|
|
577
|
+
byRole.set(r.role, { n: 0, runs: new Set(), done: 0, lost: 0, clear: 0, loop: 0, unclear: 0, declared: 0, durations: [] });
|
|
549
578
|
}
|
|
550
579
|
const s = byRole.get(r.role);
|
|
551
580
|
s.n += 1;
|
|
581
|
+
s.runs.add(runOf(r));
|
|
582
|
+
if (!r.verdict && !running(r, now)) s.lost += 1;
|
|
552
583
|
if (r.verdict) s.done += 1;
|
|
553
584
|
if (r.verdict_source === 'declared' || r.verdict_source === 'handback') s.declared += 1;
|
|
554
585
|
if (r.verdict === 'UNCLEAR' || r.verdict === 'NONE') s.unclear += 1;
|
|
@@ -561,17 +592,18 @@ export async function stats(argv, ctx) {
|
|
|
561
592
|
|
|
562
593
|
const span = [records[0]?.ts, records.at(-1)?.ts].map((t) => String(t).slice(0, 10));
|
|
563
594
|
const projectCount = new Set(records.map((r) => r.project)).size;
|
|
595
|
+
const run = byRun(records.filter((r) => r.status !== 'denied')).size;
|
|
564
596
|
console.log(
|
|
565
|
-
` ${projectCount === 1 ? `${bold(pink(projectName(records[0].project)))} · ` : ''}${bold(records.length)}
|
|
597
|
+
` ${projectCount === 1 ? `${bold(pink(projectName(records[0].project)))} · ` : ''}${bold(run)} runs in ${bold(records.length - held)} rounds · ${projectCount} project${projectCount === 1 ? '' : 's'} · ` +
|
|
566
598
|
`${span[0]} → ${span[1]}` +
|
|
567
599
|
(here ? dim(' (this project; --all for every project)') : '') +
|
|
568
600
|
(skipped > 0 ? dim(` (${skipped} non-harness project${skipped === 1 ? '' : 's'} hidden, --all to include)`) : '') +
|
|
569
601
|
(held > 0 ? ` · ${held} dispatch(es) denied by a hook, not counted as runs` : ''),
|
|
570
602
|
);
|
|
571
|
-
console.log(heading('stages', 'what each ran, and what came of it'));
|
|
603
|
+
console.log(heading('stages', 'what each ran, and what came of it; a round is one report, and a run resumed after it reported has one per report'));
|
|
572
604
|
console.log(
|
|
573
605
|
dim(
|
|
574
|
-
` ${'stage'.padEnd(20)}${'runs'.padStart(6)}${'
|
|
606
|
+
` ${'stage'.padEnd(20)}${'runs'.padStart(6)}${'rounds'.padStart(9)}${'loop-back'.padStart(11)}` +
|
|
575
607
|
`${'rate'.padStart(7)}${'declared'.padStart(10)}${'unreadable'.padStart(12)}` +
|
|
576
608
|
`${'median'.padStart(9)}`,
|
|
577
609
|
),
|
|
@@ -592,24 +624,33 @@ export async function stats(argv, ctx) {
|
|
|
592
624
|
Math.max(1, Math.round((16 * s.n) / busiest)),
|
|
593
625
|
);
|
|
594
626
|
const declared = pct(s.declared, s.done).padStart(10);
|
|
595
|
-
const unread = pct(s.unclear, s.done).padStart(12);
|
|
627
|
+
const unread = pct(s.unclear + s.lost, s.done + s.lost).padStart(12);
|
|
596
628
|
console.log(
|
|
597
|
-
` ${pink(role)}${' '.repeat(Math.max(20 - role.length, 1))}${String(s.
|
|
629
|
+
` ${pink(role)}${' '.repeat(Math.max(20 - role.length, 1))}${String(s.runs.size).padStart(6)}${String(s.n).padStart(9)}` +
|
|
598
630
|
`${zero(String(s.loop).padStart(11), s.loop)}${zero(pct(s.loop, s.clear).padStart(7), s.loop)}` +
|
|
599
|
-
`${s.declared < s.done ? amber(declared) : dim(declared)}${s.unclear > 0 ? amber(unread) : dim(unread)}` +
|
|
631
|
+
`${s.declared < s.done ? amber(declared) : dim(declared)}${s.unclear + s.lost > 0 ? amber(unread) : dim(unread)}` +
|
|
600
632
|
`${medianMin(s.durations).padStart(9)} ${runsBar}`,
|
|
601
633
|
);
|
|
602
634
|
}
|
|
603
635
|
console.log(dim(` ${' '.repeat(86)}█ forward ▓ sent back ░ no verdict read`));
|
|
604
636
|
|
|
605
637
|
const unreadable = rows.reduce((a, [, s]) => a + s.unclear, 0);
|
|
638
|
+
const lost = rows.reduce((a, [, s]) => a + s.lost, 0);
|
|
606
639
|
const done = rows.reduce((a, [, s]) => a + s.done, 0);
|
|
607
640
|
console.log('');
|
|
608
|
-
console.log(
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
)
|
|
641
|
+
if (unreadable === 0 && lost === 0) console.log(note("every finished round's verdict could be read.", 'ok'));
|
|
642
|
+
if (unreadable > 0) {
|
|
643
|
+
console.log(note(`${pct(unreadable, done + lost)} of finished rounds report no machine-readable verdict. Those stages need a verdict token on the report's first line.`, 'warn'));
|
|
644
|
+
}
|
|
645
|
+
if (lost > 0) {
|
|
646
|
+
console.log(
|
|
647
|
+
note(
|
|
648
|
+
`${lost} round(s) that started more than ${RUNNING_HOURS} hours ago have no report at all: a run that never finished, or a report ` +
|
|
649
|
+
"the snapshot could not find — which is how a change in Claude Code's transcripts first shows.",
|
|
650
|
+
'warn',
|
|
651
|
+
),
|
|
652
|
+
);
|
|
653
|
+
}
|
|
613
654
|
|
|
614
655
|
// Whether the stages that send work back say what they send back. Counted only over reports read
|
|
615
656
|
// since the record learned the field, and only over declared loop-backs, which are all it asks of.
|
|
@@ -621,12 +662,14 @@ export async function stats(argv, ctx) {
|
|
|
621
662
|
);
|
|
622
663
|
}
|
|
623
664
|
|
|
624
|
-
|
|
625
|
-
//
|
|
626
|
-
const
|
|
665
|
+
// Only a run whose own transcript could be located says anything about skill use, and a skill a run
|
|
666
|
+
// invoked in one round is in front of it in every round after.
|
|
667
|
+
const skillsOf = [...byRun(records.filter((r) => r.agent_id)).values()].map((rounds) => [...new Set(rounds.flatMap((r) => r.skills ?? []))]);
|
|
668
|
+
const withSkills = skillsOf.filter((skills) => skills.length > 0);
|
|
669
|
+
const observable = skillsOf.length;
|
|
627
670
|
if (observable > 0) {
|
|
628
671
|
const tally = new Map();
|
|
629
|
-
for (const
|
|
672
|
+
for (const skills of withSkills) for (const k of skills) tally.set(k, (tally.get(k) ?? 0) + 1);
|
|
630
673
|
const list = [...tally].sort((a, b) => b[1] - a[1]).map(([k, n]) => `${k} ×${n}`);
|
|
631
674
|
console.log(
|
|
632
675
|
note(
|
|
@@ -652,7 +695,7 @@ export async function stats(argv, ctx) {
|
|
|
652
695
|
`${role}: ${s.loop} loop-back(s) in ${s.clear} readable verdict(s) (${pct(s.loop, s.clear)}),` +
|
|
653
696
|
` against ${elsewhere} across the other gates — check whether it still gates anything.` +
|
|
654
697
|
(s.n - s.clear > 0
|
|
655
|
-
? ` ${s.n - s.clear} further
|
|
698
|
+
? ` ${s.n - s.clear} further round(s) produced no readable verdict at all, so the rate may understate it.`
|
|
656
699
|
: '') +
|
|
657
700
|
// A rate built from verdicts a parser guessed at is not the same claim as one built
|
|
658
701
|
// from verdicts the stage declared, and the difference decides whether it can be acted on.
|
|
@@ -695,7 +738,7 @@ export async function stats(argv, ctx) {
|
|
|
695
738
|
);
|
|
696
739
|
if (newest) {
|
|
697
740
|
const since = records.filter((r) => String(r.ts).slice(0, 10) > newest).length;
|
|
698
|
-
if (since > 0) console.log(note(`${since}
|
|
741
|
+
if (since > 0) console.log(note(`${since} round(s) since the newest pill (${newest}).`));
|
|
699
742
|
}
|
|
700
743
|
}
|
|
701
744
|
for (const name of pills.unresolved) {
|
package/src/commands/upgrade.mjs
CHANGED
|
@@ -436,6 +436,9 @@ export async function upgrade(argv, ctx) {
|
|
|
436
436
|
{
|
|
437
437
|
label: 'composing the harness files',
|
|
438
438
|
run: () => {
|
|
439
|
+
// The files of the project's own that the move replaces are copied above; compose will not write
|
|
440
|
+
// over one, so each goes first, and a rollback puts it back.
|
|
441
|
+
for (const p of occupied) rmSync(join(target, p), { force: true });
|
|
439
442
|
const r = nina(ctx, target, ['compose', '--project', target]);
|
|
440
443
|
const wrote = /composed (\d+) file/.exec(r.out)?.[1];
|
|
441
444
|
if (r.ok) for (const p of removable) rmSync(join(target, p), { force: true });
|
package/src/deep.mjs
CHANGED
|
@@ -17,11 +17,11 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
import { spawnSync } from 'node:child_process';
|
|
20
|
-
import {
|
|
20
|
+
import { existsSync, readdirSync } from 'node:fs';
|
|
21
21
|
import { join } from 'node:path';
|
|
22
|
-
import { createInterface } from 'node:readline';
|
|
23
22
|
import { callsDir, childEnv, forgetProject, runFailure } from './commands/eval.mjs';
|
|
24
23
|
import { isLoopBack, transcriptsOf } from './transcripts.mjs';
|
|
24
|
+
import { readRun } from './agentrun.mjs';
|
|
25
25
|
|
|
26
26
|
/** Reports per map call, and how much of each is sent: the cause is in the first screen of a report. */
|
|
27
27
|
const BATCH = 15;
|
|
@@ -69,34 +69,6 @@ export const REDUCE_SCHEMA = {
|
|
|
69
69
|
required: ['clusters'],
|
|
70
70
|
};
|
|
71
71
|
|
|
72
|
-
/**
|
|
73
|
-
* The report a run handed back last — its final verdict and reasons — from its own transcript: the
|
|
74
|
-
* `SubagentHandback` it called, or, for a run from before Claude Code had that tool, its last message,
|
|
75
|
-
* which was the report then.
|
|
76
|
-
*
|
|
77
|
-
* @param {string} file - A subagent transcript.
|
|
78
|
-
* @returns {Promise<string|null>}
|
|
79
|
-
*/
|
|
80
|
-
async function handback(file) {
|
|
81
|
-
let report = null;
|
|
82
|
-
let last = null;
|
|
83
|
-
const rl = createInterface({ input: createReadStream(file), crlfDelay: Infinity });
|
|
84
|
-
for await (const line of rl) {
|
|
85
|
-
const handed = line.includes('SubagentHandback');
|
|
86
|
-
if (!handed && !line.includes('"type":"assistant"')) continue;
|
|
87
|
-
try {
|
|
88
|
-
const row = JSON.parse(line);
|
|
89
|
-
for (const block of row?.message?.content ?? []) {
|
|
90
|
-
if (block?.type === 'tool_use' && block.name === 'SubagentHandback' && typeof block.input?.message === 'string') report = block.input.message;
|
|
91
|
-
if (row.type === 'assistant' && block?.type === 'text' && block.text.trim()) last = block.text;
|
|
92
|
-
}
|
|
93
|
-
} catch {
|
|
94
|
-
// A torn line loses one message, not the run.
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
return report ?? last;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
72
|
/**
|
|
101
73
|
* The loop-back reports still on disk for a project's records, newest first.
|
|
102
74
|
*
|
|
@@ -121,7 +93,8 @@ export async function loopBackReports(records, projectDir, since) {
|
|
|
121
93
|
let gone = 0;
|
|
122
94
|
for (const r of wanted.sort((a, b) => String(b.ts).localeCompare(String(a.ts)))) {
|
|
123
95
|
const file = files.get(r.agent_id);
|
|
124
|
-
|
|
96
|
+
// The round's own report: a run resumed after it reported holds one per round, and the last is not this one's.
|
|
97
|
+
const text = file ? (await readRun(file, r.round ?? 1))?.report : null;
|
|
125
98
|
if (!text) {
|
|
126
99
|
gone += 1;
|
|
127
100
|
continue;
|
package/src/detectors.mjs
CHANGED
|
@@ -27,6 +27,33 @@ import { CHECK } from './wiring.mjs';
|
|
|
27
27
|
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
28
28
|
import { join } from 'node:path';
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* The hook a detector runs under, in `NINA_HOOK`: `context` for the prompt hook, whose findings reach the
|
|
32
|
+
* model, `stop` for the Stop hook, whose one line reaches only the person, and nothing when run by hand.
|
|
33
|
+
*
|
|
34
|
+
* A finding that happens once — the gate failing, a lesson sent to the harness, an export that failed —
|
|
35
|
+
* used to be spent by whichever run saw it first, and that was usually the Stop hook: the person read one
|
|
36
|
+
* line saying the model would be told before their next message, and the prompt hook then found nothing
|
|
37
|
+
* new to tell it. Such a finding is now spent only by the run that hands it to the model
|
|
38
|
+
* (`handsToModel`), and every run before that reports it.
|
|
39
|
+
*/
|
|
40
|
+
const HOOK = 'NINA_HOOK';
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Whether this run is the one that hands its findings to the model, and so the one that may treat a
|
|
44
|
+
* finding that happens once as told.
|
|
45
|
+
*
|
|
46
|
+
* @returns {boolean}
|
|
47
|
+
*/
|
|
48
|
+
export const handsToModel = () => process.env[HOOK] === 'context';
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Whether this run is the Stop hook's, whose findings reach only the person.
|
|
52
|
+
*
|
|
53
|
+
* @returns {boolean}
|
|
54
|
+
*/
|
|
55
|
+
export const beforeModel = () => process.env[HOOK] === 'stop';
|
|
56
|
+
|
|
30
57
|
/**
|
|
31
58
|
* Runs one detector.
|
|
32
59
|
*
|
|
@@ -37,9 +64,10 @@ import { join } from 'node:path';
|
|
|
37
64
|
* @param {{name: string, declaredBy: string, script?: string, bin?: string, args: string[], ignore?: RegExp}} detector
|
|
38
65
|
* @param {string} root - The project root.
|
|
39
66
|
* @param {Set<string>} declared - The npm scripts the project declares.
|
|
67
|
+
* @param {string} hook - Which hook runs it (`HOOK`), or '' by hand.
|
|
40
68
|
* @returns {{name: string, state: 'ok'|'drift'|'error'|'missing'|'n/a', detail: string}}
|
|
41
69
|
*/
|
|
42
|
-
function runOne(detector, root, declared) {
|
|
70
|
+
function runOne(detector, root, declared, hook) {
|
|
43
71
|
// A detector with no `declaredBy` is one the harness ships to every project — the project has
|
|
44
72
|
// nothing to opt into, and asking it to add an npm script first is how a detector ends up
|
|
45
73
|
// installed and never run.
|
|
@@ -56,9 +84,10 @@ function runOne(detector, root, declared) {
|
|
|
56
84
|
// installed the harness the documented way, vendored, would report the harness's own detector
|
|
57
85
|
// as missing on every turn.
|
|
58
86
|
const local = detector.bin ? join(root, 'node_modules', '.bin', detector.bin) : null;
|
|
87
|
+
const env = { ...process.env, [HOOK]: hook };
|
|
59
88
|
const result = detector.bin
|
|
60
|
-
? spawnSync(local && existsSync(local) ? local : detector.bin, detector.args, { cwd: root, encoding: 'utf8' })
|
|
61
|
-
: spawnSync(process.execPath, [path, ...detector.args], { cwd: root, encoding: 'utf8' });
|
|
89
|
+
? spawnSync(local && existsSync(local) ? local : detector.bin, detector.args, { cwd: root, encoding: 'utf8', env })
|
|
90
|
+
: spawnSync(process.execPath, [path, ...detector.args], { cwd: root, encoding: 'utf8', env });
|
|
62
91
|
|
|
63
92
|
if (result.error) {
|
|
64
93
|
if (result.error.code === 'ENOENT') {
|
|
@@ -289,7 +318,7 @@ function remember(root, session, change) {
|
|
|
289
318
|
export function runDetectors(detectors, options) {
|
|
290
319
|
const { root, hook = false, context = false } = options;
|
|
291
320
|
const declared = declaredIn(root);
|
|
292
|
-
const results = detectors.map((d) => runOne(d, root, declared));
|
|
321
|
+
const results = detectors.map((d) => runOne(d, root, declared, context ? 'context' : hook ? 'stop' : ''));
|
|
293
322
|
const applicable = results.filter((r) => r.state !== 'n/a');
|
|
294
323
|
const drift = results.filter((r) => r.state === 'drift');
|
|
295
324
|
const errored = results.filter((r) => r.state === 'error');
|
package/src/gate.mjs
CHANGED
|
@@ -73,7 +73,7 @@ const DELIVERED = /<(task-notification|agent-message|cross-session-message|teamm
|
|
|
73
73
|
const DELIVERY_HEAD = 300;
|
|
74
74
|
|
|
75
75
|
/** The most of a ledger ever read. The owner's last reply bounds what counts, and a session this long is rare. */
|
|
76
|
-
const LEDGER_TAIL = 4 * 1024 * 1024;
|
|
76
|
+
export const LEDGER_TAIL = 4 * 1024 * 1024;
|
|
77
77
|
|
|
78
78
|
/** The most of a subagent's own transcript read looking for its handback. */
|
|
79
79
|
const AGENT_TAIL = 512 * 1024;
|
package/src/langfuse.mjs
CHANGED
|
@@ -131,6 +131,8 @@ export function spanOf(record, project, context = null) {
|
|
|
131
131
|
if (typeof record.files_touched === 'number') attributes.push(attribute('langfuse.observation.metadata.files_touched', record.files_touched));
|
|
132
132
|
if (record.branch) attributes.push(attribute('langfuse.observation.metadata.branch', record.branch));
|
|
133
133
|
if (record.effort) attributes.push(attribute('langfuse.observation.metadata.effort', record.effort));
|
|
134
|
+
// A round after the first is the same agent resumed after it reported; the trace says which.
|
|
135
|
+
if (typeof record.round === 'number') attributes.push(attribute('langfuse.observation.metadata.round', record.round));
|
|
134
136
|
if (tokens) attributes.push(...usageAttributes(tokens, record.usage_model, record.usage_model));
|
|
135
137
|
if (context) {
|
|
136
138
|
attributes.push(attribute('langfuse.observation.input', context.prompt ?? ''));
|