@xhulz/nina 0.32.0 → 0.33.0

package/src/commands/stats.mjs

@@ -223,6 +223,40 @@ function costReport(records) {
   console.log(bold(`    ${'all stages'.padEnd(20)}${String([...byRole.values()].flat().length).padStart(6)}${''.padStart(10)}${money(total).padStart(10)}`));
 }
 
+/**
+ * How each stage's rounds spent their context: turns, tool calls sent per turn, and the context a turn
+ * re-read when the round began and at its largest. Every turn is billed the whole context again, so the
+ * cost of a round is its turns times its context, and a round that starts where a long one ended pays for
+ * all of it on every turn. The first rounds of a run are set beside the later ones, which are resumes.
+ *
+ * @param {object[]} records - The rounds in the window.
+ */
+function contextReport(records) {
+  const shaped = records.filter((r) => typeof r.turns === 'number' && r.turns > 0);
+  if (shaped.length === 0) return;
+  const median = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)];
+  const k = (n) => (typeof n === 'number' ? `${Math.round(n / 1000)}k` : '—');
+  console.log(heading('context', 'every turn re-reads the whole context: a round costs its turns times its size, and a resumed one starts where it ended'));
+  console.log(dim(`    ${'stage'.padEnd(20)}${'rounds'.padEnd(8)}${'n'.padStart(4)}${'turns'.padStart(7)}${'calls/turn'.padStart(12)}${'starts at'.padStart(11)}${'peak'.padStart(8)}`));
+  const byRole = new Map();
+  for (const r of shaped) byRole.set(r.role, [...(byRole.get(r.role) ?? []), r]);
+  for (const [role, rounds] of [...byRole].sort((a, b) => b[1].reduce((x, r) => x + r.turns, 0) - a[1].reduce((x, r) => x + r.turns, 0))) {
+    [
+      ['first', rounds.filter((r) => (r.round ?? 1) === 1)],
+      ['resumed', rounds.filter((r) => (r.round ?? 1) > 1)],
+    ].forEach(([label, group], i) => {
+      if (group.length === 0) return;
+      const turns = group.reduce((a, r) => a + r.turns, 0);
+      const calls = group.reduce((a, r) => a + (r.tool_calls ?? 0), 0);
+      const name = i === 0 || !rounds.some((r) => (r.round ?? 1) === 1) ? pink(role) + ' '.repeat(Math.max(20 - role.length, 1)) : ' '.repeat(20);
+      console.log(
+        `    ${name}${dim(label.padEnd(8))}${String(group.length).padStart(4)}${String(median(group.map((r) => r.turns))).padStart(7)}` +
+          `${(calls / turns).toFixed(2).padStart(12)}${k(median(group.map((r) => r.context_start ?? 0))).padStart(11)}${k(median(group.map((r) => r.context_peak ?? 0))).padStart(8)}`,
+      );
+    });
+  }
+}
+
 /**
  * Whether a model a run used is the one a spec declares: an alias (`opus`) names a family — a whole
  * segment of the id, wherever it sits, so `claude-3-sonnet` is a sonnet — anything else names a model
@@ -722,6 +756,7 @@ export async function stats(argv, ctx) {
   }
 
   costReport(records.filter((r) => r.status !== 'denied'));
+  contextReport(records.filter((r) => r.status !== 'denied'));
   modelReport(records.filter((r) => r.status !== 'denied'));
   proportionReport(records.filter((r) => r.status !== 'denied'), stepLimitOf(ctx));
 

package/src/commands/upgrade.mjs

@@ -37,7 +37,8 @@ const BANNER = /^[\s█╗╔╝║═╚▄▀]*$|harness orchestration ·/;
  * fault — everything else it finds still rolls the move back.
  *
  * @param {string} target - The project.
- * @param {string[]} newSlots - Project slots this move introduces, as `<file> <slot id>`.
+ * @param {string[]} newSlots - Project slots this move introduces, as `<file> <slot id>`, and the
+ *   vocabulary names it introduces, as `{{NAME}}`.
  * @returns {{label: string, args: string[]}[]}
  */
 function validationsFor(target, newSlots = []) {
@@ -242,6 +243,10 @@ export async function upgrade(argv, ctx) {
   // A slot the new release fills itself is not asked of the project: it composes as the release wrote it.
   const defaultedAfter = await defaultedSlots(onto.dir);
   const newSlots = [...slotsAfter].filter((s) => !slotsBefore.has(s) && !filled.has(s) && !defaultedAfter.has(s)).sort();
+  // A name only the new release uses is owed the way a new slot is: it cannot be filled with meaning
+  // before the core that asks for it is pinned, and rolling back for it left no order the move completes in.
+  const newVocab = needed.filter((v) => !vocabBefore.has(v));
+  const awaited = [...newSlots, ...newVocab.map((v) => `{{${v}}}`)];
   const stranded = [...filled].filter((s) => !slotsAfter.has(s)).sort();
 
   console.log(`  ${profile.core} → ${to}\n`);
@@ -357,7 +362,7 @@ export async function upgrade(argv, ctx) {
   // What was already failing is not this upgrade's doing, and rolling back for it would blame
   // the move for a problem the project brought with it. Measured before anything is written,
   // so the only rollback is for something that WAS passing and now is not.
-  const VALIDATIONS = validationsFor(target, newSlots);
+  const VALIDATIONS = validationsFor(target, awaited);
   const DETECTORS = "running the project's own detectors";
   const hasDetectors = await declares(target, 'harness:check');
 
@@ -416,7 +421,7 @@ export async function upgrade(argv, ctx) {
   // `harness:check` — an npm script whose detector list this command does not own and cannot
   // pass an argument to. The environment is what crosses that boundary, and setting it once
   // means a reporter added to the chain later is told without anyone remembering to tell it.
-  if (newSlots.length > 0) process.env[EXPECT_ENV] = newSlots.join(',');
+  if (awaited.length > 0) process.env[EXPECT_ENV] = awaited.join(',');
 
   // Compose writes and never deletes. So the files the new version stops composing are removed right
   // after it composes — before anything is verified, so what is verified is what stays — unless they
@@ -516,6 +521,10 @@ export async function upgrade(argv, ctx) {
       console.log(`\n  ${newSlots.length} project slot(s) still to fill — only this project can write them:`);
       for (const s of newSlots) console.log(`      ${s}`);
     }
+    if (newVocab.length > 0) {
+      console.log(`\n  ${newVocab.length} vocabulary name(s) ${to} asks for and this profile does not answer — fill each, or defer it with why:`);
+      console.log(`      ${newVocab.join(', ')}`);
+    }
     console.log('');
     return 0;
   }

package/src/expected.mjs

@@ -1,8 +1,9 @@
 /**
- * The project slots a caller has declared it already knows are open.
+ * The project slots, and vocabulary names, a caller has declared it already knows are open.
  *
  * `nina upgrade --apply` creates slots: a new core can introduce one, and it cannot be filled
- * before the core that introduces it is pinned. Every reporter that fails on an unfilled slot
+ * before the core that introduces it is pinned. A name only the new core uses is the same case,
+ * named as `{{NAME}}`: `check` rolled a move back for one the preview had just listed as needed. Every reporter that fails on an unfilled slot
  * therefore has to be told, or the move deadlocks — there is no order in which it completes.
  *
  * Two reporters say this same thing, and the first fix taught only one: `check` counts a slot
@@ -21,7 +22,7 @@ export const EXPECT_ENV = 'NINA_EXPECT_UNFILLED';
 /**
  * @param {string[]} argv - The command's arguments; `--expect-unfilled <a,b>` names slots.
  * @param {NodeJS.ProcessEnv} [env] - Where to read `NINA_EXPECT_UNFILLED` from.
- * @returns {Set<string>} Slot ids as `<relative path> <slot id>`.
+ * @returns {Set<string>} Slot ids as `<relative path> <slot id>`, names as `{{NAME}}`.
  */
 export function expectedUnfilled(argv = [], env = process.env) {
   const fromArgv = argv.includes('--expect-unfilled') ? argv[argv.indexOf('--expect-unfilled') + 1] ?? '' : '';

package/src/transcripts.mjs

@@ -626,9 +626,13 @@ async function attachAgentDetail(projectDir, dispatches) {
       // 744 of one project's 826 in full on every turn, two seconds before the person saw an answer.
       const size = (await stat(join(dir, file)).catch(() => null))?.size ?? null;
       const settled = (r) => r.handed_back === false || String(r.verdict_source ?? '').startsWith('handback');
+      // `resumes` is written only by a reader from before rounds, which bills the whole run to its first
+      // record. A store both versions wrote to — one project on the old pin, a hook elsewhere on the new —
+      // had its runs marked read by the old one and skipped here, their first round billed twice.
       if (
         run.agent_read_bytes === size &&
-        [...records, ...later].every((r) => settled(r) && 'round' in r && 'lessons_read' in r && 'tokens' in r && 'files_touched' in r && 'effort' in r)
+        !('resumes' in run) &&
+        [...records, ...later].every((r) => settled(r) && 'round' in r && 'lessons_read' in r && 'tokens' in r && 'files_touched' in r && 'effort' in r && 'turns' in r)
       ) {
         continue;
       }
@@ -656,6 +660,8 @@ async function attachAgentDetail(projectDir, dispatches) {
         usage: new Map(),
         /** The files the round wrote through the edit tools. Only their number is kept. */
         written: new Set(),
+        /** The tool calls the round made, by id: over its turns, how many calls each turn sent at once. */
+        calls: new Set(),
       });
       const rounds = [fresh()];
       const roundOf = roundsOf();
@@ -677,6 +683,7 @@ async function attachAgentDetail(projectDir, dispatches) {
         const round = rounds[n - 1];
         const hasSkill = line.includes('"Skill"');
         const hasHandback = line.includes('SubagentHandback');
+        for (const m of line.matchAll(/"type":"tool_use","id":"([^"]+)"/g)) round.calls.add(m[1]);
         if (/"name":"(Edit|Write|MultiEdit|NotebookEdit)"/.test(line)) {
           try {
             for (const block of JSON.parse(line)?.message?.content ?? []) {
@@ -755,9 +762,14 @@ async function attachAgentDetail(projectDir, dispatches) {
           targets = [dispatches.get(id)];
         }
         const spent = tokensOf(round.usage);
+        const shape = contextOf(round.usage);
         targets.forEach((record, j) => {
           // One transcript, one bill: were two records ever to share an agent, the second would count it again.
           record.tokens = j === 0 ? spent.tokens : null;
+          record.turns = shape.turns;
+          record.tool_calls = round.calls.size;
+          record.context_start = shape.start;
+          record.context_peak = shape.peak;
           record.usage_model = spent.model;
           record.effort = spent.effort;
           record.files_touched = round.written.size;
@@ -790,6 +802,20 @@ async function attachAgentDetail(projectDir, dispatches) {
   }
 }
 
+/**
+ * How a round spent its context: how many turns it took, and how large the context each turn re-read was
+ * when the round began and at its largest. Every turn is billed the whole context again, so a round that
+ * begins where a long one ended — a run resumed for a fix — pays for everything before it on each turn.
+ *
+ * @param {Map<string, {usage: object}>} usage - Message id → its final usage, in the order the round sent them.
+ * @returns {{turns: number, start: number|null, peak: number|null}} Context sizes in tokens.
+ */
+export function contextOf(usage) {
+  const n = (v) => (typeof v === 'number' ? v : 0);
+  const sizes = [...usage.values()].map(({ usage: u }) => n(u.input_tokens) + n(u.cache_read_input_tokens) + n(u.cache_creation_input_tokens));
+  return { turns: sizes.length, start: sizes[0] ?? null, peak: sizes.length > 0 ? Math.max(...sizes) : null };
+}
+
 /**
  * What a run spent, from each of its messages' final usage: tokens by kind, and the model and effort
  * level that most of its messages ran at. Metadata — counts, a model id and a level, never a message.