@xhulz/nina 0.33.0 → 0.34.0

package/src/commands/langfuse.mjs

@@ -2,14 +2,15 @@
  * `nina langfuse` — the keys, set once, and which projects send their runs to Langfuse on their own.
  *
  *   nina langfuse login [--host <url>]            ask for the keys, check them against Langfuse, keep them
- *   nina langfuse on [--project <dir>] [--content]   send this project's runs after every turn
+ *   nina langfuse on [--project <dir>] [--content | --prompts]   send this project's runs after every turn
  *   nina langfuse off [--project <dir>]
  *   nina langfuse status
  *
  * Once a project is on, nobody runs an export: the `lessons` detector every composed project runs
  * already snapshots the project each turn, and it starts the export in the background whenever a run
  * is ready to go. What it sends is metadata unless the project is turned on with `--content`, which adds
- * each stage's own context — see `src/langfuse.mjs`.
+ * each stage's own context, or with `--prompts`, which adds only what passed between the agents: the
+ * prompt each stage was given and the report it handed back — see `src/langfuse.mjs`.
  *
  * The keys live in `~/.nina/exports/langfuse.json`, readable by the owner only, beside the record of what
  * was sent; `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` and `LANGFUSE_HOST` override them. They are asked
@@ -21,20 +22,26 @@
 import { chmod, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
 import { createInterface } from 'node:readline';
-import { DEFAULT_HOST, projectOf } from '../langfuse.mjs';
+import { DEFAULT_HOST, contextMode, projectOf } from '../langfuse.mjs';
 import { exportsDir, slugFor } from '../paths.mjs';
 import { projectName } from './stats.mjs';
 
 /** Where the keys and the switches live. */
 export const configPath = () => join(exportsDir(), 'langfuse.json');
 
+/** What a project sends, in words, by `contextMode`. */
+const LABELS = { full: "with each stage's own context", prompts: "with each stage's prompt and report", none: 'as metadata only' };
+
+/** The same, as `status` lists it. */
+const SHORT = { full: 'with context', prompts: 'with prompts and reports', none: 'metadata only' };
+
 /** Where the background export leaves what it last did, per project. */
 export const statusPath = (slug) => join(exportsDir(), 'langfuse', `${slug}.status.json`);
 
 /**
  * The configuration on disk: keys, host, and the projects that are on.
  *
- * @returns {Promise<{publicKey?: string, secretKey?: string, host?: string, projects: Record<string, {content: boolean, since: string}>}>}
+ * @returns {Promise<{publicKey?: string, secretKey?: string, host?: string, projects: Record<string, {content: boolean|'prompts', since: string}>}>}
  */
 export async function readConfig() {
   try {
@@ -113,7 +120,7 @@ export async function langfuse(argv, ctx = {}) {
     }
     await writeConfig({ ...config, host, publicKey, secretKey });
     console.log(`  ✓ the keys open "${opened.project}" at ${host}; kept in ${configPath()}, readable by you only`);
-    console.log('    next, in each project that should send: `nina langfuse on` (add `--content` for each stage\'s own context)\n');
+    console.log('    next, in each project that should send: `nina langfuse on` (add `--prompts` for what passed between the agents, or `--content` for each stage\'s whole context)\n');
     return 0;
   }
 
@@ -123,13 +130,21 @@ export async function langfuse(argv, ctx = {}) {
       console.error('  no keys yet — `nina langfuse login` first\n');
       return 2;
     }
-    const content = argv.includes('--content');
+    if (argv.includes('--content') && argv.includes('--prompts')) {
+      console.error('  --content already sends the prompts; pass one of the two\n');
+      return 2;
+    }
+    const content = argv.includes('--content') ? true : argv.includes('--prompts') ? 'prompts' : false;
     await writeConfig({ ...config, projects: { ...config.projects, [slug]: { content, since: new Date().toISOString() } } });
-    console.log(`  ✓ ${project} sends its runs to ${target.host} from now on, after every turn, ${content ? "with each stage's own context" : 'as metadata only'}`);
-    if (content) {
+    console.log(`  ✓ ${project} sends its runs to ${target.host} from now on, after every turn, ${LABELS[contextMode({ content }) ?? 'none']}`);
+    if (content === true) {
       console.log("    each stage's prompt, messages, tool calls and report go with it — code and whatever the stages read included;");
       console.log('    obvious secrets are masked, and nothing more is: turn it on only where that code may leave this machine');
     }
+    if (content === 'prompts') {
+      console.log("    each stage's prompt and report go with it, and nothing it read, wrote or called between them;");
+      console.log('    a prompt carries whatever the orchestrator put in it, diffs and code included, and only obvious secrets are masked');
+    }
     console.log(`    the runs from before now stay here; \`nina export --langfuse --project ${slug}\` sends them, once`);
     console.log('    it runs from the `lessons` detector in this project\'s hooks — `nina check` says whether they are wired\n');
     return 0;
@@ -150,12 +165,12 @@ export async function langfuse(argv, ctx = {}) {
     for (const [name, setting] of on) {
       const last = JSON.parse(await readFile(statusPath(name), 'utf8').catch(() => 'null'));
       const said = !last ? 'nothing sent yet' : last.error ? `✗ ${last.at}: ${last.error}` : `last sent ${last.at}: ${last.spans} trace(s), ${last.scores} score(s)`;
-      console.log(`  ${projectName(name)}: ${setting.content ? 'with context' : 'metadata only'}, since ${setting.since.slice(0, 10)} — ${said}`);
+      console.log(`  ${projectName(name)}: ${SHORT[contextMode(setting) ?? 'none']}, since ${setting.since.slice(0, 10)} — ${said}`);
     }
     console.log('');
     return 0;
   }
 
-  console.error('  usage: nina langfuse login [--host <url>] | on [--project <dir>] [--content] | off [--project <dir>] | status\n');
+  console.error('  usage: nina langfuse login [--host <url>] | on [--project <dir>] [--content | --prompts] | off [--project <dir>] | status\n');
   return 2;
 }

package/src/commands/stats.mjs

@@ -224,8 +224,8 @@ function costReport(records) {
 }
 
 /**
- * How each stage's rounds spent their context: turns, tool calls sent per turn, and the context a turn
- * re-read when the round began and at its largest. Every turn is billed the whole context again, so the
+ * How each stage's rounds spent their context: turns, tool calls sent per turn, the context a turn
+ * re-read when the round began and at its largest, and how long the prompt that opened the round was. Every turn is billed the whole context again, so the
  * cost of a round is its turns times its context, and a round that starts where a long one ended pays for
  * all of it on every turn. The first rounds of a run are set beside the later ones, which are resumes.
  *
@@ -236,8 +236,9 @@ function contextReport(records) {
   if (shaped.length === 0) return;
   const median = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)];
   const k = (n) => (typeof n === 'number' ? `${Math.round(n / 1000)}k` : '—');
-  console.log(heading('context', 'every turn re-reads the whole context: a round costs its turns times its size, and a resumed one starts where it ended'));
-  console.log(dim(`    ${'stage'.padEnd(20)}${'rounds'.padEnd(8)}${'n'.padStart(4)}${'turns'.padStart(7)}${'calls/turn'.padStart(12)}${'starts at'.padStart(11)}${'peak'.padStart(8)}`));
+  const chars = (xs) => (xs.length ? median(xs).toLocaleString('en-US') : '—');
+  console.log(heading('context', 'every turn re-reads the whole context: a round costs its turns times its size, and a resumed one starts where it ended; the prompt it was handed is in characters'));
+  console.log(dim(`    ${'stage'.padEnd(20)}${'rounds'.padEnd(8)}${'n'.padStart(4)}${'turns'.padStart(7)}${'calls/turn'.padStart(12)}${'starts at'.padStart(11)}${'peak'.padStart(8)}${'prompt'.padStart(9)}`));
   const byRole = new Map();
   for (const r of shaped) byRole.set(r.role, [...(byRole.get(r.role) ?? []), r]);
   for (const [role, rounds] of [...byRole].sort((a, b) => b[1].reduce((x, r) => x + r.turns, 0) - a[1].reduce((x, r) => x + r.turns, 0))) {
@@ -251,7 +252,8 @@ function contextReport(records) {
       const name = i === 0 || !rounds.some((r) => (r.round ?? 1) === 1) ? pink(role) + ' '.repeat(Math.max(20 - role.length, 1)) : ' '.repeat(20);
       console.log(
         `    ${name}${dim(label.padEnd(8))}${String(group.length).padStart(4)}${String(median(group.map((r) => r.turns))).padStart(7)}` +
-          `${(calls / turns).toFixed(2).padStart(12)}${k(median(group.map((r) => r.context_start ?? 0))).padStart(11)}${k(median(group.map((r) => r.context_peak ?? 0))).padStart(8)}`,
+          `${(calls / turns).toFixed(2).padStart(12)}${k(median(group.map((r) => r.context_start ?? 0))).padStart(11)}${k(median(group.map((r) => r.context_peak ?? 0))).padStart(8)}` +
+          `${chars(group.map((r) => r.prompt_chars).filter((n) => typeof n === 'number')).padStart(9)}`,
       );
     });
   }

package/src/langfuse.mjs

@@ -100,31 +100,47 @@ function usageAttributes(tokens, model, priced) {
   return out;
 }
 
+/**
+ * How much of a stage's context a project sends, from its switch in the Langfuse configuration: all of it
+ * (`--content`), only what passed between the agents (`--prompts`), or none.
+ *
+ * @param {{content?: boolean|string}|undefined} setting - The project's entry, if it is on.
+ * @returns {'full'|'prompts'|null}
+ */
+export function contextMode(setting) {
+  return setting?.content === 'prompts' ? 'prompts' : setting?.content ? 'full' : null;
+}
+
+/** A round's shape, as metadata on its root: how many turns and calls it took, and how large its context and prompt were. */
+const SHAPE = ['turns', 'tool_calls', 'context_start', 'context_peak', 'prompt_chars'];
+
 /**
  * The OTLP span at the root of a dispatch's trace. Without its context, a run that spent tokens is a
  * generation carrying the run's whole usage and estimated cost, and one that did not is a span. With it,
  * the root is the agent — its prompt in, its report out — and the usage is on the messages beneath it,
- * so no sum counts a token twice. Its verdict is the score beside it rather than an attribute on it: a
- * run whose verdict is read only later can still be given one, since that is a first score and not a
- * second span.
+ * so no sum counts a token twice. With the prompt and report alone, nothing is beneath it, so the root is
+ * a generation again, carrying the usage beside them. Its verdict is the score beside it rather than an
+ * attribute on it: a run whose verdict is read only later can still be given one, since that is a first
+ * score and not a second span.
  *
  * @param {object} record - A snapshot record.
  * @param {string} project - The name the project goes by in Langfuse.
  * @param {{prompt: string, report: string}} [context] - What the stage was given and handed back.
+ * @param {{usage?: boolean}} [options] - `usage`: the root carries the usage even with its context.
  * @returns {object|null} Null for a record with no time to place it at.
  */
-export function spanOf(record, project, context = null) {
+export function spanOf(record, project, context = null, { usage = false } = {}) {
   const start = nanos(record.ts);
   if (!start) return null;
   const end = nanos(record.result_ts) ?? start;
-  const tokens = !context && record.tokens && typeof record.tokens === 'object' ? record.tokens : null;
+  const tokens = (!context || usage) && record.tokens && typeof record.tokens === 'object' ? record.tokens : null;
   const attributes = [
     attribute('langfuse.session.id', record.session ?? 'no-session'),
     attribute('langfuse.trace.name', record.role),
     attribute('langfuse.trace.tags', ['nina', project]),
     attribute('langfuse.trace.metadata.project', project),
     attribute('langfuse.trace.metadata.role', record.role),
-    attribute('langfuse.observation.type', context ? 'agent' : tokens ? 'generation' : 'span'),
+    attribute('langfuse.observation.type', tokens ? 'generation' : context ? 'agent' : 'span'),
     attribute('langfuse.observation.metadata.role', record.role),
   ];
   if (typeof record.issues === 'number') attributes.push(attribute('langfuse.observation.metadata.issues', record.issues));
@@ -133,6 +149,7 @@ export function spanOf(record, project, context = null) {
   if (record.effort) attributes.push(attribute('langfuse.observation.metadata.effort', record.effort));
   // A round after the first is the same agent resumed after it reported; the trace says which.
   if (typeof record.round === 'number') attributes.push(attribute('langfuse.observation.metadata.round', record.round));
+  for (const key of SHAPE) if (typeof record[key] === 'number') attributes.push(attribute(`langfuse.observation.metadata.${key}`, record[key]));
   if (tokens) attributes.push(...usageAttributes(tokens, record.usage_model, record.usage_model));
   if (context) {
     attributes.push(attribute('langfuse.observation.input', context.prompt ?? ''));
@@ -144,23 +161,26 @@ export function spanOf(record, project, context = null) {
 /**
  * Everything a dispatch sends: the root, and — when its context goes too — a generation per message it
  * wrote, with that message's own tokens and cost, and an observation per tool it called, with what went
- * in and what came back. Every text is masked (`redact`), cut to a length, and has the owner's home
- * directory written as `~`.
+ * in and what came back. With `prompts`, only what passed between the agents goes: the root, with the
+ * prompt the stage was given and the report it handed back, and none of what it read, wrote or called in
+ * between. Every text is masked (`redact`), cut to a length, and has the owner's home directory written
+ * as `~`.
  *
  * @param {object} record - A snapshot record.
  * @param {string} project - The name the project goes by in Langfuse.
  * @param {object|null} run - From `readRun`, or null to send the record alone.
  * @param {{text?: number, tool?: number}} [limits] - How much of each text goes.
+ * @param {{prompts?: boolean}} [options] - `prompts`: the prompt and the report, and nothing beneath them.
  * @returns {object[]} Spans; empty for a record with no time to place it at.
  */
-export function spansOf(record, project, run, limits = {}) {
+export function spansOf(record, project, run, limits = {}, { prompts = false } = {}) {
   const textChars = limits.text ?? TEXT_CHARS;
   const toolChars = limits.tool ?? TOOL_CHARS;
   const home = homedir();
   const clean = (text, chars) => clip(redact(String(text ?? '')).replaceAll(home, '~'), chars);
-  const root = spanOf(record, project, run ? { prompt: clean(run.prompt, textChars), report: clean(run.report, textChars) } : null);
+  const root = spanOf(record, project, run ? { prompt: clean(run.prompt, textChars), report: clean(run.report, textChars) } : null, { usage: prompts });
   if (!root) return [];
-  if (!run) return [root];
+  if (!run || prompts) return [root];
   const child = (id, name, start, end, attributes) => ({
     traceId: root.traceId,
     spanId: hexId(`${record.dispatch_id}|${id}`, 16),

package/src/transcripts.mjs

@@ -632,7 +632,7 @@ async function attachAgentDetail(projectDir, dispatches) {
       if (
         run.agent_read_bytes === size &&
         !('resumes' in run) &&
-        [...records, ...later].every((r) => settled(r) && 'round' in r && 'lessons_read' in r && 'tokens' in r && 'files_touched' in r && 'effort' in r && 'turns' in r)
+        [...records, ...later].every((r) => settled(r) && 'round' in r && 'lessons_read' in r && 'tokens' in r && 'files_touched' in r && 'effort' in r && 'turns' in r && 'prompt_chars' in r)
       ) {
         continue;
       }
@@ -662,6 +662,8 @@ async function attachAgentDetail(projectDir, dispatches) {
         written: new Set(),
         /** The tool calls the round made, by id: over its turns, how many calls each turn sent at once. */
         calls: new Set(),
+        /** How long the prompt that opened the round was, in characters; never the prompt. */
+        promptChars: null,
       });
       const rounds = [fresh()];
       const roundOf = roundsOf();
@@ -681,6 +683,14 @@ async function attachAgentDetail(projectDir, dispatches) {
           }
         }
         const round = rounds[n - 1];
+        if (round.promptChars === null && line.includes('"type":"user"')) {
+          try {
+            const said = promptOf(JSON.parse(line), n);
+            if (said !== null) round.promptChars = said.length;
+          } catch {
+            // A torn line is not the prompt.
+          }
+        }
         const hasSkill = line.includes('"Skill"');
         const hasHandback = line.includes('SubagentHandback');
         for (const m of line.matchAll(/"type":"tool_use","id":"([^"]+)"/g)) round.calls.add(m[1]);
@@ -770,6 +780,7 @@ async function attachAgentDetail(projectDir, dispatches) {
           record.tool_calls = round.calls.size;
           record.context_start = shape.start;
           record.context_peak = shape.peak;
+          record.prompt_chars = round.promptChars;
           record.usage_model = spent.model;
           record.effort = spent.effort;
           record.files_touched = round.written.size;
@@ -802,6 +813,21 @@ async function attachAgentDetail(projectDir, dispatches) {
   }
 }
 
+/**
+ * What a round was told, from one transcript row, or null when the row is not it. The dispatch's prompt
+ * opens the first round; a later round opens with the message that resumed the run, which Claude Code
+ * marks as meta, while a meta row in the first round is a reminder. The snapshot measures it and the
+ * Langfuse export sends it, so both read it here.
+ *
+ * @param {object} row - A parsed transcript row.
+ * @param {number} round - The round the row is in, from 1.
+ * @returns {string|null}
+ */
+export function promptOf(row, round) {
+  const content = row?.message?.content;
+  return row?.type === 'user' && (round > 1 || !row.isMeta) && typeof content === 'string' ? content : null;
+}
+
 /**
  * How a round spent its context: how many turns it took, and how large the context each turn re-read was
  * when the round began and at its largest. Every turn is billed the whole context again, so a round that