crewforth 3.1.0-rc.4 → 3.1.0-rc.6

package/kit/studio/server/index.js

@@ -17,18 +17,20 @@ import os from 'node:os';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 
 import { getFleet, measureSpawnCost } from './lib/fleet.js';
-import { projectDir, listSessions, findSession, listProjects, sessionCwd } from './lib/projects.js';
+import { projectDir, listSessions, findSession, listProjects, sessionCwd, agentMetaFiles } from './lib/projects.js';
+import { agentOrigins } from './lib/agent-origin.js';
 import { buildGraph, agentDetail, conversation, STALE_MS } from './lib/graph.js';
 import { cachedUsage } from './lib/usage.js';
+import { readModels, readRecord, readCosts } from './lib/model-outcomes.js';
 
 const USAGE_BATCH = 12;
 import { palette } from './lib/palette.js';
 import { latestVersion, latestVersionCached, kitStatus } from './lib/kit.js';
 import { parsePeers, askAll, ask } from './lib/peers.js';
 import {
-  createSession, getSession, listSessionsOwned, reap, stopAll, ALLOWED_MODES,
+  createSession, getSession, listSessionsOwned, reap, stopAll, dropSpools, ALLOWED_MODES,
 } from './lib/session.js';
-import { decide, pending, alwaysList, revoke } from './lib/permissions.js';
+import { decide, pending, alwaysList, revoke, sweepLeft } from './lib/permissions.js';
 import { open as openTerminal, plan as terminalPlan } from './lib/terminal.js';
 import { gateLog, gateReport, sessionStats, board } from './lib/kit-telemetry.js';
 import { writeState, clearStateSync, findRunning } from './lib/instance.js';
@@ -138,8 +140,45 @@ export function writeAllowed(req) {
   return { ok: true };
 }
 
+/**
+ * The record of outcomes for a project, with each agent's cost where it can be read. Only the sessions written
+ * since the record began are opened, and each is read once and kept until it grows (usage.js).
+ */
+/** The ids of the agents of one session, from the files it left: the only lines of the record a page of it needs. */
+async function agentIdsOf(session) {
+  if (!session?.subagentsDir) return [];
+  return (await agentMetaFiles(session.subagentsDir)).map((m) => path.basename(m).replace(/^agent-/, '').replace(/\.meta\.json$/, ''));
+}
+
+/**
+ * What the classes of a project cost: each agent's tokens at list price, from the transcripts of the project's
+ * sessions that are on this machine. Reading those is the expensive part of the Models view, so it is asked for
+ * apart from the record and less often, and a session whose transcript has not changed is not read again
+ * (`cachedUsage`).
+ */
+async function modelCosts(cwd) {
+  const rec = readRecord(cwd);
+  if (!rec.measured) return { measured: false, reason: rec.reason, costs: {}, sessionsRead: 0 };
+  const per = new Map();
+  const dir = await projectDir(cwd);
+  let read = 0;
+  if (dir) {
+    const since = (rec.firstAt ?? 0) - 3600_000;
+    for (const s of (await listSessions(dir)).filter((x) => x.modifiedAt >= since).slice(0, MODELS_SESSIONS_MAX)) {
+      const u = await cachedUsage(s);
+      read += 1;
+      for (const [id, a] of Object.entries(u.agents ?? {})) per.set(id, a.cost);
+    }
+  }
+  return { measured: true, costs: readCosts(cwd, (id) => (per.has(id) ? per.get(id) : null)), sessionsRead: read };
+}
+const MODELS_SESSIONS_MAX = 60;
+
 const MAX_BODY_BYTES = 1024 * 1024;
 
+/** The session events after which a subscriber is sent the session's summary again. */
+export const STATE_AFTER = new Set(['result', 'exit', 'mode']);
+
 function readBody(req) {
   return new Promise((resolve) => {
     let size = 0;
@@ -205,7 +244,7 @@ async function handle(req, res) {
       const session = await findSession(id);
       if (!session) continue;                 // a session on another machine: not read here, and not guessed
       const u = await cachedUsage(session);
-      usage[id] = { durationMs: u.durationMs, workedMs: u.workedMs, tokens: u.tokens.total, cost: u.cost, unpriced: u.unpriced };
+      usage[id] = { durationMs: u.durationMs, workedMs: u.workedMs, startedAt: u.startedAt, tokens: u.tokens.fresh, cost: u.cost, unpriced: u.unpriced };
     }
     return sendJson(res, 200, { measured: true, usage });
   }
@@ -294,6 +333,24 @@ async function handle(req, res) {
   }
 
   // What Crewforth already measures about itself, reported as Crewforth reports it.
+  // Which model did which class of work in this project, and how it went: Crewforth's own record of outcomes,
+  // and the lines of it that belong to the session on screen. What the classes cost is the next route.
+  if (url.pathname === '/api/models') {
+    const sid = url.searchParams.get('session');
+    const session = sid ? await findSession(sid) : null;
+    const cwd = url.searchParams.get('cwd') || (session ? await sessionCwd(session.file, session.bytes) : null);
+    if (!cwd) return sendJson(res, 200, { measured: false, reason: 'no project named', classes: [], suggestions: [], byAgent: {} });
+    return sendJson(res, 200, readModels(cwd, { agents: await agentIdsOf(session) }));
+  }
+
+  if (url.pathname === '/api/models/cost') {
+    const sid = url.searchParams.get('session');
+    const session = sid ? await findSession(sid) : null;
+    const cwd = url.searchParams.get('cwd') || (session ? await sessionCwd(session.file, session.bytes) : null);
+    if (!cwd) return sendJson(res, 200, { measured: false, reason: 'no project named', costs: {}, sessionsRead: 0 });
+    return sendJson(res, 200, await modelCosts(cwd));
+  }
+
   if (url.pathname === '/api/kit') {
     const cwd = url.searchParams.get('cwd') || process.cwd();
     const sid = url.searchParams.get('session');
@@ -310,6 +367,14 @@ async function handle(req, res) {
     return sendJson(res, 200, palette());
   }
 
+  // Where the agents of a session's project are defined: the project's own files and the user's.
+  if (url.pathname === '/api/agents') {
+    const sid = url.searchParams.get('session');
+    const session = sid ? await findSession(sid) : null;
+    const cwd = session ? await sessionCwd(session.file, session.bytes) : null;
+    return sendJson(res, 200, { measured: true, ...agentOrigins(cwd) });
+  }
+
   if (url.pathname === '/api/sessions') {
     const cwd = url.searchParams.get('cwd') || process.cwd();
     const dir = await projectDir(cwd);
@@ -455,12 +520,12 @@ async function handle(req, res) {
     const body = await readBody(req);
     if (!body) return sendJson(res, 400, { ok: false, reason: 'body was not JSON' });
     const toolUseId = decodeURIComponent(permMatch[2]);
-    const out = decide(s.id, toolUseId, body.verdict);
+    const out = decide(s.id, toolUseId, body.verdict, { answers: body.answers, mode: body.mode });
     if (out.ok) s.noteDecision(toolUseId, body.verdict);
     return sendJson(res, out.ok ? 200 : 400, out);
   }
 
-  const ownedMatch = url.pathname.match(/^\/api\/owned\/([^/]+)(?:\/(message|stop|events))?$/);
+  const ownedMatch = url.pathname.match(/^\/api\/owned\/([^/]+)(?:\/(message|stop|events|mode))?$/);
   if (ownedMatch) {
     const s = getSession(decodeURIComponent(ownedMatch[1]));
     if (!s) return sendJson(res, 404, { ok: false, reason: 'no such owned session' });
@@ -476,6 +541,13 @@ async function handle(req, res) {
 
     if (verb === 'stop') return sendJson(res, 200, { ...s.stop(), session: s.summary() });
 
+    // The viewer chose another mode for the session, from the three a session may be started in.
+    if (verb === 'mode') {
+      const asked = await readBody(req);
+      const out = s.requestMode(asked?.mode);
+      return sendJson(res, out.ok ? 200 : 400, { ...out, session: s.summary() });
+    }
+
     const body = await readBody(req);
     if (!body) return sendJson(res, 400, { ok: false, reason: 'body was not JSON' });
     const sent = s.send(body.text);
@@ -728,7 +800,9 @@ function ownedStream(req, res, url, session) {
 
   const unsubscribe = session.subscribe((ev) => {
     write('event', ev);
-    if (ev.rec?.type === 'result' || ev.rec?.type === 'exit') write('state', session.summary());
+    // The summary is what a pane paints its strip from. It follows what changes it: a turn ending, the session
+    // going, and the mode Claude Code reports. Without the last, the strip waited for the beat below.
+    if (STATE_AFTER.has(ev.rec?.type)) write('state', session.summary());
   }, after);
 
   const beat = setInterval(() => write('state', session.summary()), 15000);
@@ -815,6 +889,10 @@ async function main() {
     process.exit(await selftest());
   }
 
+  // What an earlier server left under the temp directory when it was killed: gone before this one makes its own.
+  // Only a server that is about to run does this; asking for --help or --selftest removes nothing.
+  sweepLeft();
+
   const server = http.createServer((req, res) => {
     handle(req, res).catch((e) => {
       process.stderr.write(`crewforth-studio: unhandled: ${e?.stack ?? e}\n`);
@@ -900,6 +978,7 @@ async function main() {
       // Children outlive their parent unless told otherwise, and a panel that
       // leaks running sessions is worse than one that never started them.
       stopAll();
+      dropSpools();
       server.close(() => process.exit(0));
       setTimeout(() => process.exit(0), 2000).unref();
     });

package/kit/studio/server/lib/agent-origin.js

@@ -0,0 +1,44 @@
+// Where a Claude Code agent is defined, as far as that can be read from disk.
+//
+// Claude Code takes an agent that is not built in from a Markdown file whose frontmatter names it: in the project
+// (`.claude/agents/`), in the user's own directory (`~/.claude/agents/`), or in a plugin. This file reads the first
+// two and nothing else. It keeps no list of built-in types: an agent found in neither place is reported as not
+// found, which is all that is known about it.
+//
+// Not read: a plugin's files (a plugin agent is told by its name, `plugin:agent`, in web/names.js), agents given
+// on the command line, managed settings, and files in a directory under `agents/`.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { PROJECTS_ROOT } from './projects.js';
+
+/** The agents one directory defines: { name: file name }. Null when the directory is not there or cannot be read. */
+export function agentsIn(dir) {
+  let files;
+  try { files = fs.readdirSync(dir); } catch { return null; }
+  const out = {};
+  for (const f of files.sort()) {
+    if (!f.endsWith('.md')) continue;
+    let src;
+    try { src = fs.readFileSync(path.join(dir, f), 'utf8'); } catch { continue; }
+    // Frontmatter only, as the palette reads it: a "name:" in the body is not the agent's name.
+    const fm = src.replace(/\r\n/g, '\n').split(/^---$/m)[1] ?? '';
+    const name = fm.match(/^name:\s*(.+)$/m)?.[1]?.trim().replace(/^["']|["']$/g, '');
+    if (name && out[name] === undefined) out[name] = f;
+  }
+  return out;
+}
+
+/** Claude Code's own directory is the parent of its projects directory; the user's agents sit beside the projects. */
+export const USER_AGENTS_DIR = path.join(path.dirname(PROJECTS_ROOT), 'agents');
+
+/**
+ * @returns { project, user } — each { name: file name }, or null where that directory was not read. A project's
+ *          definition is the one Claude Code uses when both name the same agent, and it is listed in both here.
+ */
+export function agentOrigins(cwd, userDir = USER_AGENTS_DIR) {
+  return {
+    project: cwd ? agentsIn(path.join(cwd, '.claude', 'agents')) : null,
+    user: agentsIn(userDir),
+  };
+}

package/kit/studio/server/lib/graph.js

@@ -19,7 +19,7 @@ import path from 'node:path';
 
 import { readAll, contextFill } from './transcript.js';
 import { agentMetaFiles } from './projects.js';
-import { Usage, timesOf } from './usage.js';
+import { Usage, withTimes } from './usage.js';
 import { detailOf } from './permissions.js';
 
 const SESSION_NODE = 'session';
@@ -60,6 +60,109 @@ function harvestCompletions(text, into, at = null) {
   }
 }
 
+// The line the model gate (kit/hooks/guard-agent-model.sh) refuses a call with starts with this, whatever follows.
+const MODEL_GATE = 'GUARD (agent model):';
+
+/**
+ * What the model gate said when it refused a call, or null when the text is not its refusal.
+ * `floor` is the model it says the agent runs on "or above", when it says one; nothing is read into a sentence
+ * that does not say it.
+ */
+export function refusalOf(text) {
+  const at = typeof text === 'string' ? text.indexOf(MODEL_GATE) : -1;
+  if (at === -1) return null;
+  const line = text.slice(at).split('\n')[0].trim();
+  const floor = line.match(/ runs on ([a-z0-9.-]+) or above/);
+  return { text: line.slice(0, 400), floor: floor ? floor[1] : null };
+}
+
+// Which of two models is the larger, for the two questions asked of it: was a call repeated on a higher model,
+// and which is "one model up". A name this does not know has no rank, and nothing is concluded from it.
+const FAMILY = ['haiku', 'sonnet', 'opus', 'fable'];
+export function modelRank(model) {
+  if (typeof model !== 'string') return null;
+  const i = FAMILY.findIndex((f) => model === f || model.includes(`-${f}-`) || model.startsWith(`${f}-`) || model.endsWith(`-${f}`));
+  return i === -1 ? null : i;
+}
+
+// The tool a subagent hands its result back with. The report is the call's `message`; the agent often says
+// nothing after it, so a transcript with a handback and no closing text still holds a report.
+const HANDBACK = 'SubagentHandback';
+
+/**
+ * What an agent reported, from its own transcript, and where that was read from.
+ *
+ *   `closing`  — text the agent wrote after its last handback: its own last word, and it wins;
+ *   `handback` — the message of its last handback call, when nothing was written after it;
+ *   `text`     — with no handback at all, the last text it wrote (what came before is narration).
+ *
+ * Text written BEFORE a handback is narration on the way to it and is never the report.
+ * @returns { text, from } — text is null when the transcript holds none of the three
+ */
+export function reportIn(records) {
+  let lastText = null;       // the last text block of all
+  let afterHandback = null;  // the last text block since the last handback
+  let handback = null;
+  for (const r of records ?? []) {
+    if (r?.type !== 'assistant' || !Array.isArray(r.message?.content)) continue;
+    for (const c of r.message.content) {
+      if (c?.type === 'text' && typeof c.text === 'string' && c.text.trim()) {
+        lastText = c.text;
+        if (handback !== null) afterHandback = c.text;
+      } else if (c?.type === 'tool_use' && c.name === HANDBACK && typeof c.input?.message === 'string' && c.input.message.trim()) {
+        handback = c.input.message;
+        afterHandback = null;
+      }
+    }
+  }
+  if (afterHandback !== null) return { text: afterHandback, from: 'closing' };
+  if (handback !== null) return { text: handback, from: 'handback' };
+  return { text: lastText, from: lastText === null ? null : 'text' };
+}
+
+/** How a report says it ended: the `confidence: high|low` line a crew agent closes with, or null. */
+export function confidenceOf(text) {
+  const m = typeof text === 'string' ? text.trimEnd().match(/(?:^|\n)\W*confidence:\s*(high|low)\W*$/i) : null;
+  return m ? m[1].toLowerCase() : null;
+}
+
+/**
+ * Tie each agent call to what came before it, where the transcript says so.
+ *
+ * `firstTry` — the gate refused a call and the session made the same call again (same agent, same task): the
+ * call that ran carries what was asked first and why it was refused.
+ * Returns nothing; it writes onto the calls.
+ */
+export function linkRetries(calls) {
+  const list = [...calls.values()].sort((a, b) => a.order - b.order);
+  list.forEach((c, i) => {
+    if (!c.refused) return;
+    const again = list.slice(i + 1).find((n) => n.subagentType === c.subagentType && n.description === c.description);
+    if (!again) return;
+    // A call refused twice keeps the first thing that was asked.
+    again.firstTry = c.firstTry ?? { asked: c.model, floor: c.refused.floor, text: c.refused.text };
+  });
+}
+
+/**
+ * `escalatedFrom` / `escalatedTo` — an agent's report closed with `confidence: low`, and the next call to the same
+ * agent after it reported asked for a higher model: the same work, done again one model up. Read from the order
+ * of the calls, the two reports and the two models; where any of the three is missing nothing is linked.
+ */
+export function linkEscalations(nodes) {
+  const agents = nodes.filter((n) => n.kind === 'agent' && n.order != null).sort((a, b) => a.order - b.order);
+  agents.forEach((low, i) => {
+    if (low.confidence !== 'low' || low.reportedAt == null) return;
+    const from = modelRank(low.model ?? low.modelAsked);
+    if (from === null) return;
+    const again = agents.slice(i + 1).find((n) => n.agentType === low.agentType && n.calledAt != null && n.calledAt >= low.reportedAt);
+    const to = again ? modelRank(again.modelAsked ?? again.model) : null;
+    if (to === null || to <= from || again.escalatedFrom) return;
+    again.escalatedFrom = { id: low.id, from: low.model ?? low.modelAsked, to: again.modelAsked ?? again.model };
+    low.escalatedTo = again.id;
+  });
+}
+
 function scanMain(records) {
   const out = {
     cwd: null, gitBranch: null, version: null, model: null,
@@ -110,6 +213,8 @@ function scanMain(records) {
                 order: out.agentCalls.size + 1,
                 wave: out.waves.get(mid),
                 calledAt: at,
+                // The model the call asked for: its own `model` field. Absent, the agent runs on its default.
+                model: typeof c.input?.model === 'string' ? c.input.model : null,
               });
             }
           }
@@ -119,6 +224,12 @@ function scanMain(records) {
     }
 
     if (r?.type === 'user') {
+      // A call the model gate refused: the answer is an error that carries the gate's own line. No agent started.
+      for (const c of Array.isArray(r.message?.content) ? r.message.content : []) {
+        if (c?.type !== 'tool_result' || c.is_error !== true || !out.agentCalls.has(c.tool_use_id)) continue;
+        const said = refusalOf(textOf(typeof c.content === 'string' ? [{ type: 'text', text: c.content }] : c.content));
+        if (said) out.agentCalls.get(c.tool_use_id).refused = said;
+      }
       out.userTurns += 1;
       harvestCompletions(textOf(r.message?.content), out.completions, at);
     }
@@ -205,11 +316,13 @@ export function openCalls(records) {
 }
 
 /** Roll one agent's own transcript into the numbers its node shows. */
-async function scanAgent(file, usage = null) {
+async function scanAgent(file, usage = null, agentId = null) {
   const { records } = await readAll(file);
-  usage?.add(records);
+  usage?.add(records, agentId ?? 'agents:unnamed');
   const stats = {
     tools: {}, toolCount: 0, lastTool: null, errors: 0,
+    model: null,      // the model its own transcript names: what it actually ran on
+    lastText: null,   // the end of its report, for the line a report closes with
     tokens: null, startedAt: null, endedAt: null, turns: 0,
     owns: [], // tool_use ids this agent emitted — how nesting is resolved
   };
@@ -224,6 +337,8 @@ async function scanAgent(file, usage = null) {
     }
     if (r?.type === 'assistant') {
       stats.turns += 1;
+      const mdl = r.message?.model;
+      if (typeof mdl === 'string' && mdl && mdl !== '<synthetic>') stats.model = mdl;
       const u = r.message?.usage;
       if (u && u.cache_read_input_tokens != null) {
         stats.tokens = (u.input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0);
@@ -251,6 +366,8 @@ async function scanAgent(file, usage = null) {
   stats.durationMs = stats.startedAt !== null && stats.endedAt !== null
     ? stats.endedAt - stats.startedAt
     : null;
+  // The report, wherever the agent put it: the confidence line is its last line, handback included.
+  stats.lastText = reportIn(records).text?.slice(-400) ?? null;
   stats.midTurn = midTurn(records);
   stats.open = openCalls(records);
   return stats;
@@ -274,7 +391,7 @@ async function readAgentDir(subagentsDir, usage = null) {
     // `stats` needs the transcript read; a missing file is a normal state (the
     // agent has not written yet), so absence is a null rather than a throw.
     let stats = null;
-    if (mtime !== null) stats = await scanAgent(jsonl, usage);
+    if (mtime !== null) stats = await scanAgent(jsonl, usage, agentId);
 
     out.push({
       agentId,
@@ -310,6 +427,7 @@ export async function buildGraph(session, { staleMs = STALE_MS } = {}) {
   const { records, malformed } = await readAll(session.file);
   const main = scanMain(records);
   // What the session spent, its agents' tokens included and each response counted once.
+  linkRetries(main.agentCalls);
   const usage = new Usage().add(records);
   const agents = await readAgentDir(session.subagentsDir, usage);
   const now = Date.now();
@@ -341,7 +459,7 @@ export async function buildGraph(session, { staleMs = STALE_MS } = {}) {
     tokens: contextFill(records),
     startedAt: main.startedAt,
     endedAt: main.updatedAt,
-    usage: { ...usage.result(), ...timesOf(records) },
+    usage: withTimes(usage, records),
     // Commands sent to the background that no notice has ended, oldest first. A session that is over has none
     // running whatever this says: the page shows them only while the session is live.
     backgroundCommands: [...main.background]
@@ -394,7 +512,16 @@ export async function buildGraph(session, { staleMs = STALE_MS } = {}) {
       spawnDepth: a.spawnDepth,
       workflow: a.workflow,
       toolUseId: a.toolUseId,
-      model: link?.model ?? null,
+      // The model it ran on: what its own transcript names, and failing that what the launch answer resolved.
+      model: a.stats?.model ?? link?.model ?? null,
+      // The model the call asked for; null when the call named none and the agent ran on its default.
+      modelAsked: call?.model ?? null,
+      // Set when the model gate refused this call the first time it was made: what was asked then, and why not.
+      firstTry: call?.firstTry ?? null,
+      // `confidence: high|low`, the line a crew agent's report closes with; null when it has none.
+      confidence: confidenceOf(a.stats?.lastText ?? null),
+      // Its own share of the session's tokens, and what they cost at list price.
+      usage: usage.of(a.agentId),
       launchStatus: link?.status ?? null,
       // Started in the background, and its transcript stops in the middle of a turn. Such an agent can be quiet
       // for a long time inside one tool call; whether it is still working is the live session's to say, so the
@@ -426,6 +553,8 @@ export async function buildGraph(session, { staleMs = STALE_MS } = {}) {
     edges.push({ id: `${parentId}->${a.agentId}`, source: parentId, target: a.agentId, kind: 'spawn' });
   }
 
+  linkEscalations(nodes);
+
   // A workflow run is a real container, not a rendering trick: its agents were
   // spawned by one orchestration script, not by the session directly. Hanging
   // 243 of them straight off the root produced a graph nobody could read, and
@@ -598,9 +727,10 @@ export async function agentDetail(session, agentId) {
     }
   }
 
-  // The report is the last text block the agent produced. Earlier blocks are
-  // narration between tool calls, which is progress rather than conclusion.
-  const report = texts.length ? texts[texts.length - 1].text : null;
+  // The report: what the agent wrote last, or what it handed back when it wrote nothing after that. Earlier
+  // blocks are narration between tool calls, which is progress rather than conclusion.
+  const reported = reportIn(records);
+  const report = reported.text;
 
   return {
     agentId,
@@ -611,9 +741,11 @@ export async function agentDetail(session, agentId) {
     spawnDepth: meta.spawnDepth ?? null,
     prompt,
     report,
+    // Where the report was read from: 'closing', 'handback' or 'text' (see reportIn); null when there is none.
+    reportFrom: reported.from,
     // Kept apart so the UI can show progress for an agent that has not
     // reported yet, without pretending the narration is a conclusion.
-    narration: texts.slice(0, -1).map((t) => t.text),
+    narration: texts.filter((t) => t.text !== report).map((t) => t.text),
     thinking: lastThinking,
     timeline,
     lastError,