flowviant 0.19.0 → 0.21.0

package/bin/lib/claude.mjs

@@ -77,6 +77,84 @@ export const SINGLE_RESUME =
   'Resume your current task. Call get_blocker_resolution for the blocker you reported; ' +
   'if resolved, apply the human’s answer and finish this one intent, then stop.';
 
+// Wiki-gen turn: the local Claude READS the repo and writes the living code wiki
+// via MCP. It never edits/commits code — the ONLY writes are emit_wiki_node calls.
+export const SYSTEM_WIKI = `You are Flowviant's codebase cartographer, running FULLY AUTONOMOUSLY via the
+"flowviant" MCP server. There is NO interactive user and NO terminal. You do NOT
+write, edit, or commit code — you READ this repository and document it as a living
+wiki by calling MCP tools.
+
+Goal: map the WHOLE codebase into a graph of wiki nodes an engineer new to the
+project could read to understand it. Explore the REAL files (Read, Grep, Glob, ls,
+git) — never guess. Ground every claim in files you actually read.
+
+Emit each node with emit_wiki_node. Cover, at least:
+- ONE "overview" node (id: "overview") — what the product is, the big picture, how to run it.
+- ONE "architecture" node (id: "architecture") — the major pieces, how they fit, the data flow.
+- "schema" node(s) — the data model (DB tables / core types) when the repo has one.
+- "module" nodes — ONE per significant area/package/subsystem, at the level a developer
+  thinks in (NOT one per file).
+- "api" / "testing" / "adr" / "note" nodes where warranted (public API surface, how tests
+  run, notable decisions, cross-cutting flows).
+
+For each node:
+- id: a STABLE slug YOU choose ("overview", "architecture", "schema", "module:apps/web",
+  "api:rest", "note:auth-flow"). Reuse the SAME id to refine a node.
+- title: human-readable.
+- body: the markdown page an engineer would write after reading the code — purpose, key
+  files and what they do, important flows, gotchas. Link related nodes with [[their-id]].
+- citations: the real repo-relative files the page draws from.
+- filePaths: the files the node covers (module nodes especially).
+- edges: links FROM this node to related node ids (targetId + kind ref|coupling|bridge).
+- groundedAtSha: the commit you were told to ground to.
+
+Judge significance YOURSELF: a big/important area gets its own node; trivial things fold
+into a parent node's body. Do NOT emit a node per file.
+
+When the whole codebase is mapped, call finish_wiki_generation ONCE with keepNodeIds =
+EVERY id you emitted, then output exactly WIKI_DONE on its own line and stop.
+
+Be efficient — this spends the user's Claude quota. Read broadly and sample enough to
+document each area accurately; you needn't read every file. If a tool errors, retry a
+couple of times, then move on — never stall waiting on a human.`;
+
+export const WIKI_KICKOFF = (sha) =>
+  `Map this repository into the living code wiki now. Ground everything to commit ${sha}. ` +
+  `Read the real files, emit a node per significant area with emit_wiki_node, then call ` +
+  `finish_wiki_generation with all your node ids and output WIKI_DONE.`;
+
+// Delivery re-ground turn: a feature just MERGED. Update only the touched wiki
+// nodes + record a persistent feature-history node. INCREMENTAL — never a full
+// rewrite, never finish_wiki_generation (that prunes; this only adds/updates).
+export const SYSTEM_REGROUND = `You are Flowviant's codebase cartographer, running FULLY AUTONOMOUSLY via the
+"flowviant" MCP server. There is NO interactive user and NO terminal. You do NOT
+write, edit, or commit code — a feature just MERGED and you update the living code
+wiki to reflect it, by calling MCP tools.
+
+Steps:
+1. Call list_wiki_nodes to see the current wiki (node ids + the files each covers).
+2. For each existing node whose files OVERLAP the changed files, RE-READ that area's
+   real code and re-emit the node with emit_wiki_node using the SAME id (updating it
+   in place). Touch ONLY nodes the change actually affected — this is incremental.
+   If the change adds a genuinely new area with no node, emit a new one.
+3. Emit ONE feature-history node recording what shipped: id "feature:<short-slug>",
+   kind "note", state "built", title = the feature, body = what it added and why
+   (a durable record), citations = the changed files, edges linking to the code
+   nodes it touched. state "built" makes it permanent — a future full sweep keeps it.
+4. Do NOT call finish_wiki_generation — that is only for a full sweep and would
+   prune. Just emit, then output exactly REGROUND_DONE on its own line and stop.
+
+Ground every claim in files you actually read. Be efficient — look only at the
+changed area, not the whole repo; spend little quota.`;
+
+export const REGROUND_KICKOFF = ({ sha, title, files }) =>
+  `A feature just merged. Re-ground the living wiki for it.\n\n` +
+  `Feature: ${title}\n` +
+  `Grounded commit: ${sha}\n` +
+  `Changed files:\n${files.map((f) => `- ${f}`).join('\n')}\n\n` +
+  `Follow your instructions: list_wiki_nodes, re-emit the touched nodes (same ids), ` +
+  `emit the feature-history node (state "built"), then output REGROUND_DONE.`;
+
 // Unattended (default) skips prompts so the agent never stalls with no terminal;
 // FLOWVIANT_SAFE=1 restricts to a curated toolset instead.
 const PERM = SAFE
@@ -121,13 +199,88 @@ export function mcpConfigFor(token, mcpUrl) {
   return { dir, path: p };
 }
 
+// Shorten an absolute tool path to a repo-relative one for legible output.
+const shortPath = (p, cwd) => {
+  if (typeof p !== 'string') return '';
+  let s = p;
+  if (cwd && s.startsWith(cwd)) s = s.slice(cwd.length).replace(/^\/+/, '');
+  return s;
+};
+
+// Turn one Claude tool_use into a compact activity {kind, label}, or null for
+// tools not worth surfacing. `kind:'read'` is what the file counter counts;
+// an emit_wiki_node flips the phase to "writing". Used by wiki turns to stream
+// exactly which files Claude is touching (daemon console + app cover).
+export function humanizeToolUse(name, input = {}, cwd = '') {
+  switch (name) {
+    case 'Read':
+      return { kind: 'read', label: `read ${shortPath(input.file_path, cwd)}` };
+    case 'Grep':
+      return {
+        kind: 'search',
+        label: `grep ${JSON.stringify(input.pattern ?? '')}${input.path ? ` in ${shortPath(input.path, cwd)}` : ''}`,
+      };
+    case 'Glob':
+      return { kind: 'glob', label: `glob ${input.pattern ?? ''}` };
+    case 'LS':
+      return { kind: 'list', label: `ls ${shortPath(input.path ?? '.', cwd)}` };
+    case 'Bash':
+      return { kind: 'bash', label: `$ ${String(input.command ?? '').replace(/\s+/g, ' ').slice(0, 60)}` };
+    default:
+      if (typeof name !== 'string') return null;
+      if (name.includes('emit_wiki_node')) return { kind: 'write', label: `+ node ${input.id ?? ''}` };
+      if (name.includes('finish_wiki_generation')) return { kind: 'write', label: 'finalize wiki' };
+      if (name.includes('list_wiki_nodes')) return { kind: 'mcp', label: 'list wiki nodes' };
+      return null; // other tools: silent
+  }
+}
+
+// Parse ONE line of `--output-format stream-json` NDJSON. Pulls assistant text
+// into `out` (so sentinel detection still works) and turns each tool_use into a
+// streamed activity line. A line that isn't JSON (a stray warning) is treated as
+// raw text so nothing is lost.
+function handleStreamLine(line, { cwd, emit, onActivity, appendText }) {
+  let ev;
+  try {
+    ev = JSON.parse(line);
+  } catch {
+    appendText(line + '\n');
+    emit(line + '\n');
+    return;
+  }
+  if (ev.type === 'assistant' && Array.isArray(ev.message?.content)) {
+    for (const b of ev.message.content) {
+      if (b.type === 'text' && b.text) appendText(b.text + '\n');
+      else if (b.type === 'tool_use') {
+        const a = humanizeToolUse(b.name, b.input || {}, cwd);
+        if (a) {
+          emit(a.label + '\n');
+          onActivity?.(a);
+        }
+      }
+    }
+  } else if (ev.type === 'result' && typeof ev.result === 'string') {
+    // The final assistant text (carries WIKI_DONE / REGROUND_DONE).
+    appendText(ev.result + '\n');
+  }
+}
+
 // One Claude Code turn. Output is captured (for sentinel detection) and streamed
 // through, line-prefixed with the worker label so a fleet stays legible.
-export function runTurn({ prompt, resume, system, cwd, mcpConfig, label, onSpawn }) {
+//
+// `streamJson` switches to `--output-format stream-json` and parses the event
+// stream: only the humanized tool activity reaches the console (a legible
+// stream of `read …`, `grep …`, `+ node …`), assistant text is folded into the
+// returned string for sentinel detection, and each activity is handed to
+// `onActivity` so the caller can forward progress. Build-agent turns leave it
+// off and keep the raw text passthrough + line sentinels.
+export function runTurn({ prompt, resume, system, cwd, mcpConfig, label, onSpawn, streamJson, onActivity }) {
   return new Promise((resolve) => {
     const args = [];
     if (resume) args.push('--continue');
-    args.push('-p', prompt, '--mcp-config', mcpConfig, '--append-system-prompt', system, ...PERM);
+    args.push('-p', prompt, '--mcp-config', mcpConfig, '--append-system-prompt', system);
+    if (streamJson) args.push('--output-format', 'stream-json', '--verbose');
+    args.push(...PERM);
     // Force the user's Claude Code subscription — never the API. A key exported in
     // the shell would otherwise silently bill every poll-mode turn as raw API
     // usage (same invariant live mode enforces on its SDK session env).
@@ -138,10 +291,47 @@ export function runTurn({ prompt, resume, system, cwd, mcpConfig, label, onSpawn
     onSpawn?.(child);
     let out = '';
     const pfx = label ? `${label} ` : '';
+    const emit = (s) => process.stdout.write(pfx ? s.replace(/\n/g, `\n${pfx}`) : s);
+
+    if (streamJson) {
+      let buf = '';
+      const appendText = (t) => {
+        out += t;
+      };
+      child.stdout.on('data', (d) => {
+        buf += d.toString();
+        let nl;
+        while ((nl = buf.indexOf('\n')) >= 0) {
+          const line = buf.slice(0, nl);
+          buf = buf.slice(nl + 1);
+          if (line.trim()) handleStreamLine(line, { cwd, emit, onActivity, appendText });
+        }
+      });
+      // stderr is not JSON (warnings/errors) — pass through and keep for sentinels.
+      child.stderr.on('data', (d) => {
+        const s = d.toString();
+        out += s;
+        emit(s);
+      });
+      child.on('error', (e) => {
+        if (e.code === 'ENOENT') {
+          console.error("\nerror: 'claude' CLI not found on PATH. Install Claude Code first.");
+          process.exit(1);
+        }
+        console.error(e);
+        resolve(out);
+      });
+      child.on('close', () => {
+        if (buf.trim()) handleStreamLine(buf, { cwd, emit, onActivity, appendText });
+        resolve(out);
+      });
+      return;
+    }
+
     const onChunk = (d) => {
       const s = d.toString();
       out += s;
-      process.stdout.write(pfx ? s.replace(/\n/g, `\n${pfx}`) : s);
+      emit(s);
     };
     child.stdout.on('data', onChunk);
     child.stderr.on('data', onChunk);

package/bin/lib/config.mjs

@@ -4,7 +4,7 @@ import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { homedir } from 'node:os';
 
-export const VERSION = '0.19.0';
+export const VERSION = '0.21.0';
 
 // Credential stored by `flowviant login` (device auth) — the no-token,
 // no-env-var path. An explicit --fleet flag or FLOWVIANT_FLEET env still wins.

package/bin/lib/fleet.mjs

@@ -45,6 +45,10 @@ import {
   SYSTEM_SINGLE,
   SINGLE_KICKOFF,
   SINGLE_RESUME,
+  SYSTEM_WIKI,
+  WIKI_KICKOFF,
+  SYSTEM_REGROUND,
+  REGROUND_KICKOFF,
 } from './claude.mjs';
 import { runLiveWorker } from './live.mjs';
 import { reapOrphanPreviews } from './preview.mjs';
@@ -306,6 +310,12 @@ export async function runFleetDaemon() {
             mergeAttempts.delete(job.id);
             await reportMergeOutcome(MERGE_DONE_URL, { intentId: job.id });
             ok(`${c.cyan('merged')} ${c.dim(`— ${job.title} → ${baseRef}`)}`);
+            // The code just landed — re-ground the living wiki for what shipped
+            // (touched nodes re-read + a persistent feature-history node).
+            // Direct enqueue = immediacy; the server's durable regroundJobs list
+            // (created by merge-done above, cleared by our reground-done report)
+            // is the restart-safe backstop — dedup'd here by groundedIntents.
+            enqueueReground(job.id, job.prUrl, job.title);
           } else if (failedReason) {
             // Report into the thread (server narrates + re-arms the merge
             // button + notifies) — the job disappears from the roster.
@@ -379,6 +389,203 @@ export async function runFleetDaemon() {
     }
   };
 
+  // Living-wiki work runs ONE turn at a time in a dedicated worktree (off the
+  // agents' checkouts), via the emit_wiki_node MCP tool — never code edits. Two
+  // triggers enqueue: a Regenerate click (full SWEEP) and a successful merge
+  // (incremental RE-GROUND → feature-history). One queue + runner serializes
+  // them so they never collide on the worktree. Each turn runs under its own
+  // wiki-scoped credential (minted per task below) — the server allows ONLY the
+  // wiki tools on it, and build agents' worker tokens can't touch the map. Also
+  // means wiki work needs no agent online.
+  const wikiWt = join(baseDir, 'wiki');
+  const REGROUND_DONE_URL = FLEET_URL.replace(/\/agents\/?$/, '/reground-done');
+  const WIKI_TOKEN_URL = FLEET_URL.replace(/\/agents\/?$/, '/wiki-token');
+  const WIKI_PROGRESS_URL = FLEET_URL.replace(/\/agents\/?$/, '/wiki-progress');
+  const wikiQueue = [];
+  let wikiBusy = false;
+  let lastSweepAt = null; // dedup: run each Regenerate request once
+  const groundedIntents = new Set(); // dedup: re-ground each delivery once
+
+  const mintWikiToken = async () => {
+    try {
+      const res = await fetch(WIKI_TOKEN_URL, {
+        method: 'POST',
+        headers: { Authorization: `Bearer ${FLEET_TOKEN}`, 'User-Agent': USER_AGENT },
+        signal: AbortSignal.timeout(30_000),
+      });
+      if (!res.ok) return null;
+      return (await res.json())?.data?.token ?? null;
+    } catch {
+      return null;
+    }
+  };
+
+  // Stream what the wiki turn is doing to the app (the canvas renders the read
+  // phase). Throttled to ~1/s — the FIRST activity of a run and the terminal
+  // `done` frame force-send so the cover appears fast and clears cleanly.
+  let lastProgressAt = 0;
+  const postWikiProgress = async (body, force = false) => {
+    const now = Date.now();
+    if (!force && now - lastProgressAt < 1000) return;
+    lastProgressAt = now;
+    try {
+      await fetch(WIKI_PROGRESS_URL, {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${FLEET_TOKEN}`,
+          'User-Agent': USER_AGENT,
+          'Content-Type': 'application/json',
+        },
+        signal: AbortSignal.timeout(15_000),
+        body: JSON.stringify(body),
+      });
+    } catch {
+      /* best-effort — a dropped frame is harmless, the next one supersedes it */
+    }
+  };
+
+  const enqueueSweep = (job) => {
+    if (!job || job.requestedAt === lastSweepAt) return;
+    lastSweepAt = job.requestedAt;
+    wikiQueue.push({ type: 'sweep' });
+    void drainWiki();
+  };
+  const enqueueReground = (intentId, prUrl, title) => {
+    if (!intentId || groundedIntents.has(intentId)) return;
+    groundedIntents.add(intentId);
+    wikiQueue.push({ type: 'reground', intentId, prUrl, title: title || 'a delivered task' });
+    void drainWiki();
+  };
+
+  // Changed files of a (merged) PR, for the re-ground prompt. Capped so a huge
+  // PR can't blow up the prompt. prUrl was already validated before the merge.
+  const changedFilesForPr = (prUrl) => {
+    try {
+      const out = execFileSync('gh', ['pr', 'view', prUrl, '--json', 'files'], {
+        cwd: repoRoot,
+        encoding: 'utf8',
+        stdio: ['ignore', 'pipe', 'pipe'],
+      });
+      return (JSON.parse(out).files ?? []).map((f) => f.path).filter(Boolean).slice(0, 60);
+    } catch {
+      return [];
+    }
+  };
+
+  async function drainWiki() {
+    if (wikiBusy || wikiQueue.length === 0) return;
+    wikiBusy = true;
+    try {
+      while (wikiQueue.length) {
+        // Fresh wiki-scoped credential per task — dedicated, so no roster
+        // re-mint can rotate it out from under a long sweep.
+        const token = await mintWikiToken();
+        if (!token) {
+          warn('wiki: could not mint the cartographer token — retrying on a later poll');
+          break; // queue intact — the reconcile loop re-drains
+        }
+        const task = wikiQueue.shift();
+        const { dir, path: mcpConfig } = mcpConfigFor(token, mcpUrl);
+        // Live progress for this turn: count the files Claude reads, flip to the
+        // "writing" phase once it starts emitting nodes, and stream each action
+        // to the app (throttled). elapsedSec is on the daemon's own clock.
+        const mode = task.type === 'sweep' ? 'sweep' : 'reground';
+        const startedAt = Date.now();
+        let filesRead = 0;
+        let phase = 'reading';
+        const onActivity = (a) => {
+          if (a.kind === 'read') filesRead++;
+          if (a.kind === 'write') phase = 'writing';
+          void postWikiProgress({
+            mode,
+            phase,
+            activity: a.label,
+            filesRead,
+            elapsedSec: Math.round((Date.now() - startedAt) / 1000),
+          });
+        };
+        try {
+          if (!existsSync(wikiWt)) {
+            try {
+              git(['worktree', 'add', '--detach', wikiWt, baseRef], repoRoot);
+            } catch {
+              git(['worktree', 'prune'], repoRoot);
+              git(['worktree', 'add', '--detach', wikiWt, baseRef], repoRoot);
+            }
+          }
+          resetWorktree(wikiWt, baseRef);
+          let sha = '';
+          try {
+            sha = git(['rev-parse', 'HEAD'], wikiWt);
+          } catch {
+            /* detached/no HEAD — still writes the map, just ungrounded */
+          }
+          if (task.type === 'sweep') {
+            note(`${c.cyan('wiki')} ${c.dim('— regenerating: your Claude is reading the repo…')}`);
+            const out = await runTurn({
+              prompt: WIKI_KICKOFF(sha),
+              resume: false,
+              system: SYSTEM_WIKI,
+              cwd: wikiWt,
+              mcpConfig,
+              label: c.cyan('[wiki]'),
+              streamJson: true,
+              onActivity,
+            });
+            if (sawSentinel(out, 'WIKI_DONE'))
+              ok(`${c.cyan('wiki')} ${c.dim('— regenerated from your code.')}`);
+            else
+              warn('code-wiki regeneration ended without WIKI_DONE — retry from the app if incomplete.');
+          } else {
+            const files = changedFilesForPr(task.prUrl);
+            if (files.length === 0) {
+              note(`${c.cyan('wiki')} ${c.dim(`— "${task.title}": no changed files to re-ground`)}`);
+            } else {
+              note(`${c.cyan('wiki')} ${c.dim(`— re-grounding after "${task.title}"…`)}`);
+              const out = await runTurn({
+                prompt: REGROUND_KICKOFF({ sha, title: task.title, files }),
+                resume: false,
+                system: SYSTEM_REGROUND,
+                cwd: wikiWt,
+                mcpConfig,
+                label: c.cyan('[wiki]'),
+                streamJson: true,
+                onActivity,
+              });
+              if (sawSentinel(out, 'REGROUND_DONE'))
+                ok(`${c.cyan('wiki')} ${c.dim(`— wiki updated for "${task.title}".`)}`);
+              else warn(`wiki re-ground for "${task.title}" ended without REGROUND_DONE.`);
+            }
+            // Consume the durable job: attempted = done (success or not — emits
+            // are idempotent and a failed turn heals on the next full sweep), so
+            // a failing re-ground can't loop-burn quota. Only a crash BEFORE
+            // this line leaves the job listed for a retry after restart.
+            await reportMergeOutcome(REGROUND_DONE_URL, { intentId: task.intentId });
+          }
+        } catch (e) {
+          warn(`wiki ${task.type} failed: ${e.message}`);
+        } finally {
+          // Terminal frame so the app cover clears promptly (don't wait for the
+          // freshness window to lapse). force-sent past the throttle.
+          await postWikiProgress(
+            {
+              mode,
+              phase,
+              activity: '',
+              filesRead,
+              elapsedSec: Math.round((Date.now() - startedAt) / 1000),
+              done: true,
+            },
+            true
+          );
+          rmSync(dir, { recursive: true, force: true });
+        }
+      }
+    } finally {
+      wikiBusy = false;
+    }
+  }
+
   let connected = false; // log the first successful poll once
   let rosterSig = null; // last roster membership, to log changes only
   let idleBeatAt = 0; // throttle the "still alive" idle heartbeat
@@ -511,6 +718,15 @@ export async function runFleetDaemon() {
       }
     }
 
+    // Living-wiki work (runs under its own minted wiki token — no agent
+    // needed). enqueueSweep queues a Regenerate; regroundJobs re-offers merged
+    // deliveries whose re-ground never ran (e.g. we restarted between merge and
+    // turn) until we report reground-done; the bare drain flushes anything
+    // whose earlier mint failed.
+    enqueueSweep(roster.codeMapJob);
+    for (const j of roster.regroundJobs ?? []) enqueueReground(j.intentId, j.prUrl, j.title);
+    void drainWiki();
+
     // Stop workers whose agent left the roster (removed in the app).
     for (const [id, w] of [...workers]) {
       if (!rosterIds.has(id)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flowviant",
-  "version": "0.19.0",
+  "version": "0.21.0",
   "description": "Run your own Claude Code as headless build agents for Flowviant — on your own credentials. Claims dispatched work, opens PRs, captures review evidence, and routes questions back to you.",
   "type": "module",
   "bin": {