accrete-kit 0.1.0 → 0.2.0

package/dist/ui/server/documents.mjs

@@ -0,0 +1,285 @@
+// One scan of a Workspace's managed documents and one set of watchers, shared by
+// the objects and activity readers. Nothing here writes.
+import { createHash } from 'node:crypto';
+import { watch } from 'node:fs';
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { extname, isAbsolute, join, relative } from 'node:path';
+import { resolveWorkspace } from './workspace.mjs';
+
+export const KINDS = ['model', 'iteration', 'tool', 'data', 'result', 'memory'];
+
+const SETTLE_MS = 150; // Quiet time after the last event before rescanning.
+const MAX_WAIT_MS = 1000; // A steady stream of events still gets a rescan this often.
+const ROOT_CHECK_MS = 2000; // Stores appear on first use, and a checkout can replace them.
+
+export const revisionOf = (bytes) => createHash('sha256').update(bytes).digest('hex');
+
+function scalar(text) {
+  if (text.startsWith('"')) {
+    try {
+      return JSON.parse(text);
+    } catch {
+      return text;
+    }
+  }
+  if (text.startsWith("'") && text.endsWith("'")) return text.slice(1, -1).replaceAll("''", "'");
+  return text;
+}
+
+// The Kernel's test for managed Memory: a frontmatter whose kind is memory. Only the
+// top-level scalars are read here; links come from the index.
+export function memoryFrontmatter(text) {
+  if (!text.startsWith('---\n')) return null;
+  const end = text.indexOf('\n---\n', 4);
+  if (end < 0) return /^kind: memory$/m.test(text) ? { kind: 'memory', problem: 'frontmatter is not closed' } : null;
+  const frontmatter = text.slice(4, end);
+  if (frontmatter.trimStart().startsWith('{')) {
+    try {
+      const meta = JSON.parse(frontmatter);
+      return meta?.kind === 'memory' ? meta : null;
+    } catch {
+      return /"kind"\s*:\s*"memory"/.test(frontmatter) ? { kind: 'memory', problem: 'frontmatter is not valid JSON' } : null;
+    }
+  }
+  const meta = {};
+  for (const line of frontmatter.split('\n')) {
+    const match = line.match(/^([A-Za-z_]+):[ \t]+(.+?)[ \t]*$/);
+    if (match) meta[match[1]] = scalar(match[2]);
+  }
+  return meta.kind === 'memory' ? meta : null;
+}
+
+function jsonSummary(kind, bytes) {
+  let doc;
+  try {
+    doc = JSON.parse(bytes.toString('utf8'));
+  } catch {
+    return { problem: 'not valid JSON' };
+  }
+  if (doc === null || typeof doc !== 'object' || Array.isArray(doc)) return { problem: 'not a JSON object' };
+  if (doc.kind !== kind) return { ...doc, problem: `kind is ${JSON.stringify(doc.kind ?? null)}, expected ${kind}` };
+  return doc;
+}
+
+const text = (value) => (typeof value === 'string' && value ? value : undefined);
+
+function summarize(kind, path, rel, bytes) {
+  const meta = kind === 'memory' ? memoryFrontmatter(bytes.toString('utf8')) : jsonSummary(kind, bytes);
+  if (!meta) return null; // Ordinary Markdown under memory/.
+  return {
+    kind,
+    id: text(meta.id) ?? rel, // A document with no ID can still be listed and opened.
+    name: text(meta.name) ?? text(meta.title) ?? text(meta.activity),
+    aliases: Array.isArray(meta.aliases) ? meta.aliases.filter(text) : [],
+    status: text(meta.status),
+    path,
+    rel,
+    revision: revisionOf(bytes),
+    created_at: text(meta.created_at),
+    updated_at: text(meta.updated_at),
+    authored_at: text(meta.authored_at),
+    problem: meta.problem ?? (text(meta.id) ? undefined : 'no id'),
+  };
+}
+
+// A temporary sibling (.tmpXXXXXX) has no extension, so it never counts. Symlinks
+// don't count either, as for the Kernel.
+async function listFiles(dir, extension, out = []) {
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch {
+    return out; // Absent on a fresh Workspace, or gone mid-scan.
+  }
+  for (const entry of entries) {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) await listFiles(path, extension, out);
+    else if (entry.isFile() && extname(entry.name) === extension) out.push(path);
+  }
+  return out;
+}
+
+const stampOf = (info) => (info ? `${info.ino}:${info.size}:${info.mtimeMs}` : null);
+
+const stores = new Map();
+
+export function openStore(root) {
+  if (!stores.has(root)) stores.set(root, createStore(root));
+  return stores.get(root);
+}
+
+async function createStore(root) {
+  const workspace = await resolveWorkspace(root);
+  const { metadata, results, memory } = workspace.paths;
+  const stateDir = workspace.local_state?.directory;
+  const indexPath = workspace.local_state?.index;
+  const base = workspace.project_root ?? root;
+  const roots = {
+    model: join(metadata, 'models'),
+    iteration: join(metadata, 'iterations'),
+    tool: join(metadata, 'tools'),
+    data: join(metadata, 'data'),
+    result: results,
+    memory,
+  };
+
+  const relToBase = (path) => {
+    const rel = relative(base, path);
+    return rel.startsWith('..') || isAbsolute(rel) ? path : rel;
+  };
+
+  const store = {
+    root,
+    workspace,
+    docs: new Map(), // path -> summary
+    byId: new Map(),
+    byName: new Map(), // `${kind}:${name or alias}` -> summary, or null when ambiguous
+    receipts: [],
+    relToBase,
+    subscribe: (listener) => listeners.add(listener),
+    close,
+  };
+  const listeners = new Set();
+  const cache = new Map(); // path -> { stamp, doc }, so a rescan only reads what changed
+  let indexStamp = null;
+
+  async function scanDocuments() {
+    const docs = new Map();
+    const seen = new Set();
+    for (const kind of KINDS) {
+      for (const path of await listFiles(roots[kind], kind === 'memory' ? '.md' : '.json')) {
+        const stamp = stampOf(await stat(path).catch(() => null));
+        if (!stamp) continue;
+        let entry = cache.get(path);
+        if (entry?.stamp !== stamp) {
+          const bytes = await readFile(path).catch(() => null);
+          if (!bytes) continue;
+          entry = { stamp, doc: summarize(kind, path, relToBase(path), bytes) };
+          cache.set(path, entry);
+        }
+        seen.add(path);
+        if (entry.doc) docs.set(path, entry.doc);
+      }
+    }
+    for (const path of cache.keys()) if (!seen.has(path)) cache.delete(path);
+    return docs;
+  }
+
+  // Short-lived markers of a Kernel operation: publication receipts, and the Data and
+  // worktree operations that `accrete recover` also reads.
+  async function scanReceipts() {
+    if (!stateDir) return [];
+    const files = [];
+    for (const dir of ['receipts', 'operations']) await listFiles(join(stateDir, dir), '.json', files);
+    return files.map((path) => relative(stateDir, path)).sort();
+  }
+
+  function setDocs(docs) {
+    store.docs = docs;
+    store.byId = new Map();
+    store.byName = new Map();
+    for (const doc of docs.values()) {
+      store.byId.set(doc.id, doc);
+      for (const name of [doc.name, ...doc.aliases]) {
+        if (!name) continue;
+        const key = `${doc.kind}:${name}`;
+        store.byName.set(key, store.byName.has(key) ? null : doc);
+      }
+    }
+  }
+
+  async function rescan() {
+    const docs = await scanDocuments();
+    const receipts = await scanReceipts();
+    const stamp = indexPath ? stampOf(await stat(indexPath).catch(() => null)) : null;
+
+    const change = { added: [], changed: [], removed: [] };
+    for (const [path, doc] of docs) {
+      const before = store.docs.get(path);
+      if (!before) change.added.push(doc);
+      else if (before.revision !== doc.revision) change.changed.push({ before, after: doc });
+    }
+    for (const [path, doc] of store.docs) if (!docs.has(path)) change.removed.push(doc);
+    change.receiptsAdded = receipts.filter((name) => !store.receipts.includes(name));
+    change.receiptsRemoved = store.receipts.filter((name) => !receipts.includes(name));
+    change.indexChanged = stamp !== indexStamp;
+
+    setDocs(docs);
+    store.receipts = receipts;
+    indexStamp = stamp;
+
+    const documentsMoved = change.added.length || change.changed.length || change.removed.length;
+    const receiptsMoved = change.receiptsAdded.length || change.receiptsRemoved.length;
+    if (!documentsMoved && !receiptsMoved && !change.indexChanged) return;
+    await Promise.all([...listeners].map((listener) => listener(change)));
+  }
+
+  // Rescans never overlap: a change seen during one runs another straight after.
+  let timer = null;
+  let deadline = 0;
+  let running = false;
+  let again = false;
+
+  function schedule() {
+    const now = Date.now();
+    if (!timer) deadline = now + MAX_WAIT_MS;
+    clearTimeout(timer);
+    timer = setTimeout(run, Math.max(0, Math.min(SETTLE_MS, deadline - now)));
+    timer.unref();
+  }
+
+  async function run() {
+    timer = null;
+    if (running) {
+      again = true;
+      return;
+    }
+    running = true;
+    do {
+      again = false;
+      await rescan().catch((err) => console.error(`accrete-ui: document scan failed: ${err.message}`));
+    } while (again);
+    running = false;
+  }
+
+  // Watch whichever roots exist now, and re-attach when one appears or is replaced.
+  // Watching the whole local state directory takes in receipts/, operations/ and the index.
+  const watchers = new Map(); // dir -> { ino, watcher }
+  const watched = [metadata, results, memory, stateDir].filter(Boolean);
+
+  async function syncWatchers() {
+    for (const dir of watched) {
+      const info = await stat(dir).catch(() => null);
+      const current = watchers.get(dir);
+      if (current && current.ino === info?.ino) continue;
+      current?.watcher.close();
+      watchers.delete(dir);
+      if (current) schedule();
+      if (!info) continue;
+      const watcher = watch(dir, { recursive: true, persistent: false }, schedule);
+      watcher.on('error', () => {
+        watcher.close();
+        if (watchers.get(dir)?.watcher === watcher) watchers.delete(dir);
+      });
+      watchers.set(dir, { ino: info.ino, watcher });
+      schedule();
+    }
+  }
+
+  const rootCheck = setInterval(syncWatchers, ROOT_CHECK_MS);
+  rootCheck.unref();
+
+  function close() {
+    clearInterval(rootCheck);
+    clearTimeout(timer);
+    timer = null;
+    for (const { watcher } of watchers.values()) watcher.close();
+    watchers.clear();
+    listeners.clear();
+    stores.delete(root);
+  }
+
+  await rescan();
+  await syncWatchers();
+  return store;
+}

package/dist/ui/server/events.mjs

@@ -0,0 +1,18 @@
+// One SSE stream for the whole page. emit(event, data) sends a named event to
+// every open page. The last value of each event name is replayed to a page that
+// connects later, so emit whole state, not deltas.
+const clients = new Set();
+const latest = new Map();
+
+export function emit(event, data) {
+  const frame = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+  latest.set(event, frame);
+  for (const res of clients) res.write(frame);
+}
+
+export function serveEvents(req, res) {
+  res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-store' });
+  for (const frame of latest.values()) res.write(frame);
+  clients.add(res);
+  req.on('close', () => clients.delete(res));
+}

package/dist/ui/server/processes.mjs

@@ -0,0 +1,110 @@
+// The process table and open files, from ps and lsof. macOS has no /proc.
+import { execFile } from 'node:child_process';
+
+// Both exit non-zero when a listed pid has gone, and what they printed is still good.
+function run(command, args) {
+  return new Promise((resolve) => {
+    execFile(command, args, { env: { ...process.env, LC_ALL: 'C' }, maxBuffer: 64 * 1024 * 1024, timeout: 15000 }, (_err, stdout) => {
+      resolve(stdout ?? '');
+    });
+  });
+}
+
+const PS_LINE = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+([\d.]+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\w{3} \w{3}\s+\d+ [\d:]+ \d{4})\s(.*)$/;
+
+export function parsePs(text) {
+  const rows = [];
+  for (const line of text.split('\n')) {
+    const m = PS_LINE.exec(line);
+    if (!m) continue;
+    rows.push({
+      uid: Number(m[1]),
+      pid: Number(m[2]),
+      ppid: Number(m[3]),
+      cpu: Number(m[4]),
+      rss: Number(m[5]) * 1024,
+      state: m[6],
+      tty: m[7],
+      startedAt: Date.parse(m[8]),
+      command: m[9].trim(),
+    });
+  }
+  return rows;
+}
+
+let lastTable = null;
+
+// A caller that does not need the very latest can share the table another poll just read.
+export async function listProcesses({ maxAgeMs = 0 } = {}) {
+  if (lastTable && Date.now() - lastTable.at <= maxAgeMs) return lastTable.rows;
+  const text = await run('ps', ['-axww', '-o', 'uid=,pid=,ppid=,pcpu=,rss=,state=,tty=,lstart=,command=']);
+  lastTable = { at: Date.now(), rows: parsePs(text) };
+  return lastTable.rows;
+}
+
+// lsof writes non-ASCII bytes of a name as \xHH under LC_ALL=C.
+function decodeName(name) {
+  if (!name.includes('\\x')) return name;
+  const bytes = name.replace(/\\x([0-9a-f]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
+  return Buffer.from(bytes, 'latin1').toString('utf8');
+}
+
+// Output of `lsof -F`, one field per line, a p line opening each process and an
+// f line opening each of its files. Returns pid -> [{ fd, access, type, size, name }].
+export function parseLsof(text) {
+  const byPid = new Map();
+  let files = null;
+  let file = null;
+  for (const line of text.split('\n')) {
+    const value = line.slice(1);
+    switch (line[0]) {
+      case 'p':
+        files = [];
+        byPid.set(Number(value), files);
+        break;
+      case 'f':
+        file = { fd: value };
+        files?.push(file);
+        break;
+      case 'a':
+        if (file) file.access = value;
+        break;
+      case 't':
+        if (file) file.type = value;
+        break;
+      case 's':
+        if (file) file.size = Number(value);
+        break;
+      case 'n':
+        if (file) file.name = decodeName(value);
+        break;
+    }
+  }
+  return byPid;
+}
+
+// pid -> working directory, for the given pids only.
+export async function workingDirectories(pids) {
+  const cwds = new Map();
+  if (pids.length === 0) return cwds;
+  const text = await run('lsof', ['-nP', '-a', '-d', 'cwd', '-p', pids.join(','), '-Fpn']);
+  for (const [pid, files] of parseLsof(text)) {
+    if (files[0]?.name) cwds.set(pid, files[0].name);
+  }
+  return cwds;
+}
+
+// pid -> open files, for the given pids only.
+export async function openFiles(pids) {
+  if (pids.length === 0) return new Map();
+  return parseLsof(await run('lsof', ['-nP', '-p', pids.join(','), '-Fpfatsn']));
+}
+
+// Which agent a process is, by its executable. Processes retitle themselves, so
+// `claude` may be all the table shows.
+export function agentOf(command) {
+  const m =
+    /^(?:\S*\/)?(?:node\s+(?:\S*\/)?)?(claude|codex)(?:\s|$)/.exec(command) ??
+    /^(?:\S*\/)?node\s+\S*\/(claude)-code\/cli\.m?js(?:\s|$)/.exec(command);
+  return m ? m[1] : null;
+}

package/dist/ui/server/readers/activity.mjs

@@ -0,0 +1,134 @@
+// Reads the Workspace for the activity panel: a time-ordered feed of what was seen in
+// documents, local state and Git. It says what changed, never who changed it. The feed
+// lives only in this process.
+import { execFile } from 'node:child_process';
+import { openStore } from '../documents.mjs';
+
+const CAP = 500;
+const GIT_POLL_MS = 3000;
+const COMMITS = 50;
+
+const short = (revision) => revision.slice(0, 8);
+
+function git(root, args) {
+  return new Promise((resolve) => {
+    execFile('git', ['--no-optional-locks', '-C', root, ...args], { timeout: 10000, maxBuffer: 2 ** 26 }, (err, stdout) => resolve(err ? null : stdout));
+  });
+}
+
+function parseStatus(text) {
+  const state = { repo: true, branch: null, upstream: null, ahead: 0, behind: 0, dirty: 0 };
+  for (const line of text.split('\n')) {
+    if (!line) continue;
+    if (!line.startsWith('# ')) state.dirty += 1;
+    else if (line.startsWith('# branch.head ')) state.branch = line.slice('# branch.head '.length);
+    else if (line.startsWith('# branch.upstream ')) state.upstream = line.slice('# branch.upstream '.length);
+    else if (line.startsWith('# branch.ab ')) {
+      const [, ahead, behind] = line.match(/\+(\d+) -(\d+)/) ?? [];
+      state.ahead = Number(ahead);
+      state.behind = Number(behind);
+    }
+  }
+  return state;
+}
+
+function parseWorktrees(text) {
+  const worktrees = [];
+  for (const line of (text ?? '').split('\n')) {
+    if (line.startsWith('worktree ')) worktrees.push({ path: line.slice('worktree '.length), branch: null });
+    else if (line.startsWith('branch ')) worktrees.at(-1).branch = line.slice('branch '.length).replace('refs/heads/', '');
+  }
+  return worktrees;
+}
+
+// All local branches, so a commit made in an iteration's worktree shows too.
+async function readCommits(root) {
+  const out = await git(root, ['log', `-n${COMMITS}`, '--branches', 'HEAD', '--format=%H%x00%h%x00%cI%x00%an%x00%s']);
+  if (!out) return [];
+  return out
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => {
+      const [sha, abbreviated, committed_at, author, subject] = line.split('\0');
+      return { sha, abbreviated, committed_at, author, subject };
+    });
+}
+
+function backfill(doc) {
+  const items = [];
+  const add = (field, verb) => {
+    const at = Date.parse(doc[field]);
+    if (!Number.isNaN(at)) items.push({ at, source: 'document', text: `${doc.rel} ${verb} (${field})`, detail: doc.name, open: doc.id });
+  };
+  add('created_at', 'created');
+  add('updated_at', 'updated');
+  add('authored_at', 'authored');
+  return items;
+}
+
+const commitItem = (commit, at, live) => ({
+  at,
+  live,
+  source: 'git',
+  text: `commit ${commit.abbreviated} ${commit.subject}`,
+  detail: commit.author,
+  committed_at: commit.committed_at,
+});
+
+function documentItems(change, at) {
+  const item = (doc, text) => ({ at, live: true, source: 'document', text, detail: doc.name, open: doc.id });
+  return [
+    ...change.added.map((doc) => item(doc, `${doc.rel} added, revision ${short(doc.revision)}`)),
+    ...change.changed.map(({ before, after }) => item(after, `${after.rel} changed, revision ${short(before.revision)} to ${short(after.revision)}`)),
+    ...change.removed.map((doc) => item(doc, `${doc.rel} removed, was revision ${short(doc.revision)}`)),
+    ...change.receiptsAdded.map((name) => ({ at, live: true, source: 'receipt', text: `${name} appeared in local state` })),
+    ...change.receiptsRemoved.map((name) => ({ at, live: true, source: 'receipt', text: `${name} gone from local state` })),
+  ];
+}
+
+export async function start({ root, emit }) {
+  const store = await openStore(root);
+  let feed = [];
+  let gitState = null;
+  const seenCommits = new Set();
+
+  const add = (items) => {
+    feed = [...feed, ...items].sort((a, b) => b.at - a.at).slice(0, CAP);
+  };
+  const publish = () => emit('activity', { feed, git: gitState, receipts: store.receipts });
+
+  add([...store.docs.values()].flatMap(backfill));
+  store.subscribe((change) => {
+    const items = documentItems(change, Date.now());
+    if (!items.length) return;
+    add(items);
+    publish();
+  });
+
+  // A commit from the first look is history, dated by Git. A later one is placed when
+  // it was seen, since a merge or pull can bring in old dates.
+  async function poll(first) {
+    const status = await git(root, ['status', '--porcelain=v2', '--branch']);
+    const next = status === null ? { repo: false } : parseStatus(status);
+    if (next.repo) next.worktrees = parseWorktrees(await git(root, ['worktree', 'list', '--porcelain']));
+
+    const fresh = (next.repo ? await readCommits(root) : []).filter((commit) => !seenCommits.has(commit.sha));
+    for (const commit of fresh) seenCommits.add(commit.sha);
+    const items = fresh.map((commit) => (first ? commitItem(commit, Date.parse(commit.committed_at), false) : commitItem(commit, Date.now(), true)));
+    if (items.length) add(items);
+
+    const gitChanged = JSON.stringify(next) !== JSON.stringify(gitState);
+    gitState = next;
+    if (first || items.length || gitChanged) publish();
+  }
+
+  function loop() {
+    setTimeout(async () => {
+      await poll(false).catch((err) => console.error(`accrete-ui: git poll failed: ${err.message}`));
+      loop();
+    }, GIT_POLL_MS).unref();
+  }
+
+  await poll(true);
+  loop();
+}

package/dist/ui/server/readers/agents.mjs

@@ -0,0 +1,45 @@
+// Agent sessions for this Workspace, from transcripts, session files and the process table.
+import { homedir } from 'node:os';
+import { createSessionIndex } from '../agent-sessions.mjs';
+import { agentOf, listProcesses, openFiles, workingDirectories } from '../processes.mjs';
+import { readScope } from '../scope.mjs';
+
+const POLL_MS = 5000;
+const SCOPE_MS = 30000;
+const PS_SHARE_MS = 4500; // the running panel reads the same table every 4 s
+
+export async function snapshot(index, scope) {
+  const table = await listProcesses({ maxAgeMs: PS_SHARE_MS });
+  const agents = table.filter((row) => agentOf(row.command));
+  const [cwds, open] = await Promise.all([
+    workingDirectories(agents.map((row) => row.pid)),
+    openFiles(agents.filter((row) => agentOf(row.command) === 'codex').map((row) => row.pid)),
+  ]);
+  return { ...(await index.update({ scope, table, cwds, open })), home: homedir() };
+}
+
+export function start({ root, emit }) {
+  emit('agents', { root, scope: [root], sessions: [], unattached: [], loading: true, home: homedir() });
+  const index = createSessionIndex();
+  let scope = [root];
+  let scopeAt = 0;
+  let last = null;
+
+  async function poll() {
+    try {
+      if (Date.now() - scopeAt > SCOPE_MS) {
+        scope = await readScope(root);
+        scopeAt = Date.now();
+      }
+      const state = JSON.stringify(await snapshot(index, scope));
+      if (state !== last) {
+        last = state;
+        emit('agents', JSON.parse(state));
+      }
+    } catch (err) {
+      console.error(`accrete-ui: agents poll failed: ${err.message}`);
+    }
+    setTimeout(poll, POLL_MS).unref();
+  }
+  poll();
+}