@agentpulselabs/cli-plugin-pulse 1.1.3 → 1.1.6

package/src/runner/channelDiscipline.js

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { channelFiles, normRole, parsePosts } from '../hub/lib/channels.mjs';
 import { isTerminal } from '../hub/lib/closure.mjs';
+import { channelFilesAsync, readPostsAsync } from './channelWalk.js';
 
 /**
  * Pulse 1.1.2: the delivery-discipline rules as checks in the runner, from the Cursor hook and a sweep.
@@ -129,6 +130,9 @@ function readPosts(file) {
   } catch { return null; } finally { if (fd !== undefined) fs.closeSync(fd); }
 }
 
+/** What E3 needs of a channel: its newest post, how many posts it has and its mtime; null when it can't be read. */
+const staleState = (read) => (read ? { last: read.posts[read.posts.length - 1] || null, count: read.posts.length, mtime: read.mtime } : null);
+
 const keyOf = (file) => {
   const m = file.match(new RegExp(`(?:^|/)(${NAME})/\\.agents/(${NAME})/channel\\.md$`));
   return m ? `${m[1]}/${m[2]}` : null;
@@ -352,49 +356,79 @@ export function createChannelDiscipline(hub, {
   }
 
   /** E3's owner of a channel's newest post: the session that wrote it, else the agent of that stream and role, else its architect. */
-  function ownerOf(key, last) {
+  function ownerOf(key, last, directory) {
     const wrote = q.author.get(key, last.role, last.head);
     if (wrote) return wrote.session;
-    const dir = registry.directory();
+    const dir = directory();
     const crew = dir.filter((a) => a.streamKey === key && normRole(a.role) === last.role).sort((a, b) => (b.seenAt || 0) - (a.seenAt || 0));
     if (crew.length) return crew[0].id;
     const arch = dir.find((a) => a.isArchitect && (a.streams || []).includes(key));
     return last.role === 'ARCHITECT' && arch ? arch.id : null;
   }
 
-  function architectOf(key, owner) {
-    const dir = registry.directory();
+  function architectOf(key, owner, directory) {
+    const dir = directory();
     const a = owner ? dir.find((d) => d.id === owner) : null;
     if (a && a.deliveryId) return a.deliveryId;
     const arch = dir.find((d) => d.isArchitect && (d.streams || []).includes(key));
     return arch ? arch.deliveryId || arch.id : null;
   }
 
+  /** E3's view of every channel written in the last STALE_WATCH_MS, read now: `[{ key, file, state }]`. */
+  const channelStates = (now) => channelFiles(channelRoots, now - STALE_WATCH_MS).map((c) => ({ key: c.key, file: c.file, state: staleState(readPosts(c.file)) }));
+
+  /** What `scan` last read of each channel, by its mtime and size: an unchanged file is not read again. */
+  const marks = new Map();
+
+  /**
+   * The same view as the sweep's own read, without blocking the event loop or holding a transaction: the walk and the
+   * reads are asynchronous, and only channels whose mtime or size moved since the last scan are read and parsed.
+   */
+  async function scan(now = clock()) {
+    const out = [];
+    const live = new Set();
+    for (const c of await channelFilesAsync(channelRoots, now - STALE_WATCH_MS)) {
+      live.add(c.file);
+      const mark = marks.get(c.file);
+      let state = mark && mark.mtime === c.mtime && mark.size === c.size ? mark.state : undefined;
+      if (state === undefined) {
+        state = staleState(await readPostsAsync(c.file, CHANNEL_MAX_BYTES));
+        if (state) marks.set(c.file, { mtime: c.mtime, size: c.size, state });
+        else marks.delete(c.file);
+      }
+      out.push({ key: c.key, file: c.file, state });
+    }
+    for (const file of marks.keys()) if (!live.has(file)) marks.delete(file);
+    return out;
+  }
+
   /**
    * E3, from the lease holder's sweep: each channel written in the last STALE_WATCH_MS whose newest post is not
    * terminal. Its age counts from when this runner first saw that post (never before the runner started), not from
-   * the header, whose zone and date are unknown. Returns the messages it sent.
+   * the header, whose zone and date are unknown. `seen` is a `scan` taken before the sweep's transaction; without it
+   * the channels are read here. Returns the messages it sent.
    */
-  function sweep(now = clock()) {
+  function sweep({ now = clock(), seen = null } = {}) {
     const sent = [];
     const live = new Set();
-    for (const c of channelFiles(channelRoots, now - STALE_WATCH_MS)) {
-      const read = readPosts(c.file);
-      const last = read && read.posts[read.posts.length - 1];
+    let dir = null;
+    const directory = () => (dir ??= registry.directory());
+    for (const c of seen ?? channelStates(now)) {
+      const last = c.state?.last;
       if (!last) continue;
       live.add(c.key);
       if (isTerminal(last)) { q.staleDrop.run(c.key); continue; }
-      const sig = `${last.role}\0${last.head}\0${read.posts.length}`;
+      const sig = `${last.role}\0${last.head}\0${c.state.count}`;
       let row = q.stale.get(c.key);
       if (!row || row.sig !== sig) {
-        q.staleSet.run(c.key, c.file, sig, Math.max(started, Math.min(now, read.mtime)));
+        q.staleSet.run(c.key, c.file, sig, Math.max(started, Math.min(now, c.state.mtime)));
         row = q.stale.get(c.key);
       }
       const age = now - row.first_seen;
       const quiet = `${Math.floor(age / 60e3)} min`;
       const lastLine = `\`## [${last.role}] … ${last.head.slice(0, 120)}\``;
       if (age >= staleAskMs && !row.asked_at) {
-        const owner = ownerOf(c.key, last);
+        const owner = ownerOf(c.key, last, directory);
         q.staleAsked.run(now, owner, c.key);
         row.owner = owner;
         if (owner) {
@@ -410,7 +444,7 @@ export function createChannelDiscipline(hub, {
       if (age >= staleEscalateMs && !row.escalated_at) {
         q.staleEscalated.run(now, c.key);
         const body = `\`${c.key}\` has had no post for ${quiet}; its last post ${lastLine} is not terminal.${row.owner ? ` STATUS? went to \`${row.owner}\` with no post since.` : ' Pulse found no owner to ask.'}`;
-        const architect = architectOf(c.key, row.owner);
+        const architect = architectOf(c.key, row.owner, directory);
         for (const topic of [...(architect && architect !== row.owner ? [`delivery/${architect}`] : []), 'user']) {
           const env = publish({ from: 'manager', fromName: 'Pulse', topic, type: 'STALE', body, meta: { stale: c.key } });
           if (env && !env.error) sent.push({ key: c.key, to: topic, type: 'STALE' });
@@ -421,5 +455,5 @@ export function createChannelDiscipline(hub, {
     return sent;
   }
 
-  return { onHook, sweep };
+  return { onHook, scan, sweep };
 }

package/src/runner/channelFiles.js

@@ -0,0 +1,39 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { readTaskBrief } from '../hub/lib/brief.mjs';
+import { findAgentQuery } from '../hub/lib/transcripts.mjs';
+
+const BRIEF_CACHE_MS = 30e3;
+const BRIEF_CACHE_MAX = 200;
+const briefCache = new Map();
+
+/** A task channel's file under one of the roots, or null; a path that leaves its root through a link is refused. */
+export function channelFileFor(key, roots) {
+  const m = String(key || '').match(/^([A-Za-z0-9._-]+)\/([A-Za-z0-9._-]+)$/);
+  if (!m || m[1].startsWith('.') || m[2].startsWith('.')) return null;
+  for (const root of roots) {
+    const file = path.join(root, m[1], '.agents', m[2], 'channel.md');
+    try {
+      const st = fs.lstatSync(file);
+      if (!st.isFile()) continue;
+      const realRoot = fs.realpathSync(root);
+      if (fs.realpathSync(file) !== path.join(realRoot, m[1], '.agents', m[2], 'channel.md')) continue;
+      return file;
+    } catch { /* not here */ }
+  }
+  return null;
+}
+
+/** A task's brief: brief.md and the spec, else the longest opening prompt that names the channel (a transcript scan). */
+export function taskBriefFor(key, roots) {
+  const hit = briefCache.get(key);
+  if (hit && Date.now() - hit.at < BRIEF_CACHE_MS) return hit.body;
+  const file = channelFileFor(key, roots);
+  if (!file) return null;
+  const fromFiles = readTaskBrief(file);
+  const thin = !fromFiles || !fromFiles.sections.some((s) => s.body && s.body.trim());
+  const body = thin ? (readTaskBrief(file, { query: findAgentQuery(key) }) || fromFiles || { title: null, source: null, sections: [] }) : fromFiles;
+  if (briefCache.size > BRIEF_CACHE_MAX) briefCache.clear();
+  briefCache.set(key, { at: Date.now(), body });
+  return body;
+}

package/src/runner/channelWalk.js

@@ -0,0 +1,42 @@
+import { constants } from 'node:fs';
+import { open, readdir, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { parsePosts } from '../hub/lib/channels.mjs';
+
+/**
+ * hub/lib/channels.mjs `channelFiles` without blocking the event loop: every roots/<repo>/.agents/<task>/channel.md
+ * changed since sinceMs, in the same order, with its size as well, so a caller can tell an unchanged file by its
+ * mtime and size without reading it.
+ */
+export async function channelFilesAsync(roots, sinceMs) {
+  const out = [];
+  for (const root of roots) {
+    let repos = [];
+    try { repos = await readdir(root); } catch { continue; }
+    for (const repo of repos) {
+      const dir = path.join(root, repo, '.agents');
+      let tasks = [];
+      try { tasks = await readdir(dir); } catch { continue; }
+      const found = await Promise.all(tasks.map(async (task) => {
+        const file = path.join(dir, task, 'channel.md');
+        try {
+          const st = await stat(file);
+          return st.isFile() && st.mtimeMs >= sinceMs ? { key: `${repo}/${task}`, task, repo, file, mtime: st.mtimeMs, size: st.size } : null;
+        } catch { return null; }
+      }));
+      out.push(...found.filter(Boolean));
+    }
+  }
+  return out;
+}
+
+/** A channel's posts, read without following a symlink: `{ posts, mtime }`, or null when it can't be read or is over maxBytes. */
+export async function readPostsAsync(file, maxBytes) {
+  let handle;
+  try {
+    handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW);
+    const st = await handle.stat();
+    if (!st.isFile() || st.size > maxBytes) return null;
+    return { posts: parsePosts(await handle.readFile('utf8'), st.mtimeMs), mtime: st.mtimeMs };
+  } catch { return null; } finally { await handle?.close(); }
+}

package/src/runner/collectWorker.js

@@ -0,0 +1,92 @@
+// The runner's discovery worker: the hub's lib/collect-worker.mjs, except that once a pass is saved a graph read never
+// waits for the next one. A read answers from the rows the last pass saved and, when its epoch is newer or that pass is
+// older than DISCOVERY_REUSE_MS, asks for a new pass, which runs and writes its rows after the reads that came with it.
+// The runner's first read still waits for the first pass. `{ type: 'discovered' }` tells the facade a pass was saved.
+import path from 'node:path';
+import { parentPort } from 'node:worker_threads';
+import { openHub } from '../hub/core/hub.mjs';
+import { hubHome, runnerDb } from '../hub/home.mjs';
+import { collect } from '../hub/lib/collect.mjs';
+import { projectGraph } from '../hub/projections/graph.mjs';
+import { CANONICAL_HOURS, DISCOVERY_REUSE_MS, persistDiscovery } from '../hub/workers/collector.mjs';
+
+/** A pass starts once reads have stopped this long, so a mirror pass's window reads are all answered before it. */
+const SETTLE_MS = 100;
+/** Reads that never stop for SETTLE_MS still let a wanted pass start this soon after it was first wanted. */
+const SETTLE_MAX_MS = 2000;
+
+const hub = openHub(path.join(hubHome(), 'state'), runnerDb());
+let saved = null;
+let wanted = null;
+let timer = null;
+let wantedSince = null;
+
+const discover = (epoch, aliases) => {
+  persistDiscovery(hub, collect({ hours: CANONICAL_HOURS, runs: hub.registry.facts('run'), aliases: aliases || [] }));
+  saved = { epoch, at: Date.now() };
+};
+const stale = (epoch) => !saved || epoch == null || saved.epoch !== epoch || Date.now() - saved.at >= DISCOVERY_REUSE_MS;
+
+const passSoon = () => {
+  clearTimeout(timer);
+  wantedSince ??= Date.now();
+  timer = setTimeout(() => {
+    timer = null;
+    wantedSince = null;
+    const next = wanted;
+    wanted = null;
+    if (!next || !stale(next.epoch)) return;
+    try {
+      discover(next.epoch, next.aliases);
+      parentPort.postMessage({ type: 'discovered' });
+    } catch (e) {
+      console.error(`discovery failed: ${(e && e.stack) || e}`);
+    }
+  }, Math.min(SETTLE_MS, Math.max(0, wantedSince + SETTLE_MAX_MS - Date.now())));
+};
+
+/** Registry reads the windows of one burst all make: read once per burst; the projections copy what they change. */
+const SHARED_READS = ['all', 'get', 'presence', 'presences', 'fact', 'facts', 'directory', 'registrations', 'eventsSince'];
+function burstView() {
+  const memo = new Map();
+  const registry = Object.create(hub.registry);
+  for (const name of SHARED_READS) {
+    const read = hub.registry[name];
+    registry[name] = (...args) => {
+      const key = `${name}\0${JSON.stringify(args)}`;
+      if (!memo.has(key)) memo.set(key, read.apply(hub.registry, args));
+      return memo.get(key);
+    };
+  }
+  return { ...hub, registry };
+}
+
+/**
+ * Reads that queued while the worker was busy (a mirror pass reads every window at once) are projected together, at
+ * one clock and over one set of registry reads.
+ */
+let burst = [];
+function project() {
+  const reads = burst;
+  burst = [];
+  const now = Date.now();
+  let view = saved ? burstView() : null;
+  for (const { id, hours, managerView, aliases, discoveryEpoch } of reads) {
+    try {
+      if (!saved) { discover(discoveryEpoch, aliases); view = burstView(); }
+      else if (stale(discoveryEpoch)) wanted = { epoch: discoveryEpoch, aliases };
+      // As JSON text the body crosses to the main thread as one string, not an object graph it must clone and stringify.
+      parentPort.postMessage({ id, text: JSON.stringify(projectGraph(view, { hours, managerView: managerView || {}, now })) });
+    } catch (e) {
+      parentPort.postMessage({ id, error: String((e && e.stack) || e) });
+    }
+  }
+  // Armed only now: a timer that came due while this burst projected would run before the reads queued behind it.
+  if (wanted) passSoon();
+}
+
+parentPort.on('message', (read) => {
+  burst.push(read);
+  if (burst.length === 1) setImmediate(project);
+  if (timer) passSoon();
+});

package/src/runner/deliveryStates.js

@@ -134,18 +134,32 @@ export function createDeliveryStates(hub, bus, { now: clock = Date.now } = {}) {
    * waited BOUNCE_GRACE_MS for a recipient nobody can read: its sender gets one UNDELIVERABLE notice per recipient,
    * from the manager, with an id derived from the message and recipient. A notice is never itself bounced.
    */
-  function bounce(now = clock()) {
-    const rows = db.prepare(`SELECT d.message_id AS id, d.recipient, m.sender, m.type, m.body FROM deliveries d JOIN messages m ON m.id = d.message_id
+  const BOUNCE_FROM = `FROM deliveries d JOIN messages m ON m.id = d.message_id
       WHERE d.state = 'queued' AND m.expires_at > ? AND m.created_at <= ? AND m.created_at > ?
         AND m.sender <> 'manager' AND m.type <> 'UNDELIVERABLE' AND d.recipient NOT IN ('user', 'manager')
         AND NOT EXISTS (SELECT 1 FROM pulse_reported r WHERE r.message_id = m.id)
-        AND NOT EXISTS (SELECT 1 FROM pulse_bounced b WHERE b.message_id = d.message_id AND b.recipient = d.recipient)
-      ORDER BY m.created_at LIMIT ?`).all(now, now - BOUNCE_GRACE_MS, now - DEFAULT_TTL_MS, BOUNCE_BATCH);
+        AND NOT EXISTS (SELECT 1 FROM pulse_bounced b WHERE b.message_id = d.message_id AND b.recipient = d.recipient)`;
+  const bounceRows = db.prepare(`SELECT d.message_id AS id, d.recipient, m.sender, m.type, m.body ${BOUNCE_FROM} ORDER BY m.created_at LIMIT ?`);
+  const stillBounceable = db.prepare(`SELECT 1 AS found ${BOUNCE_FROM} AND d.message_id = ? AND d.recipient = ?`);
+  const bounceWindow = (now) => [now, now - BOUNCE_GRACE_MS, now - DEFAULT_TTL_MS];
+
+  function bounce(now = clock()) {
     const sent = [];
+    for (const step of bounceSteps({ now, sent })) step();
+    return sent;
+  }
+
+  /**
+   * The bounce as one step per candidate row, for capture's chunks: the candidates are read up front, and each step
+   * checks its row is still queued and not yet bounced or reported in the transaction it runs in. Pushes to `sent`.
+   */
+  function bounceSteps({ now = clock(), sent = [] } = {}) {
+    const rows = bounceRows.all(...bounceWindow(now), BOUNCE_BATCH);
     let dir = null;
-    for (const r of rows) {
+    return rows.map((r) => () => {
+      if (!stillBounceable.get(...bounceWindow(now), r.id, r.recipient)) return;
       const why = bus.reach(r.recipient, now);
-      if (why.state !== 'undeliverable') continue;
+      if (why.state !== 'undeliverable') return;
       dir ??= registry.directory(now);
       const id = edgeId('u', r.id, r.recipient);
       const reason = why.reason === 'stale' ? `its session ${why.session} has had no activity for ${why.idleMin} min`
@@ -157,8 +171,7 @@ export function createDeliveryStates(hub, bus, { now: clock = Date.now } = {}) {
       }, dir);
       q.bounced.run(r.id, r.recipient, env.error ? `error:${env.error}` : id, now);
       if (!env.error) sent.push({ id: r.id, to: r.recipient, notice: id });
-    }
-    return sent;
+    });
   }
 
   /**
@@ -216,5 +229,5 @@ export function createDeliveryStates(hub, bus, { now: clock = Date.now } = {}) {
     return [...open].filter(([, settled]) => settled).map(([id]) => id);
   }
 
-  return { readOnHook, managerFyi, reports, honest, withDelivery, status, senderOf, report, bounce, postedSince, settleNudges, settledNudgeRecipients };
+  return { readOnHook, managerFyi, reports, honest, withDelivery, status, senderOf, report, bounce, bounceSteps, postedSince, settleNudges, settledNudgeRecipients };
 }

package/src/runner/directoryCache.js

@@ -0,0 +1,34 @@
+import { ONLINE_MS } from '../hub/core/ids.mjs';
+
+const WATCHED = ['actors', 'presence'];
+
+/**
+ * `registry.directory(now)` for the runner's connection, built again only when it could come out different: after a
+ * write to actors or presence on this connection (temp triggers set a new random token, which a rollback puts back),
+ * after a commit on another connection (`PRAGMA data_version`), or once `now` crosses a presence's online edge
+ * (`last_seen + ONLINE_MS`), the only way the clock changes it. Callers share the array and must not change it.
+ */
+export function cachedDirectory(db, registry) {
+  db.prepare('CREATE TEMP TABLE IF NOT EXISTS pulse_directory_token (one INTEGER PRIMARY KEY CHECK (one = 1), token TEXT NOT NULL)').run();
+  db.prepare('INSERT OR IGNORE INTO temp.pulse_directory_token (one, token) VALUES (1, hex(randomblob(8)))').run();
+  for (const table of WATCHED) {
+    for (const op of ['INSERT', 'UPDATE', 'DELETE']) {
+      db.prepare(`CREATE TEMP TRIGGER IF NOT EXISTS pulse_directory_${table}_${op.toLowerCase()} AFTER ${op} ON main.${table}
+        BEGIN UPDATE temp.pulse_directory_token SET token = hex(randomblob(8)) WHERE one = 1; END`).run();
+    }
+  }
+  const token = db.prepare('SELECT token FROM temp.pulse_directory_token WHERE one = 1');
+  const version = db.prepare('PRAGMA data_version');
+  const lastEdge = db.prepare('SELECT MAX(last_seen) AS t FROM presence WHERE last_seen <= ?');
+  const nextEdge = db.prepare('SELECT MIN(last_seen) AS t FROM presence WHERE last_seen > ?');
+  let kept = null;
+  return (now = Date.now()) => {
+    const key = `${version.get().data_version}:${token.get().token}`;
+    if (kept && kept.key === key && now >= kept.from && now < kept.until) return kept.dir;
+    const dir = registry.directory(now);
+    const before = lastEdge.get(now - ONLINE_MS).t;
+    const after = nextEdge.get(now - ONLINE_MS).t;
+    kept = { key, dir, from: before == null ? -Infinity : before + ONLINE_MS, until: after == null ? Infinity : after + ONLINE_MS };
+    return dir;
+  };
+}