@agentpulselabs/cli-plugin-pulse 1.1.2 → 1.1.5

package/src/runner/hubCapture.js

@@ -46,52 +46,89 @@ export function createHubCapture({ store, hubDir, roots, home, approvedRoots = [
     return { message: { ...message, meta: parse(message.meta) ?? {} }, to: snapshot.recipients([id]).get(id) || [] };
   }
 
-  return {
-    scan,
-    commands(change, db) {
-      const snapshot = new HubSnapshot(db);
-      const context = contextFor(db);
-      switch (change.tbl) {
-        case 'messages': {
-          if (change.op === 'delete') {
-            const recent = db.prepare('SELECT id, grp FROM recent_message WHERE id = ?').get(change.k1);
-            if (recent && !snapshot.messageExists(change.k1)) scan.messageGone(context, { id: recent.id, group: recent.grp });
-            return [];
-          }
-          const found = messageWithRecipients(snapshot, db, change.k1);
-          if (found) scan.messageRow(context, found.message, found.to);
-          return [];
-        }
-        case 'deliveries': {
-          if (change.op === 'delete') return [];
-          if (!store.known(change.k1)) {
-            const found = messageWithRecipients(snapshot, db, change.k1);
-            if (found) scan.messageRow(context, found.message, found.to);
-            return [];
-          }
-          const row = snapshot.receipt(change.k1, change.k2);
-          scan.receiptRow(context, { messageId: change.k1, recipient: change.k2, state: store.receiptState(change.k1, change.k2), row });
-          return [];
-        }
-        case 'actors': {
-          const row = db.prepare('SELECT * FROM actors WHERE id = ?').get(change.k1);
-          if (row) scan.actorRow(context, rowActor(row));
-          return [];
-        }
-        case 'facts': {
-          const row = db.prepare('SELECT kind, key, payload, updated_at FROM facts WHERE kind = ? AND key = ?').get(change.k1, change.k2);
-          if (row) scan.factRow(context, { ...row });
-          else if (REMOVABLE_KINDS.includes(change.k1)) scan.factRemoved(context, change.k1, change.k2);
+  const actorById = (db, id) => db.prepare('SELECT * FROM actors WHERE id = ?').get(id);
+  const factByKey = (db, kind, key) => db.prepare('SELECT kind, key, payload, updated_at FROM facts WHERE kind = ? AND key = ?').get(kind, key);
+
+  function commandsIn(snapshot, context, change, db) {
+    switch (change.tbl) {
+      case 'messages': {
+        if (change.op === 'delete') {
+          const recent = db.prepare('SELECT id, grp FROM recent_message WHERE id = ?').get(change.k1);
+          if (recent && !snapshot.messageExists(change.k1)) scan.messageGone(context, { id: recent.id, group: recent.grp });
           return [];
         }
-        case 'events': {
-          const event = db.prepare('SELECT id, at, kind, actor, payload FROM events WHERE id = ?').get(change.k1);
-          if (event) scan.eventRow(context, { ...event, payload: parse(event.payload) ?? {} });
+        const found = messageWithRecipients(snapshot, db, change.k1);
+        if (found) scan.messageRow(context, found.message, found.to);
+        return [];
+      }
+      case 'deliveries': {
+        if (change.op === 'delete') return [];
+        if (!store.known(change.k1)) {
+          const found = messageWithRecipients(snapshot, db, change.k1);
+          if (found) scan.messageRow(context, found.message, found.to);
           return [];
         }
-        default:
-          return [];
+        const row = snapshot.receipt(change.k1, change.k2);
+        scan.receiptRow(context, { messageId: change.k1, recipient: change.k2, state: store.receiptState(change.k1, change.k2), row });
+        return [];
+      }
+      case 'actors': {
+        const row = actorById(db, change.k1);
+        if (row) scan.actorRow(context, rowActor(row));
+        return [];
+      }
+      case 'facts': {
+        const row = factByKey(db, change.k1, change.k2);
+        if (row) scan.factRow(context, { ...row });
+        else if (REMOVABLE_KINDS.includes(change.k1)) scan.factRemoved(context, change.k1, change.k2);
+        return [];
+      }
+      case 'events': {
+        const event = db.prepare('SELECT id, at, kind, actor, payload FROM events WHERE id = ?').get(change.k1);
+        if (event) scan.eventRow(context, { ...event, payload: parse(event.payload) ?? {} });
+        return [];
       }
+      default:
+        return [];
+    }
+  }
+
+  /** The keys of every removable fact the hub still has, as the sweep's fact pass would count them present. */
+  function presentFacts(db) {
+    const present = new Map(REMOVABLE_KINDS.map((kind) => [kind, new Set()]));
+    const statement = db.prepare('SELECT key, payload FROM facts WHERE kind = ?');
+    for (const kind of REMOVABLE_KINDS.filter((k) => FACT_KINDS.includes(k))) {
+      for (const row of statement.all(kind)) {
+        const fact = parse(row.payload);
+        if (!fact) continue;
+        present.get(kind).add(row.key);
+        if (kind === 'delivery') for (const stream of fact.streams || []) present.get('stream').add(stream.key);
+      }
+    }
+    return present;
+  }
+
+  function removeDropped(context, db) {
+    const present = presentFacts(db);
+    for (const kind of REMOVABLE_KINDS) {
+      for (const entity of store.entities(`fact:${kind}:`)) {
+        const key = entity.slice(`fact:${kind}:`.length);
+        if (!present.get(kind).has(key)) scan.factRemoved(context, kind, key);
+      }
+    }
+  }
+
+  return {
+    scan,
+    commands(change, db) {
+      return commandsIn(new HubSnapshot(db), contextFor(db), change, db);
+    },
+
+    /** `commands` for one drained page of journal rows, which share one scan context. */
+    batch(db) {
+      const snapshot = new HubSnapshot(db);
+      let context = null;
+      return (change) => commandsIn(snapshot, (context ??= contextFor(db)), change, db);
     },
 
     /**
@@ -103,7 +140,7 @@ export function createHubCapture({ store, hubDir, roots, home, approvedRoots = [
       const context = contextFor(db);
       switch (change.tbl) {
         case 'actors': {
-          const row = db.prepare('SELECT * FROM actors WHERE id = ?').get(change.k1);
+          const row = actorById(db, change.k1);
           if (!row) return;
           const actor = rowActor(row);
           const entity = `actor:${actor.id}`;
@@ -116,7 +153,7 @@ export function createHubCapture({ store, hubDir, roots, home, approvedRoots = [
           return;
         }
         case 'facts': {
-          const row = db.prepare('SELECT kind, key, payload, updated_at FROM facts WHERE kind = ? AND key = ?').get(change.k1, change.k2);
+          const row = factByKey(db, change.k1, change.k2);
           const fact = row && parse(row.payload);
           if (!fact || !FACT_KINDS.includes(row.kind)) return;
           scan.factCommands(context, { ...row }, fact).forEach((item, index) => {
@@ -141,20 +178,32 @@ export function createHubCapture({ store, hubDir, roots, home, approvedRoots = [
       const snapshot = new HubSnapshot(db);
       const context = contextFor(db);
       for (const actor of snapshot.actors()) scan.actorRow(context, actor);
-      const present = new Map(REMOVABLE_KINDS.map((kind) => [kind, new Set()]));
-      for (const row of snapshot.facts(ALL_FACT_KINDS)) {
-        const fact = scan.factRow(context, row);
-        if (!fact) continue;
-        present.get(row.kind)?.add(row.key);
-        if (row.kind === 'delivery') for (const stream of fact.streams || []) present.get('stream').add(stream.key);
-      }
-      for (const kind of REMOVABLE_KINDS) {
-        for (const entity of store.entities(`fact:${kind}:`)) {
-          const key = entity.slice(`fact:${kind}:`.length);
-          if (!present.get(kind).has(key)) scan.factRemoved(context, kind, key);
-        }
-      }
+      for (const row of snapshot.facts(ALL_FACT_KINDS)) scan.factRow(context, row);
+      removeDropped(context, db);
       return context.report;
     },
+
+    /**
+     * The sweep as steps for capture's chunks, in the sweep's own order: each actor and fact read again when its step
+     * runs (a row written between chunks is swept as it is then), and the removed facts last, from the keys the hub
+     * holds at that moment, so a fact added mid-sweep is never taken for a removed one.
+     */
+    sweepSteps(db) {
+      let context = null;
+      const current = () => (context ??= contextFor(db));
+      const ids = db.prepare('SELECT id FROM actors ORDER BY rowid').all().map((row) => row.id);
+      const facts = new HubSnapshot(db).factKeys(ALL_FACT_KINDS);
+      return [
+        ...ids.map((id) => () => {
+          const row = actorById(db, id);
+          if (row) scan.actorRow(current(), rowActor(row));
+        }),
+        ...facts.map(({ kind, key }) => () => {
+          const row = factByKey(db, kind, key);
+          if (row) scan.factRow(current(), { ...row });
+        }),
+        () => removeDropped(current(), db),
+      ];
+    },
   };
 }

package/src/runner/loopMonitor.js

@@ -0,0 +1,89 @@
+import { monitorEventLoopDelay } from 'node:perf_hooks';
+
+/** The window `runner.loopDelay` in daemon-health.json covers, in one-minute histograms. */
+export const LOOP_WINDOW_MS = 5 * 60e3;
+/** A tick that holds the event loop this long is logged as `loop_stall`, with its name. */
+export const STALL_MS = 2_000;
+const MINUTE_MS = 60e3;
+const PROBE_MS = 100;
+const RESOLUTION_MS = 20;
+
+const ms = (ns) => Math.round(ns / 1e5) / 10;
+
+/**
+ * The runner's event-loop delay (perf_hooks.monitorEventLoopDelay), kept as one histogram per minute for the last five:
+ * `view()` gives the window's exact max, the worst minute's p99 and the median of the minutes' p50s (histograms of
+ * this kind can't be merged). `tick(name, fn)` runs a tick under its name: one whose synchronous part runs past
+ * stallMs is logged as `loop_stall` with that name, and any other stall the probe sees is logged with the name of the
+ * tick that started last.
+ */
+export function createLoopMonitor({ log, windowMs = LOOP_WINDOW_MS, stallMs = STALL_MS, now = () => performance.now() }) {
+  const minutes = [];
+  const open = () => {
+    const histogram = monitorEventLoopDelay({ resolution: RESOLUTION_MS });
+    histogram.enable();
+    return { at: now(), histogram };
+  };
+  let live = open();
+  let current = null;
+  let claimed = false;
+  let last = now();
+
+  const probe = setInterval(() => {
+    const at = now();
+    const gap = at - last - PROBE_MS;
+    last = at;
+    if (gap > stallMs && !claimed) log('loop_stall', { ms: Math.round(gap), tick: current?.name ?? 'untracked', ...(current ? { running: current.running } : {}) });
+    claimed = false;
+    if (at - live.at >= MINUTE_MS) {
+      live.histogram.disable();
+      minutes.push(live);
+      while (minutes.length && at - minutes[0].at >= windowMs) minutes.shift();
+      live = open();
+    }
+  }, PROBE_MS);
+  probe.unref();
+
+  function tick(name, fn) {
+    const entry = { name, running: true };
+    current = entry;
+    const started = now();
+    const settle = () => { entry.running = false; };
+    let out;
+    try {
+      out = fn();
+    } finally {
+      const held = now() - started;
+      if (held > stallMs) {
+        claimed = true;
+        log('loop_stall', { ms: Math.round(held), tick: name });
+      }
+    }
+    if (out && typeof out.then === 'function') out.then(settle, settle);
+    else settle();
+    return out;
+  }
+
+  function view() {
+    const all = [...minutes, live].filter((m) => m.histogram.count > 0);
+    if (!all.length) return { p50Ms: null, p99Ms: null, maxMs: null, windowMs, minutes: 0 };
+    const p50s = all.map((m) => m.histogram.percentile(50)).sort((a, b) => a - b);
+    return {
+      p50Ms: ms(p50s[Math.floor(p50s.length / 2)]),
+      p99Ms: ms(Math.max(...all.map((m) => m.histogram.percentile(99)))),
+      maxMs: ms(Math.max(...all.map((m) => m.histogram.max))),
+      windowMs,
+      minutes: all.length,
+    };
+  }
+
+  function close() {
+    clearInterval(probe);
+    live.histogram.disable();
+  }
+
+  return { tick, view, close };
+}
+
+/** For a facade served without a runner process (tests): ticks run as they are. */
+export const UNMONITORED = Object.freeze({ tick: (name, fn) => fn(), view: () => null, close() {} });

package/src/runner/runnerHub.js

@@ -4,6 +4,7 @@ import { openHub } from '../hub/core/hub.mjs';
 import { HubError } from '../lib/errors.js';
 import { OUTBOX_MAX_PENDING, RunnerStore, SEED_META } from '../lib/runnerStore.js';
 import { createCapture, installCapture } from './capture.js';
+import { cachedDirectory } from './directoryCache.js';
 import { installRunnerSchema } from './runnerSchema.js';
 
 /**
@@ -73,7 +74,8 @@ export async function openRunnerHub({ directory, stateDir, mapping, now = Date.n
   const init = await RunnerStore.open(directory, { now, maxPending });
   const { file } = init;
   init.close();
-  const hub = openHub(stateDir, file);
+  const opened = openHub(stateDir, file);
+  const hub = { ...opened, registry: { ...opened.registry, directory: cachedDirectory(opened.db, opened.registry) } };
   installCapture(hub.db);
   installRunnerSchema(hub.db);
   const store = new RunnerStore({ db: hub.db, file, now, maxPending });

package/src/runner/runnerProcess.js

@@ -2,17 +2,22 @@ import { mkdirSync, watch } from 'node:fs';
 import path from 'node:path';
 import { RUNNER_DB_FILE } from '../lib/runnerStore.js';
 import { createApplier } from './applier.js';
+import { CHUNK_MS } from './capture.js';
 import { createLeaseState } from './lease.js';
+import { createLoopMonitor } from './loopMonitor.js';
 import { createRenderer } from './render.js';
 import { mappingFor } from './captureMapping.js';
 import { serve } from './facade.js';
 import { openRunnerHub } from './runnerHub.js';
+import { serial } from './serial.js';
 import { importSpool } from './spool.js';
 
 const PUMP_MS = 250;
 const SPOOL_SCAN_MS = 5000;
 /** Actors coalesced by capture (a clock-only change goes up at most once a minute) and dropped facts. */
 const SWEEP_MS = 30_000;
+/** How often the runner tells the daemon its event-loop delay (daemon-health.json `runner.loopDelay`). */
+const LOOP_REPORT_MS = 5_000;
 
 /** Runs the runner in this process: runner.db as the hub, capture, the facade. Resolves once it is listening. */
 export async function startRunner({
@@ -20,9 +25,16 @@ export async function startRunner({
   enableManager = true, pulseUrl = null, hookBudgetMs, writeBudgetMs, machineId = null, cutoverSeq = 0, lease: leaseMode = 'cloud', managerOptions, maxPending,
   seedFrom = null, seedSha256 = null, workspaceRoot = path.dirname(hubHome), discipline, send = process.send ? (message) => process.send(message) : null, log = (event, details) => {
     process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...details })}\n`);
-  },
+  }, loopStallMs,
 }) {
   if (path.basename(dbFile) !== RUNNER_DB_FILE) throw new Error(`the runner database must be named ${RUNNER_DB_FILE}`);
+  const loop = createLoopMonitor({
+    ...(loopStallMs ? { stallMs: loopStallMs } : {}),
+    log: (event, details) => {
+      log(event, details);
+      if (send) send({ type: event, details });
+    },
+  });
   const stateDir = path.join(hubHome, 'state');
   // The values capture scrubs besides its secret rules: the hub token, and the daemon's held credentials (IPC).
   let heldSecrets = [];
@@ -75,29 +87,58 @@ export async function startRunner({
   const facade = serve({
     hub: runner.hub, capture: runner.capture, port, listenHost: host, stateDir, dbFile, enableCollector, enableManager,
     pulseUrl, onWrite, health: () => health, ...(lease ? { lease } : {}), ...(managerOptions ? { managerOptions } : {}), ...(hookBudgetMs !== undefined ? { hookBudgetMs } : {}), ...(writeBudgetMs !== undefined ? { writeBudgetMs } : {}),
-    channelRoots: roots, workspaceRoot, ...(discipline ? { discipline } : {}), ...(transcriptsDir !== undefined ? { transcriptsDir } : {}),
+    channelRoots: roots, workspaceRoot, loop, ...(discipline ? { discipline } : {}), ...(transcriptsDir !== undefined ? { transcriptsDir } : {}),
+  });
+  let closing = false;
+  const failed = (event, error) => log(event, { code: error.code || 'error', message: String(error.message).slice(0, 200) });
+  /** What other connections (the collect worker) left in the capture journal, a short transaction at a time. */
+  const drainJournal = serial(async () => {
+    try {
+      while (!closing) {
+        if (runner.capture.pump({ budgetMs: CHUNK_MS })) onWrite();
+        if (!runner.capture.pending()) return;
+        await runner.capture.yieldLock();
+      }
+    } catch (error) { failed('capture_pump_failed', error); }
   });
-  const pump = setInterval(() => {
-    try { if (runner.capture.pump()) onWrite(); } catch (error) { log('capture_pump_failed', { code: error.code || 'error', message: String(error.message).slice(0, 200) }); }
-  }, PUMP_MS);
+  const pump = setInterval(() => loop.tick('capture pump', drainJournal), PUMP_MS);
   pump.unref();
-  const sweep = setInterval(() => {
-    try { runner.capture.tx(() => runner.mapping.sweep(runner.hub.db), { label: 'capture sweep' }); onWrite(); } catch (error) { log('capture_sweep_failed', { code: error.code || 'error', message: String(error.message).slice(0, 200) }); }
-  }, SWEEP_MS);
+  const sweepAll = serial(async () => {
+    try {
+      const { db } = runner.hub;
+      const steps = runner.mapping.sweepSteps ? runner.mapping.sweepSteps(db) : [() => runner.mapping.sweep(db)];
+      await runner.capture.chunks(steps, { label: 'capture sweep' });
+      onWrite();
+    } catch (error) { failed('capture_sweep_failed', error); }
+  });
+  const sweep = setInterval(() => loop.tick('capture sweep', sweepAll), SWEEP_MS);
   sweep.unref();
   const spoolDir = path.join(stateDir, 'spool');
   mkdirSync(spoolDir, { recursive: true, mode: 0o700 });
-  const drainSpool = () => {
+  const spoolMarks = new Map();
+  const spoolPass = (budgetMs) => importSpool({ hub: runner.hub, capture: runner.capture, spoolDir, log, marks: spoolMarks, budgetMs });
+  const drainSpool = serial(async () => {
     try {
-      const { imported } = importSpool({ hub: runner.hub, capture: runner.capture, spoolDir, log });
+      let imported = 0;
+      while (!closing) {
+        const pass = spoolPass(CHUNK_MS);
+        imported += pass.imported;
+        if (!pass.more) break;
+        await runner.capture.yieldLock();
+      }
       if (imported) { log('spool_imported', { imported }); onWrite(); }
-    } catch (error) { log('spool_import_failed', { code: error.code || 'error', message: String(error.message).slice(0, 200) }); }
-  };
-  drainSpool();
-  const spoolWatch = watch(spoolDir, () => drainSpool());
+    } catch (error) { failed('spool_import_failed', error); }
+  });
+  try {
+    const { imported } = spoolPass(Infinity);
+    if (imported) { log('spool_imported', { imported }); onWrite(); }
+  } catch (error) { failed('spool_import_failed', error); }
+  const spoolWatch = watch(spoolDir, () => loop.tick('spool import', drainSpool));
   spoolWatch.on('error', () => {});
-  const spoolScan = setInterval(drainSpool, SPOOL_SCAN_MS);
+  const spoolScan = setInterval(() => loop.tick('spool import', drainSpool), SPOOL_SCAN_MS);
   spoolScan.unref();
+  const loopReport = send ? setInterval(() => send({ type: 'loop', loopDelay: loop.view() }), LOOP_REPORT_MS) : null;
+  loopReport?.unref();
   await new Promise((resolve, reject) => {
     facade.server.once('listening', resolve);
     facade.server.once('error', reject);
@@ -105,13 +146,14 @@ export async function startRunner({
   const address = facade.server.address();
   log('runner_ready', { port: address.port, pid: process.pid });
   if (send) send({ type: 'ready', port: address.port, pid: process.pid });
-  let closing = false;
   const close = () => {
     if (closing) return;
     closing = true;
     clearInterval(pump);
     clearInterval(sweep);
     clearInterval(spoolScan);
+    clearInterval(loopReport);
+    loop.close();
     spoolWatch.close();
     facade.server.close();
     facade.server.closeAllConnections?.();

package/src/runner/runnerService.js

@@ -1,3 +1,4 @@
+import { readFileSync } from 'node:fs';
 import { readFile, writeFile } from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
@@ -15,6 +16,18 @@ export const RUNNER_CONFIG_FILE = 'runner.json';
 const RESTART_MIN_MS = 1_000;
 const RESTART_MAX_MS = 30_000;
 const SECRETS_EVERY_MS = 60_000;
+/** What daemon-health.json shows of the runner's manager (state/manager-status.json): its state and what last went wrong. */
+const MANAGER_FIELDS = ['state', 'error', 'messageId', 'lastFailure', 'gaveUp', 'lastReplyAt', 'at'];
+
+/** The manager's status for daemon-health.json, or null while it has written none. */
+export function managerHealth(file) {
+  let status;
+  try { status = JSON.parse(readFileSync(file, 'utf8')); } catch { return null; }
+  if (!status || typeof status !== 'object') return null;
+  const out = Object.fromEntries(MANAGER_FIELDS.filter((key) => status[key] != null).map((key) => [key, status[key]]));
+  if (status.queue) out.queue = { depth: status.queue.depth, oldestAgeMs: status.queue.oldestAgeMs };
+  return out;
+}
 
 /** Whether this machine has cut over: the daemon runs the hub (the runner) and serves the local API (design §11). */
 export const runnerMode = (config) => config?.cloudFirst?.mode === 'runner';
@@ -38,13 +51,22 @@ export class RunnerService {
     this.log = log;
     this.now = now;
     this.key = null;
+    this.generation = -Infinity;
     this.current = null;
     this.restarts = 0;
     this.state = { state: 'off' };
   }
 
-  /** Start, keep or stop the runner for this config. Never waits on the runner. */
-  update(config) {
+  /**
+   * Start, keep or stop the runner for this config. Never waits on the runner. `generation` orders the reads of
+   * config.json: a config read before the last one applied is ignored, so a heartbeat that read it before a cutover's
+   * `serve` cannot switch the runner back (CUT-03).
+   */
+  update(config, { generation = null } = {}) {
+    if (generation !== null) {
+      if (generation < this.generation) return this.state;
+      this.generation = generation;
+    }
     const on = runnerMode(config) && pairedStatus(config).ok;
     const key = on ? JSON.stringify({ c: config.cloudFirst, m: config.paired.machineId, v: config.paired.policyVersion }) : null;
     if (key === this.key) return this.state;
@@ -67,7 +89,8 @@ export class RunnerService {
 
   health() {
     const bridge = this.current?.bridge;
-    return { ...this.state, ...(bridge ? { runner: bridge.health(), lease: bridge.lease ?? null } : {}) };
+    const manager = this.current?.managerStatusFile ? managerHealth(this.current.managerStatusFile) : null;
+    return { ...this.state, ...(bridge ? { runner: bridge.health(), lease: bridge.lease ?? null } : {}), ...(manager ? { manager } : {}) };
   }
 
   async #start(config) {
@@ -85,11 +108,13 @@ export class RunnerService {
     const store = await this.openStore(this.directory);
     const transport = await this.transportFactory(config);
     const held = () => transport.auth?.heldSecrets?.() ?? [];
-    const current = { store, transport, stopping: false };
+    const current = { store, transport, stopping: false, managerStatusFile: path.join(hubHome, 'state', 'manager-status.json') };
     this.current = current;
     const launch = async () => {
       const runner = this.fork({ hubHome, dbFile, configFile });
       runner.child.stderr?.on('data', (chunk) => this.log('runner_stderr', { line: String(chunk).slice(0, 300) }));
+      // The runner's event-loop delay (runner/loopMonitor.js), as `runner.loopDelay` in daemon-health.json.
+      runner.on('loop', (message) => { if (this.current === current) this.state = { ...this.state, loopDelay: message.loopDelay ?? null }; });
       const ready = await runner.ready;
       const hubToken = (await readFile(path.join(hubHome, 'state', 'hub.token'), 'utf8').catch(() => '')).trim() || null;
       const fileScan = new HubScan({

package/src/runner/serial.js

@@ -0,0 +1,5 @@
+/** An async tick that never runs twice at once: called while it still runs, it returns the run in progress. */
+export function serial(fn) {
+  let running = null;
+  return () => (running ??= Promise.resolve().then(fn).finally(() => { running = null; }));
+}

package/src/runner/spool.js

@@ -1,5 +1,5 @@
 import { createHash } from 'node:crypto';
-import { readFileSync, readdirSync } from 'node:fs';
+import { closeSync, fstatSync, openSync, readFileSync, readSync, readdirSync, statSync } from 'node:fs';
 import path from 'node:path';
 
 const SPOOL_FILE = /^\d{8}\.jsonl$/;
@@ -42,28 +42,61 @@ function completeLines(text) {
   return end < 0 ? [] : text.slice(0, end).split('\n');
 }
 
+/** A spool file from byte `offset` on: the whole file when there is no watermark. */
+function readFrom(file, offset) {
+  if (!offset) return readFileSync(file, 'utf8');
+  const fd = openSync(file, 'r');
+  try {
+    const size = fstatSync(fd).size;
+    const buffer = Buffer.alloc(Math.max(0, size - offset));
+    readSync(fd, buffer, 0, buffer.length, offset);
+    return buffer.toString('utf8');
+  } finally { closeSync(fd); }
+}
+
 /**
  * Imports every spool line not imported before, oldest file first, each in its own capture transaction with its
  * pulse_spool row, so a crash between lines imports the rest exactly once on the next pass.
+ *
+ * `marks` (a Map the caller keeps) remembers how far each file was read: a file whose size and mtime have not moved
+ * is not read, and one that grew is read from where the last pass stopped. A file that shrank or was replaced is read
+ * from the start again (its lines are known by their sha256). Once `budgetMs` has passed and the pass has applied a
+ * line, it stops before its next line with `more` set, so the caller can yield the event loop and go on from there.
  */
-export function importSpool({ hub, capture, spoolDir, now = Date.now, log = () => {} }) {
+export function importSpool({ hub, capture, spoolDir, now = Date.now, log = () => {}, marks = null, budgetMs = Infinity }) {
   let files;
   try {
     files = readdirSync(spoolDir).filter((f) => SPOOL_FILE.test(f)).sort();
   } catch (error) {
-    if (error.code === 'ENOENT') return { imported: 0, refused: 0 };
+    if (error.code === 'ENOENT') return { imported: 0, refused: 0, more: false };
     throw error;
   }
+  const started = performance.now();
   const seen = hub.db.prepare('SELECT 1 AS found FROM pulse_spool WHERE sha256 = ?');
   const record = hub.db.prepare('INSERT INTO pulse_spool (sha256, file, line, outcome, imported_at) VALUES (?, ?, ?, ?, ?)');
   let imported = 0;
   let refused = 0;
+  let applied = 0;
+  if (marks) for (const file of marks.keys()) if (!files.includes(file)) marks.delete(file);
   for (const file of files) {
-    const lines = completeLines(readFileSync(path.join(spoolDir, file), 'utf8'));
-    lines.forEach((line, index) => {
-      if (!line.trim()) return;
+    const full = path.join(spoolDir, file);
+    const st = marks ? statSync(full) : null;
+    let mark = marks?.get(file);
+    if (mark && mark.ino === st.ino && mark.size === st.size && mark.mtimeMs === st.mtimeMs) continue;
+    if (mark && (mark.ino !== st.ino || st.size < mark.offset)) mark = null;
+    const lines = completeLines(readFrom(full, mark?.offset ?? 0));
+    let offset = mark?.offset ?? 0;
+    let line0 = mark?.lines ?? 0;
+    for (const [index, line] of lines.entries()) {
+      if (applied && performance.now() - started >= budgetMs) {
+        marks?.set(file, { ino: st.ino, size: null, mtimeMs: null, offset, lines: line0 + index });
+        return { imported, refused, more: true };
+      }
+      offset += Buffer.byteLength(line, 'utf8') + 1;
+      if (!line.trim()) continue;
       const sha = createHash('sha256').update(line).digest('hex');
-      if (seen.get(sha)) return;
+      if (seen.get(sha)) continue;
+      applied += 1;
       capture.tx(() => {
         let outcome;
         try {
@@ -73,14 +106,16 @@ export function importSpool({ hub, capture, spoolDir, now = Date.now, log = () =
         } catch {
           outcome = 'refused: unreadable line';
         }
-        record.run(sha, file, index + 1, outcome, now());
+        record.run(sha, file, line0 + index + 1, outcome, now());
         if (outcome === 'applied') imported += 1;
         else {
           refused += 1;
-          log('spool_line_refused', { file, line: index + 1, outcome });
+          log('spool_line_refused', { file, line: line0 + index + 1, outcome });
         }
       }, { label: 'spool import' });
-    });
+    }
+    line0 += lines.length;
+    marks?.set(file, { ino: st.ino, size: st.size, mtimeMs: st.mtimeMs, offset, lines: line0 });
   }
-  return { imported, refused };
+  return { imported, refused, more: false };
 }