@agentpulselabs/cli-plugin-pulse 1.1.3 → 1.1.5

package/src/runner/runnerProcess.js

@@ -2,17 +2,22 @@ import { mkdirSync, watch } from 'node:fs';
 import path from 'node:path';
 import { RUNNER_DB_FILE } from '../lib/runnerStore.js';
 import { createApplier } from './applier.js';
+import { CHUNK_MS } from './capture.js';
 import { createLeaseState } from './lease.js';
+import { createLoopMonitor } from './loopMonitor.js';
 import { createRenderer } from './render.js';
 import { mappingFor } from './captureMapping.js';
 import { serve } from './facade.js';
 import { openRunnerHub } from './runnerHub.js';
+import { serial } from './serial.js';
 import { importSpool } from './spool.js';
 
 const PUMP_MS = 250;
 const SPOOL_SCAN_MS = 5000;
 /** Actors coalesced by capture (a clock-only change goes up at most once a minute) and dropped facts. */
 const SWEEP_MS = 30_000;
+/** How often the runner tells the daemon its event-loop delay (daemon-health.json `runner.loopDelay`). */
+const LOOP_REPORT_MS = 5_000;
 
 /** Runs the runner in this process: runner.db as the hub, capture, the facade. Resolves once it is listening. */
 export async function startRunner({
@@ -20,9 +25,16 @@ export async function startRunner({
   enableManager = true, pulseUrl = null, hookBudgetMs, writeBudgetMs, machineId = null, cutoverSeq = 0, lease: leaseMode = 'cloud', managerOptions, maxPending,
   seedFrom = null, seedSha256 = null, workspaceRoot = path.dirname(hubHome), discipline, send = process.send ? (message) => process.send(message) : null, log = (event, details) => {
     process.stdout.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...details })}\n`);
-  },
+  }, loopStallMs,
 }) {
   if (path.basename(dbFile) !== RUNNER_DB_FILE) throw new Error(`the runner database must be named ${RUNNER_DB_FILE}`);
+  const loop = createLoopMonitor({
+    ...(loopStallMs ? { stallMs: loopStallMs } : {}),
+    log: (event, details) => {
+      log(event, details);
+      if (send) send({ type: event, details });
+    },
+  });
   const stateDir = path.join(hubHome, 'state');
   // The values capture scrubs besides its secret rules: the hub token, and the daemon's held credentials (IPC).
   let heldSecrets = [];
@@ -75,29 +87,58 @@ export async function startRunner({
   const facade = serve({
     hub: runner.hub, capture: runner.capture, port, listenHost: host, stateDir, dbFile, enableCollector, enableManager,
     pulseUrl, onWrite, health: () => health, ...(lease ? { lease } : {}), ...(managerOptions ? { managerOptions } : {}), ...(hookBudgetMs !== undefined ? { hookBudgetMs } : {}), ...(writeBudgetMs !== undefined ? { writeBudgetMs } : {}),
-    channelRoots: roots, workspaceRoot, ...(discipline ? { discipline } : {}), ...(transcriptsDir !== undefined ? { transcriptsDir } : {}),
+    channelRoots: roots, workspaceRoot, loop, ...(discipline ? { discipline } : {}), ...(transcriptsDir !== undefined ? { transcriptsDir } : {}),
+  });
+  let closing = false;
+  const failed = (event, error) => log(event, { code: error.code || 'error', message: String(error.message).slice(0, 200) });
+  /** What other connections (the collect worker) left in the capture journal, a short transaction at a time. */
+  const drainJournal = serial(async () => {
+    try {
+      while (!closing) {
+        if (runner.capture.pump({ budgetMs: CHUNK_MS })) onWrite();
+        if (!runner.capture.pending()) return;
+        await runner.capture.yieldLock();
+      }
+    } catch (error) { failed('capture_pump_failed', error); }
   });
-  const pump = setInterval(() => {
-    try { if (runner.capture.pump()) onWrite(); } catch (error) { log('capture_pump_failed', { code: error.code || 'error', message: String(error.message).slice(0, 200) }); }
-  }, PUMP_MS);
+  const pump = setInterval(() => loop.tick('capture pump', drainJournal), PUMP_MS);
   pump.unref();
-  const sweep = setInterval(() => {
-    try { runner.capture.tx(() => runner.mapping.sweep(runner.hub.db), { label: 'capture sweep' }); onWrite(); } catch (error) { log('capture_sweep_failed', { code: error.code || 'error', message: String(error.message).slice(0, 200) }); }
-  }, SWEEP_MS);
+  const sweepAll = serial(async () => {
+    try {
+      const { db } = runner.hub;
+      const steps = runner.mapping.sweepSteps ? runner.mapping.sweepSteps(db) : [() => runner.mapping.sweep(db)];
+      await runner.capture.chunks(steps, { label: 'capture sweep' });
+      onWrite();
+    } catch (error) { failed('capture_sweep_failed', error); }
+  });
+  const sweep = setInterval(() => loop.tick('capture sweep', sweepAll), SWEEP_MS);
   sweep.unref();
   const spoolDir = path.join(stateDir, 'spool');
   mkdirSync(spoolDir, { recursive: true, mode: 0o700 });
-  const drainSpool = () => {
+  const spoolMarks = new Map();
+  const spoolPass = (budgetMs) => importSpool({ hub: runner.hub, capture: runner.capture, spoolDir, log, marks: spoolMarks, budgetMs });
+  const drainSpool = serial(async () => {
     try {
-      const { imported } = importSpool({ hub: runner.hub, capture: runner.capture, spoolDir, log });
+      let imported = 0;
+      while (!closing) {
+        const pass = spoolPass(CHUNK_MS);
+        imported += pass.imported;
+        if (!pass.more) break;
+        await runner.capture.yieldLock();
+      }
       if (imported) { log('spool_imported', { imported }); onWrite(); }
-    } catch (error) { log('spool_import_failed', { code: error.code || 'error', message: String(error.message).slice(0, 200) }); }
-  };
-  drainSpool();
-  const spoolWatch = watch(spoolDir, () => drainSpool());
+    } catch (error) { failed('spool_import_failed', error); }
+  });
+  try {
+    const { imported } = spoolPass(Infinity);
+    if (imported) { log('spool_imported', { imported }); onWrite(); }
+  } catch (error) { failed('spool_import_failed', error); }
+  const spoolWatch = watch(spoolDir, () => loop.tick('spool import', drainSpool));
   spoolWatch.on('error', () => {});
-  const spoolScan = setInterval(drainSpool, SPOOL_SCAN_MS);
+  const spoolScan = setInterval(() => loop.tick('spool import', drainSpool), SPOOL_SCAN_MS);
   spoolScan.unref();
+  const loopReport = send ? setInterval(() => send({ type: 'loop', loopDelay: loop.view() }), LOOP_REPORT_MS) : null;
+  loopReport?.unref();
   await new Promise((resolve, reject) => {
     facade.server.once('listening', resolve);
     facade.server.once('error', reject);
@@ -105,13 +146,14 @@ export async function startRunner({
   const address = facade.server.address();
   log('runner_ready', { port: address.port, pid: process.pid });
   if (send) send({ type: 'ready', port: address.port, pid: process.pid });
-  let closing = false;
   const close = () => {
     if (closing) return;
     closing = true;
     clearInterval(pump);
     clearInterval(sweep);
     clearInterval(spoolScan);
+    clearInterval(loopReport);
+    loop.close();
     spoolWatch.close();
     facade.server.close();
     facade.server.closeAllConnections?.();

package/src/runner/runnerService.js

@@ -1,3 +1,4 @@
+import { readFileSync } from 'node:fs';
 import { readFile, writeFile } from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
@@ -15,6 +16,18 @@ export const RUNNER_CONFIG_FILE = 'runner.json';
 const RESTART_MIN_MS = 1_000;
 const RESTART_MAX_MS = 30_000;
 const SECRETS_EVERY_MS = 60_000;
+/** What daemon-health.json shows of the runner's manager (state/manager-status.json): its state and what last went wrong. */
+const MANAGER_FIELDS = ['state', 'error', 'messageId', 'lastFailure', 'gaveUp', 'lastReplyAt', 'at'];
+
+/** The manager's status for daemon-health.json, or null while it has written none. */
+export function managerHealth(file) {
+  let status;
+  try { status = JSON.parse(readFileSync(file, 'utf8')); } catch { return null; }
+  if (!status || typeof status !== 'object') return null;
+  const out = Object.fromEntries(MANAGER_FIELDS.filter((key) => status[key] != null).map((key) => [key, status[key]]));
+  if (status.queue) out.queue = { depth: status.queue.depth, oldestAgeMs: status.queue.oldestAgeMs };
+  return out;
+}
 
 /** Whether this machine has cut over: the daemon runs the hub (the runner) and serves the local API (design §11). */
 export const runnerMode = (config) => config?.cloudFirst?.mode === 'runner';
@@ -76,7 +89,8 @@ export class RunnerService {
 
   health() {
     const bridge = this.current?.bridge;
-    return { ...this.state, ...(bridge ? { runner: bridge.health(), lease: bridge.lease ?? null } : {}) };
+    const manager = this.current?.managerStatusFile ? managerHealth(this.current.managerStatusFile) : null;
+    return { ...this.state, ...(bridge ? { runner: bridge.health(), lease: bridge.lease ?? null } : {}), ...(manager ? { manager } : {}) };
   }
 
   async #start(config) {
@@ -94,11 +108,13 @@ export class RunnerService {
     const store = await this.openStore(this.directory);
     const transport = await this.transportFactory(config);
     const held = () => transport.auth?.heldSecrets?.() ?? [];
-    const current = { store, transport, stopping: false };
+    const current = { store, transport, stopping: false, managerStatusFile: path.join(hubHome, 'state', 'manager-status.json') };
     this.current = current;
     const launch = async () => {
       const runner = this.fork({ hubHome, dbFile, configFile });
       runner.child.stderr?.on('data', (chunk) => this.log('runner_stderr', { line: String(chunk).slice(0, 300) }));
+      // The runner's event-loop delay (runner/loopMonitor.js), as `runner.loopDelay` in daemon-health.json.
+      runner.on('loop', (message) => { if (this.current === current) this.state = { ...this.state, loopDelay: message.loopDelay ?? null }; });
       const ready = await runner.ready;
       const hubToken = (await readFile(path.join(hubHome, 'state', 'hub.token'), 'utf8').catch(() => '')).trim() || null;
       const fileScan = new HubScan({

package/src/runner/serial.js

@@ -0,0 +1,5 @@
+/** An async tick that never runs twice at once: called while it still runs, it returns the run in progress. */
+export function serial(fn) {
+  let running = null;
+  return () => (running ??= Promise.resolve().then(fn).finally(() => { running = null; }));
+}

package/src/runner/spool.js

@@ -1,5 +1,5 @@
 import { createHash } from 'node:crypto';
-import { readFileSync, readdirSync } from 'node:fs';
+import { closeSync, fstatSync, openSync, readFileSync, readSync, readdirSync, statSync } from 'node:fs';
 import path from 'node:path';
 
 const SPOOL_FILE = /^\d{8}\.jsonl$/;
@@ -42,28 +42,61 @@ function completeLines(text) {
   return end < 0 ? [] : text.slice(0, end).split('\n');
 }
 
+/** A spool file from byte `offset` on: the whole file when there is no watermark. */
+function readFrom(file, offset) {
+  if (!offset) return readFileSync(file, 'utf8');
+  const fd = openSync(file, 'r');
+  try {
+    const size = fstatSync(fd).size;
+    const buffer = Buffer.alloc(Math.max(0, size - offset));
+    readSync(fd, buffer, 0, buffer.length, offset);
+    return buffer.toString('utf8');
+  } finally { closeSync(fd); }
+}
+
 /**
  * Imports every spool line not imported before, oldest file first, each in its own capture transaction with its
  * pulse_spool row, so a crash between lines imports the rest exactly once on the next pass.
+ *
+ * `marks` (a Map the caller keeps) remembers how far each file was read: a file whose size and mtime have not moved
+ * is not read, and one that grew is read from where the last pass stopped. A file that shrank or was replaced is read
+ * from the start again (its lines are known by their sha256). Once `budgetMs` has passed and the pass has applied a
+ * line, it stops before its next line with `more` set, so the caller can yield the event loop and go on from there.
  */
-export function importSpool({ hub, capture, spoolDir, now = Date.now, log = () => {} }) {
+export function importSpool({ hub, capture, spoolDir, now = Date.now, log = () => {}, marks = null, budgetMs = Infinity }) {
   let files;
   try {
     files = readdirSync(spoolDir).filter((f) => SPOOL_FILE.test(f)).sort();
   } catch (error) {
-    if (error.code === 'ENOENT') return { imported: 0, refused: 0 };
+    if (error.code === 'ENOENT') return { imported: 0, refused: 0, more: false };
     throw error;
   }
+  const started = performance.now();
   const seen = hub.db.prepare('SELECT 1 AS found FROM pulse_spool WHERE sha256 = ?');
   const record = hub.db.prepare('INSERT INTO pulse_spool (sha256, file, line, outcome, imported_at) VALUES (?, ?, ?, ?, ?)');
   let imported = 0;
   let refused = 0;
+  let applied = 0;
+  if (marks) for (const file of marks.keys()) if (!files.includes(file)) marks.delete(file);
   for (const file of files) {
-    const lines = completeLines(readFileSync(path.join(spoolDir, file), 'utf8'));
-    lines.forEach((line, index) => {
-      if (!line.trim()) return;
+    const full = path.join(spoolDir, file);
+    const st = marks ? statSync(full) : null;
+    let mark = marks?.get(file);
+    if (mark && mark.ino === st.ino && mark.size === st.size && mark.mtimeMs === st.mtimeMs) continue;
+    if (mark && (mark.ino !== st.ino || st.size < mark.offset)) mark = null;
+    const lines = completeLines(readFrom(full, mark?.offset ?? 0));
+    let offset = mark?.offset ?? 0;
+    let line0 = mark?.lines ?? 0;
+    for (const [index, line] of lines.entries()) {
+      if (applied && performance.now() - started >= budgetMs) {
+        marks?.set(file, { ino: st.ino, size: null, mtimeMs: null, offset, lines: line0 + index });
+        return { imported, refused, more: true };
+      }
+      offset += Buffer.byteLength(line, 'utf8') + 1;
+      if (!line.trim()) continue;
       const sha = createHash('sha256').update(line).digest('hex');
-      if (seen.get(sha)) return;
+      if (seen.get(sha)) continue;
+      applied += 1;
       capture.tx(() => {
         let outcome;
         try {
@@ -73,14 +106,16 @@ export function importSpool({ hub, capture, spoolDir, now = Date.now, log = () =
         } catch {
           outcome = 'refused: unreadable line';
         }
-        record.run(sha, file, index + 1, outcome, now());
+        record.run(sha, file, line0 + index + 1, outcome, now());
         if (outcome === 'applied') imported += 1;
         else {
           refused += 1;
-          log('spool_line_refused', { file, line: index + 1, outcome });
+          log('spool_line_refused', { file, line: line0 + index + 1, outcome });
         }
       }, { label: 'spool import' });
-    });
+    }
+    line0 += lines.length;
+    marks?.set(file, { ino: st.ino, size: st.size, mtimeMs: st.mtimeMs, offset, lines: line0 });
   }
-  return { imported, refused };
+  return { imported, refused, more: false };
 }