@agentpulselabs/cli-plugin-pulse 1.1.3 → 1.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentpulselabs/cli-plugin-pulse",
-  "version": "1.1.3",
+  "version": "1.1.6",
   "description": "AgentPulse CLI plugin for installing and operating the Pulse local connector (replaces the deprecated build-discipline Pulse plugin).",
   "type": "module",
   "license": "MIT",
@@ -21,8 +21,9 @@
   "scripts": {
     "test": "node --import ./test/support/register-loader.mjs --test \"test/*.test.js\" \"test/hub/*.test.js\"",
     "contract:check": "node scripts/sync-contract.mjs --local",
-    "check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cutover.js && node --check src/lib/cutoverWiring.js && node --check src/lib/cutoverRequest.js && node --check src/commands/pulse/cutover.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
+    "check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cutover.js && node --check src/lib/cutoverWiring.js && node --check src/lib/cutoverRequest.js && node --check src/commands/pulse/cutover.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/loopMonitor.js && node --check src/runner/channelWalk.js && node --check src/runner/serial.js && node --check src/runner/directoryCache.js && node --check src/runner/channelFiles.js && node --check src/runner/fileReads.js && node --check src/runner/fileReadsWorker.js && node --check src/runner/transcriptFeed.js && node --check src/runner/collectWorker.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
     "test:t2": "node --import ./test/support/register-loader.mjs --test --test-timeout=120000 \"test/t2/*.test.js\"",
+    "test:perf": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=900000 \"test/perf/*.test.js\"",
     "test:t3": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=240000 \"test/t3/*.test.js\""
   },
   "engines": {

package/src/hub/core/db.mjs

@@ -3,6 +3,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
+import { isBusy } from './ticks.mjs';
 
 export const SCHEMA_VERSION = 1;
 
@@ -231,27 +232,61 @@ export function withTx(db, fn, label = null) {
 }
 
 const pause = (ms) => { if (ms > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); };
+/** How long a busy chunk of writeInChunks waits before each new try; the wait blocks its thread, the collect worker's. */
+const BUSY_PAUSES_MS = [250, 500, 1000, 2000];
 
 /**
  * Runs `steps` (functions) in as many short write transactions as it takes for none to hold the lock past `budgetMs`,
  * pausing `pauseMs` between them so writers on other connections get in. Inside an open transaction it simply joins it.
- * Returns how many transactions it used.
+ * A chunk that finds the database busy past busy_timeout is rolled back and run again from its first step, after
+ * `busyPausesMs`, so another writer's long transaction delays a pass instead of failing it. Returns how many
+ * transactions it used.
  */
-export function writeInChunks(db, steps, { budgetMs = 50, pauseMs = 15, label = 'chunked write', onPause = null } = {}) {
+export function writeInChunks(db, steps, { budgetMs = 50, pauseMs = 15, label = 'chunked write', onPause = null, busyPausesMs = BUSY_PAUSES_MS } = {}) {
   if (TX.get(db)) { for (const step of steps) step(); return 0; }
   let i = 0;
   let chunks = 0;
+  let busy = 0;
   while (i < steps.length) {
     if (chunks) { pause(pauseMs); if (onPause) onPause(); }
     const started = performance.now();
-    withTx(db, () => {
-      do steps[i++](); while (i < steps.length && performance.now() - started < budgetMs);
-    }, `${label} (chunk ${chunks + 1})`);
+    const first = i;
+    try {
+      withTx(db, () => {
+        do steps[i++](); while (i < steps.length && performance.now() - started < budgetMs);
+      }, `${label} (chunk ${chunks + 1})`);
+    } catch (e) {
+      if (!isBusy(e) || busy >= busyPausesMs.length) throw e;
+      txTiming.log(`${label}: database busy, chunk ${chunks + 1} again in ${busyPausesMs[busy]} ms`);
+      pause(busyPausesMs[busy++]);
+      i = first;
+      continue;
+    }
+    busy = 0;
     chunks++;
   }
   return chunks;
 }
 
+/**
+ * Pulse runner-stalls round 2: when discovery last saved a pass. A pass rewrites only the rows that moved, so a row's
+ * updated_at no longer says when a pass last confirmed it; this does. Kept out of the facts, where capture and the
+ * change feed would see it on every pass.
+ */
+export function recordDiscoveryPass(db, at = Date.now()) {
+  runSql(db, 'CREATE TABLE IF NOT EXISTS discovery_pass (id INTEGER PRIMARY KEY CHECK (id = 1), at INTEGER NOT NULL)');
+  db.prepare('INSERT INTO discovery_pass (id, at) VALUES (1, ?) ON CONFLICT (id) DO UPDATE SET at = excluded.at').run(at);
+}
+
+/** When discovery last saved a pass (recordDiscoveryPass), or 0 before the first. */
+export function lastDiscoveryPass(db) {
+  try {
+    return Number(db.prepare('SELECT at FROM discovery_pass WHERE id = 1').get()?.at) || 0;
+  } catch {
+    return 0;
+  }
+}
+
 export function jsonCol(v) {
   if (v == null) return null;
   return typeof v === 'string' ? v : JSON.stringify(v);

package/src/hub/core/wake.mjs

@@ -4,6 +4,7 @@
 // and again after WAKE_AGAIN_MS if your message is still waiting.
 import { ADDRESS, managerChats } from './ids.mjs';
 import { edgeId } from './edgeIds.mjs';
+import { lastDiscoveryPass } from './db.mjs';
 
 export const WAKE_AGAIN_MS = 15 * 60e3;
 /** No hook event for this long: the subagent is not running (long commands can be silent for minutes). */
@@ -59,8 +60,9 @@ const nudgeText = (every) => `Scheduled check-in from the hub (every ${every} mi
 /**
  * Agents finished by what the Board shows: a stream whose final review passed, or a done agent row or delivery. Rows
  * and deliveries are only as fresh as the last collect, so one older than the agent's last sign of life does not count.
+ * A row is as fresh as the last saved pass (`passAt`), which rewrites only the rows that moved.
  */
-function doneIds(registry, actors, seen) {
+function doneIds(registry, actors, seen, passAt = 0) {
   const rows = new Map(registry.facts('agent-row').map((r) => [r.key, r]));
   const deliveries = new Map(registry.facts('delivery').map((d) => [d.key, d]));
   const done = new Set();
@@ -69,7 +71,7 @@ function doneIds(registry, actors, seen) {
     const key = a.stream_key || (row && row.streamKey);
     const stream = key && registry.fact('stream', key);
     const d = a.node_type === 'orchestrator' && deliveries.get(a.id);
-    const current = (f) => f && f.status === 'done' && f.updated_at >= (seen.get(a.id) || 0);
+    const current = (f) => f && f.status === 'done' && Math.max(f.updated_at, passAt) >= (seen.get(a.id) || 0);
     if ((stream && stream.done) || current(row) || current(d)) done.add(a.id);
   }
   return done;
@@ -91,7 +93,7 @@ export function scheduledNudge(hub, { everyMin, now = Date.now(), skip = null, r
   for (const chat of managerChats(registry.fact('manager', 'config'))) exclude.add(chat);
   if (skip) for (const id of skip(last)) exclude.add(id);
   if (released) for (const id of released()) pending.delete(id);
-  const targets = nudgeTargets({ actors, seen, done: doneIds(registry, actors, seen), pending, exclude, now });
+  const targets = nudgeTargets({ actors, seen, done: doneIds(registry, actors, seen, lastDiscoveryPass(db)), pending, exclude, now });
   const dir = targets.length ? registry.directory(now) : [];
   const sent = [];
   for (const t of targets) {

package/src/hub/workers/collector.mjs

@@ -2,7 +2,7 @@
 // Discovery worker: scans transcripts and channels, writes inferred actors and facts. Does not serve GET.
 import crypto from 'node:crypto';
 import { DERIVED_REQUEST_PREFIX, collect } from '../lib/collect.mjs';
-import { writeInChunks } from '../core/db.mjs';
+import { recordDiscoveryPass, writeInChunks } from '../core/db.mjs';
 import { nodeTypeOf } from '../core/ids.mjs';
 import { projectGraph } from '../projections/graph.mjs';
 
@@ -16,6 +16,15 @@ export const DISCOVERY_REUSE_MS = 30e3;
 export const CANONICAL_HOURS = 168;
 
 const lastWritten = new WeakMap();
+/**
+ * Pulse runner-stalls: per db, every row the last saved pass wrote, as hashes of what it wrote and of the row it left.
+ * A row whose input and current row both still match is not written again, so a pass where little moved takes the
+ * write lock, and gives capture journal rows, only for what did.
+ */
+const lastRows = new WeakMap();
+const rowHash = (text) => crypto.createHash('sha1').update(String(text)).digest('base64');
+/** The message ids conversations.importTranscriptTurn and forgetOpening use for a session's opening turn. */
+const openingIds = (id) => { const base = `t-${`${id}-open`.slice(0, 40)}`; return [`${base}-p`, `${base}-r`]; };
 
 /** What a pass would write, minus what moves on every pass by itself (generatedAt, the manager node's window). */
 function discoverySignature(registry, graph) {
@@ -26,7 +35,7 @@ function discoverySignature(registry, graph) {
 }
 
 /**
- * Saves a discovery pass. Returns false, writing nothing, when it matches the last pass saved through this db. A fact
+ * Saves a discovery pass. Returns false, writing no rows (only the pass time), when it matches the last pass saved through this db. A fact
  * kind written here must be listed in DISCOVERY_FACTS (core/changes.mjs) or each pass wakes the change feed.
  *
  * A full pass is ~1.5k actor and ~1.1k fact writes (0.3-0.4 s on the live hub.db), so it goes in short transactions
@@ -36,18 +45,39 @@ function discoverySignature(registry, graph) {
 export function persistDiscovery(hub, graph, chunks = {}) {
   const { registry, conversations, db } = hub;
   const signature = discoverySignature(registry, graph);
-  if (lastWritten.get(db) === signature) return false;
+  if (lastWritten.get(db) === signature) {
+    // Nothing moved, but this pass still confirmed every row (core/wake.mjs doneIds reads when).
+    writeInChunks(db, [() => recordDiscoveryPass(db)], { label: 'discovery pass', ...chunks });
+    return false;
+  }
   const now = graph.generatedAt || Date.now();
+  const rows = lastRows.get(db) || new Map();
+  const kept = new Map();
   const steps = [];
-  const step = (fn) => steps.push(fn);
-  for (const n of graph.nodes || []) step(() => {
+  /** A keyed step is skipped while `input` and the row `read` returns are what the last saved pass left. */
+  const step = (fn, key = null, input = null, read = null) => {
+    if (key === null) { steps.push(fn); return; }
+    const want = rowHash(JSON.stringify(input));
+    const was = rows.get(key);
+    if (was && was.input === want && was.row === rowHash(read())) { kept.set(key, was); return; }
+    steps.push(() => { fn(); kept.set(key, { input: want, row: rowHash(read()) }); });
+  };
+  const actorRow = db.prepare('SELECT * FROM actors WHERE id = ?');
+  const factRow = db.prepare('SELECT payload FROM facts WHERE kind = ? AND key = ?');
+  const factKeys = db.prepare("SELECT kind, key FROM facts WHERE kind IN ('delivery', 'stream', 'agent-row') ORDER BY kind, key");
+  const restRows = db.prepare("SELECT kind, key, payload FROM facts WHERE kind IN ('request', 'totals') ORDER BY kind, key");
+  const openingRows = db.prepare('SELECT m.id, d.recipient FROM messages m LEFT JOIN deliveries d ON d.message_id = m.id WHERE m.id IN (?, ?) ORDER BY m.id');
+  const readActor = (id) => () => JSON.stringify(actorRow.get(id) ?? null);
+  const readFact = (kind, key) => () => factRow.get(kind, key)?.payload ?? '';
+  const readOpening = (id) => () => JSON.stringify(openingRows.all(...openingIds(id)));
+  for (const n of graph.nodes || []) {
     const parentId = n.parent || n.launchedBy || null;
     const synthetic = !!n.synthetic;
     const nodeType = n.node_type || nodeTypeOf({
       id: n.id, parentId, synthetic, role: n.role,
     });
     const persona = n.persona || {};
-    registry.upsertInferred({
+    const patch = {
       id: n.id,
       node_type: nodeType,
       role: n.role || (nodeType === 'orchestrator' ? 'ARCHITECT' : null),
@@ -65,22 +95,29 @@ export function persistDiscovery(hub, graph, chunks = {}) {
       started_at: n.startedAt, last_at: n.lastAt, tokens_est: n.tokensEst,
       msgs_in: n.msgsIn, msgs_out: n.msgsOut, repo: n.repo || n.project, channel: n.channel,
       synthetic, extra: { streams: n.streams, status: n.status, kind: n.kind },
-    });
+    };
+    step(() => registry.upsertInferred(patch), `actor:${n.id}`, patch, readActor(n.id));
     // Presence is owned by the hook heartbeat. Discovery only records last_at on the actor.
-  });
+  }
   const deliveries = graph.deliveries || [];
   const accepted = new Map(registry.facts('accepted').map((a) => [a.key, a.at || a.updated_at]));
-  for (const d of deliveries) step(() => registry.setFact('delivery', d.id, accepted.has(d.id) ? { ...d, acceptedAt: accepted.get(d.id) } : d));
-  for (const d of deliveries) for (const s of d.streams || []) step(() => registry.setFact('stream', s.key, { ...s, deliveryId: d.id }));
-  for (const a of graph.agents || []) step(() => registry.setFact('agent-row', a.id, a));
+  const setFact = (kind, key, payload) => step(() => registry.setFact(kind, key, payload), `fact:${kind}:${key}`, payload, readFact(kind, key));
+  for (const d of deliveries) setFact('delivery', d.id, accepted.has(d.id) ? { ...d, acceptedAt: accepted.get(d.id) } : d);
+  for (const d of deliveries) for (const s of d.streams || []) setFact('stream', s.key, { ...s, deliveryId: d.id });
+  for (const a of graph.agents || []) setFact('agent-row', a.id, a);
+  const keys = {
+    delivery: deliveries.map((d) => d.id),
+    stream: deliveries.flatMap((d) => (d.streams || []).map((s) => s.key)),
+    'agent-row': (graph.agents || []).map((a) => a.id),
+  };
   step(() => {
-    registry.keepFacts('delivery', deliveries.map((d) => d.id));
-    registry.keepFacts('stream', deliveries.flatMap((d) => (d.streams || []).map((s) => s.key)));
-    registry.keepFacts('agent-row', (graph.agents || []).map((a) => a.id));
-  });
-  for (const n of graph.nodes || []) if (n.kind === 'agent') step(() => conversations.forgetOpening(n.id));
+    registry.keepFacts('delivery', keys.delivery);
+    registry.keepFacts('stream', keys.stream);
+    registry.keepFacts('agent-row', keys['agent-row']);
+  }, 'keep', keys, () => JSON.stringify(factKeys.all()));
+  for (const n of graph.nodes || []) if (n.kind === 'agent') step(() => conversations.forgetOpening(n.id), `forget:${n.id}`, null, readOpening(n.id));
   const requests = (graph.requests || []).map((r) => [r.id || `r-${r.at}`, r]);
-  for (const [key, r] of requests) step(() => registry.setFact('request', key, r));
+  for (const [key, r] of requests) setFact('request', key, r);
   step(() => {
     // Only post-derived requests are discovery's to drop; filed ones (hub.mjs request) live as facts alone.
     const current = new Set(requests.map(([key]) => key));
@@ -90,18 +127,21 @@ export function persistDiscovery(hub, graph, chunks = {}) {
     // The projection counts each window's totals from this one basis; per-window facts older passes wrote are dropped.
     if (graph.totalsBasis) registry.setFact('totals', 'basis', graph.totalsBasis);
     registry.keepFacts('totals', graph.totalsBasis ? ['basis'] : []);
-  });
+  }, 'requests', [[...requests.map(([key]) => key)].sort(), graph.totalsBasis ?? null], () => JSON.stringify(restRows.all()));
   for (const n of graph.nodes || []) {
     if (n.kind !== 'session' || n.parent || n.launchedBy) continue;
     if (n.promptTitle || n.title) {
+      // A transcript turn is inserted once and never updated, so a pass's own clock is no part of its input.
       step(() => conversations.importTranscriptTurn({
         sessionId: n.id, turnId: `${n.id}-open`, at: n.startedAt || now,
         prompt: n.promptTitle || n.title, reply: null,
-      }));
+      }), `opening:${n.id}`, [n.startedAt || null, n.promptTitle || n.title], readOpening(n.id));
     }
   }
+  step(() => recordDiscoveryPass(db));
   writeInChunks(db, steps, { label: 'discovery', ...chunks });
   lastWritten.set(db, signature);
+  lastRows.set(db, kept);
   return true;
 }
 

package/src/hub/workers/manager.mjs

@@ -20,6 +20,10 @@ const MAX_OUTPUT = 2 * 1024 * 1024;
 /** Answers took 0.7-9.8 min in one chat resumed for two days; a fresh, small chat answers well inside this. */
 export const RUN_TIMEOUT_MS = 5 * 60e3;
 const MAX_ATTEMPTS = 3;
+/** Claims also count a run's own lapsed claim taken back after a stall; past this many, a message no run finishes is given up. */
+const MAX_CLAIMS = 10;
+/** A run that ends while the lease is away (a stalled runner lapses it for a moment) waits this long for it to come back. */
+export const HELD_WAIT_MS = 60e3;
 /** Runs cut off by restarts do not use up attempts; past this many restarts a message is given up, not looped on. */
 export const MAX_RESTARTS = 3;
 /** Runs at once, one chat each. One, as before: a second question waits one run at most, FYIs none. */
@@ -129,10 +133,19 @@ ${voice || ''}`.trim();
 const topicOf = (sender) => (sender === 'user' ? 'user' : `agent/${sender}`);
 const errorText = (error) => String(error && error.message ? error.message : error);
 
+/** Why a cursor-agent run failed (exit, signal, stderr), never its command line: that holds the whole prompt. */
+function runError(error) {
+  if (!error || !String(error.message || '').startsWith('Command failed:')) return errorText(error);
+  const how = error.killed ? `timed out (${error.signal})` : error.code != null ? `exit ${error.code}` : `signal ${error.signal}`;
+  const said = String(error.stderr || error.stdout || '').trim().split('\n').slice(-3).join(' ').slice(0, 300);
+  return `cursor-agent ${how}${said ? `: ${said}` : ''}`;
+}
+
 export function createManager({
   hub, state, run = runFile, onChange = () => {}, parallel = PARALLEL, now = Date.now, log = console.error, ackRetryMs = RETRY_MS,
   held = () => true,
   fyi = null,
+  heldWaitMs = HELD_WAIT_MS,
 }) {
   const configFile = path.join(state, 'manager.json');
   const statusFile = path.join(state, 'manager-status.json');
@@ -143,6 +156,8 @@ export function createManager({
   const { messaging, registry, leases } = hub;
   /** message id → { message, leaseToken, slot, startedAt, abort } */
   const inflight = new Map();
+  /** message id → { count, error } of its runs that failed; claims alone are no attempts. */
+  const failures = new Map();
   const runs = new Set();
   let lastHeartbeat = 0;
   let timer = null;
@@ -222,6 +237,15 @@ export function createManager({
     }, 'manager answer'), { delaysMs: ackRetryMs, label: `manager answer to ${m.id}`, log });
   }
 
+  /** Only the lease holder settles; a stall that lapsed the lease here gets it back moments later. */
+  async function heldAgain(signal) {
+    for (let waited = 0; !held(); waited += 250) {
+      if (signal.aborted || waited >= heldWaitMs) return false;
+      await new Promise((resolve) => setTimeout(resolve, 250));
+    }
+    return !signal.aborted;
+  }
+
   async function answer(f) {
     const { message: m, slot, abort } = f;
     const chat = await chatFor(slot, abort.signal);
@@ -236,6 +260,10 @@ export function createManager({
     if (envelope && envelope.is_error) throw new Error(`cursor-agent failed: ${String(textOf(envelope) || 'error').slice(0, 300)}`);
     const actions = parseManagerActions(stdout);
     if (!actions.reply) throw new Error('manager returned no reply');
+    if (!(await heldAgain(abort.signal))) {
+      if (!abort.signal.aborted) log(`manager answer to ${m.id} held back: the manager lease is not held here`);
+      return null;
+    }
     await settle(m, actions);
     recordTurn(slot, chat.chatId, m, prompt, actions.reply);
     if (registry.fact('manager-restart', m.id)) registry.dropFact('manager-restart', m.id);
@@ -252,11 +280,16 @@ export function createManager({
     const done = answer(f).then((replied) => ({ replied }), (error) => ({ error })).then(({ replied, error }) => {
       inflight.delete(m.id);
       runs.delete(done);
-      if (f.abort.signal.aborted) return;
-      if (error) log(`manager failed for ${m.id}: ${errorText(error)}`);
+      if (f.abort.signal.aborted) { log(`manager run for ${m.id} was stopped before it finished; it is not counted as a try`); return; }
+      const why = error ? runError(error).slice(0, 500) : null;
+      if (error) {
+        const failed = { count: ((failures.get(m.id) || {}).count || 0) + 1, error: why };
+        failures.set(m.id, failed);
+        log(`manager failed for ${m.id} (try ${failed.count} of ${MAX_ATTEMPTS}): ${why}`);
+      } else if (replied) failures.delete(m.id);
       status({
         ...(replied || {}), state: error ? 'error' : inflight.size ? 'working' : 'online', messageId: error ? m.id : null,
-        error: error ? errorText(error).slice(0, 500) : null, inflight: inflightView(),
+        error: why, inflight: inflightView(), ...(error ? { lastFailure: { id: m.id, from: m.from, error: why, at: now() } } : {}),
       });
     }).catch((e) => log(`manager status after ${m.id} failed: ${errorText(e)}`));
     runs.add(done);
@@ -315,9 +348,14 @@ export function createManager({
       const own = inflight.get(message.id);
       if (own) { own.leaseToken = message.leaseToken; continue; }
       if (isFyi(message)) { if (!(fyi && fyi(message))) messaging.ack('manager', message.id, {}, registry.directory()); continue; }
-      if (message.attempts > MAX_ATTEMPTS) {
-        const last = readJson(statusFile, {}).error || 'unknown error';
-        messaging.ack('manager', message.id, { reply: `I could not answer this after ${MAX_ATTEMPTS} tries, so I stopped retrying. Last error: ${last}`, fromName: 'Manager' }, registry.directory());
+      const failed = failures.get(message.id);
+      if ((failed && failed.count >= MAX_ATTEMPTS) || message.attempts > MAX_CLAIMS) {
+        const tries = failed ? failed.count : message.attempts - 1;
+        const last = failed ? failed.error : `none of its ${tries} runs finished (each was cut off by a restart or a lost lease)`;
+        messaging.ack('manager', message.id, { reply: `I could not answer this after ${tries} tries, so I stopped retrying. Last error: ${last}`, fromName: 'Manager' }, registry.directory());
+        failures.delete(message.id);
+        log(`manager gave up on ${message.id} after ${tries} tries: ${last}`);
+        status({ gaveUp: { id: message.id, from: message.from, tries, error: last, at: t } });
         continue;
       }
       dispatch(message);

package/src/lib/contract.sha256

@@ -1,2 +1,2 @@
-ce1142e1e133c8aff45a999d0d4b67faa6177f420ddcd1b449af81f8e806d8d0  connector-contract.json
+c9db92b38d0c978d4fa18d8a731fadbf3013a19659b524d615583ad099a8fafc  connector-contract.json
 7b7a59c08797c100b46436123dfcb09b517730357fa90cd0890da9b80308faf3  mirror-contract.json

package/src/lib/daemonRuntime.js

@@ -152,6 +152,7 @@ export class DaemonRuntime {
           attemptNo: attempt.attemptNo,
           fenceToken: attempt.fenceToken,
           events,
+          itemRefusals: true,
         });
       } catch (error) {
         if (machineWideError(error)) throw error;
@@ -164,15 +165,32 @@ export class DaemonRuntime {
         this.log('events_discarded', { ...trace, discarded });
         continue;
       }
-      if (Array.isArray(response.events)) {
-        await this.ledger.acknowledgeEvents(
-          response.events.map((event) => ({
-            sourceEventId: event.sourceEventId,
-            cloudCursor: event.cloudCursor,
-          }))
-        );
+      await this.#settleUpload(attempt, events, response);
+    }
+  }
+
+  /**
+   * Apply the cloud's answer to one uploaded batch. A stored event comes back with its `cursor` (`cloudCursor` is read
+   * when that is the only one) and leaves the outbox. A refused one ({ refused }) is dropped on its own, so the rest of
+   * the batch is kept. An event the answer leaves out stays pending and is sent again.
+   */
+  async #settleUpload(attempt, events, response) {
+    if (!Array.isArray(response?.events)) return;
+    const sent = new Set(events.map((event) => event.sourceEventId));
+    const accepted = [];
+    const refused = new Map();
+    for (const item of response.events) {
+      if (!sent.has(item?.sourceEventId)) continue;
+      if (item.refused) refused.set(item.sourceEventId, item.refused.code || 'HUB_EVENT_REFUSED');
+      else if (Number.isSafeInteger(item.cursor ?? item.cloudCursor)) {
+        accepted.push({ sourceEventId: item.sourceEventId, cursor: item.cursor ?? item.cloudCursor });
       }
     }
+    if (accepted.length) await this.ledger.acknowledgeEvents(accepted);
+    if (!refused.size) return;
+    const codes = [...new Set(refused.values())].sort().join(',');
+    const { discarded } = await this.ledger.discardEvents(attempt.jobId, codes, { sourceEventIds: [...refused.keys()] });
+    this.log('events_refused', { jobId: attempt.jobId, type: attempt.jobType, attemptNo: attempt.attemptNo, code: codes, discarded });
   }
 
   #compose(grant) {

package/src/lib/hubDbReader.js

@@ -129,4 +129,10 @@ export class HubSnapshot {
     const statement = this.db.prepare('SELECT kind, key, payload, updated_at FROM facts WHERE kind = ? ORDER BY key');
     return kinds.flatMap((kind) => statement.all(kind).map(plain));
   }
+
+  /** The `{ kind, key }` of the same facts in the same order, without their payloads. */
+  factKeys(kinds) {
+    const statement = this.db.prepare('SELECT kind, key FROM facts WHERE kind = ? ORDER BY key');
+    return kinds.flatMap((kind) => statement.all(kind).map(plain));
+  }
 }

package/src/lib/hubScan.js

@@ -17,6 +17,7 @@ export const SKIPPED_FACT_KINDS = Object.freeze({
   run: 'machine_runtime_state',
   wake: 'machine_runtime_state',
   manager: 'machine_runtime_state',
+  'manager-restart': 'machine_runtime_state',
   totals: 'phase1_deviation_totals',
 });
 export const REMOVABLE_KINDS = Object.freeze(['delivery', 'stream', 'agent-row', 'request', 'status_override']);

package/src/lib/ledger.js

@@ -226,6 +226,7 @@ export class DurableLedger {
     return clone(this.data.outbox.slice(0, Math.max(0, Math.min(100, limit))));
   }
 
+  /** Events the cloud stored, as { sourceEventId, cursor } with the cloud cursor it gave each, the wire's names. */
   async acknowledgeEvents(acknowledgements) {
     return this.#mutate((state) => {
       if (!Array.isArray(acknowledgements) || acknowledgements.length > 100) {
@@ -237,15 +238,15 @@ export class DurableLedger {
         if (
           !pendingIds.has(item.sourceEventId) ||
           seen.has(item.sourceEventId) ||
-          !Number.isSafeInteger(item.cloudCursor) ||
-          item.cloudCursor < 1
+          !Number.isSafeInteger(item.cursor) ||
+          item.cursor < 1
         ) {
           throw new HubError('HUB_EVENT_ACK_INVALID', 'Event acknowledgement does not match the sent batch');
         }
         seen.add(item.sourceEventId);
       }
       const accepted = new Map(
-        acknowledgements.map((item) => [item.sourceEventId, Number(item.cloudCursor)])
+        acknowledgements.map((item) => [item.sourceEventId, Number(item.cursor)])
       );
       for (const event of state.outbox) {
         if (!accepted.has(event.sourceEventId) || !TERMINAL_STATES.has(event.terminalState)) continue;
@@ -465,19 +466,27 @@ export class DurableLedger {
   }
 
   /**
-   * Drop a job's pending events the cloud refused for good (its fence is gone or the batch is invalid). They never
-   * count toward acceptedLocalCursor, and a terminal attempt has nothing left to upload.
+   * Drop a job's pending events the cloud refused for good: all of them (its fence is gone or the batch is invalid),
+   * or only sourceEventIds (the cloud refused those items and stored the rest). They never count toward
+   * acceptedLocalCursor. A terminal attempt has nothing left to upload once its terminal event is gone.
    */
-  async discardEvents(jobId, code) {
+  async discardEvents(jobId, code, { sourceEventIds = null } = {}) {
+    const only = sourceEventIds ? new Set(sourceEventIds) : null;
     return this.#mutate((state) => {
-      const discarded = state.outbox.filter((event) => event.jobId === jobId).length;
-      state.outbox = state.outbox.filter((event) => event.jobId !== jobId);
+      const dropped = (event) => event.jobId === jobId && (!only || only.has(event.sourceEventId));
+      const gone = state.outbox.filter(dropped);
+      state.outbox = state.outbox.filter((event) => !dropped(event));
       const attempt = state.attempts[jobId];
-      if (attempt && discarded) {
-        attempt.eventsDiscarded = { count: discarded, code, at: new Date().toISOString() };
-        if (TERMINAL_STATES.has(attempt.state)) attempt.terminalUploaded = true;
+      if (attempt && gone.length) {
+        attempt.eventsDiscarded = {
+          count: (only ? attempt.eventsDiscarded?.count || 0 : 0) + gone.length,
+          code,
+          at: new Date().toISOString(),
+        };
+        const terminalGone = !only || gone.some((event) => event.terminalState === attempt.state);
+        if (TERMINAL_STATES.has(attempt.state) && terminalGone) attempt.terminalUploaded = true;
       }
-      return { discarded };
+      return { discarded: gone.length };
     });
   }
 

package/src/lib/lifecycle.js

@@ -3,10 +3,12 @@ import { HubError } from './errors.js';
 import { validateJobResult } from './intents.js';
 import { redactText } from './redaction.js';
 
+// Hex, so the id always starts with a letter or digit: Pulse clouds before 1.1.3 refuse a sourceEventId starting with
+// '-' or '_', which a base64url digest does about 3% of the time.
 function eventId(tuple, type) {
   return createHash('sha256')
     .update(`${tuple.jobId}\0${tuple.attemptNo}\0${tuple.fenceToken}\0${type}`)
-    .digest('base64url');
+    .digest('hex');
 }
 
 export class AttemptCoordinator {

package/src/lib/mirrorSource.js

@@ -150,6 +150,7 @@ export class MirrorSource {
     this.redaction = new UploadRedactor({ workspace, home, approvedRoots });
     this.windowCache = new Map();
     this.rowMemo = new RowMemo();
+    this.itemMemo = new RowMemo();
     this.briefs = new Map();
     this.taskKeys = null;
   }
@@ -388,12 +389,18 @@ export class MirrorSource {
     checkpoint();
     const secrets = [await this.reader.tokenValue().catch(() => null), ...this.heldSecrets()].filter(Boolean);
     const redactor = this.redaction.redactor(secrets);
+    const basis = this.redaction.basis(secrets);
     const items = new Map();
+    // An item's redacted, checked form depends on its kind and data and on the redaction basis, so one that did not
+    // move since the last pass is not redacted and checked again.
+    this.itemMemo.begin(basis);
     for (const entry of raw) {
       if (typeof entry.externalId !== 'string' || !entry.externalId) continue;
       const externalId = externalIdFor(entry.externalId);
-      const redacted = redactor.redactValue(entry.data, { keep: entry.kind === 'rule_pack' ? ['file'] : [] });
-      const result = sanitizeItem(entry.kind, entry.kind === 'brief' ? markTruncated(redacted) : redacted, { secrets });
+      const result = this.itemMemo.take(`${entry.kind}\0${JSON.stringify(entry.data)}`, () => {
+        const redacted = redactor.redactValue(entry.data, { keep: entry.kind === 'rule_pack' ? ['file'] : [] });
+        return sanitizeItem(entry.kind, entry.kind === 'brief' ? markTruncated(redacted) : redacted, { secrets });
+      });
       if (result.error) {
         withheld.push({ kind: entry.kind, externalId, error: result.error });
         continue;
@@ -405,19 +412,25 @@ export class MirrorSource {
     const graphDropped = {};
     if (windows.length === WINDOW_HOURS.length) {
       // A row's redacted, checked form depends on the roots the redactor knows and the secrets, and on nothing else.
-      this.rowMemo.begin(this.redaction.basis(secrets));
+      this.rowMemo.begin(basis);
       const options = { redact: (value) => redactor.redactValue(value), secrets, memo: this.rowMemo };
-      const built = mergeWindows(windows.map(({ hours, body, fresh }) => {
+      const perWindow = windows.map(({ hours, body, fresh }) => {
         if (!fresh) return this.windowCache.get(hours).rows;
         const rows = windowRows(hours, body, options);
         this.windowCache.set(hours, { body, rows });
         return rows;
-      }));
+      });
+      // Each checked row's canonical text, so an item is keyed by its windows and that text, not stringified again.
+      const textOf = new Map(perWindow.flatMap(({ rows }) => rows.map(({ row, text }) => [row, text])));
+      const built = mergeWindows(perWindow);
       Object.assign(graphDropped, built.dropped);
       const updatedAt = firstTime(Math.max(...windows.map(({ body }) => Number(body.generatedAt) || 0)), fallback);
       for (const row of built.items) {
         const externalId = externalIdFor(row.externalId);
-        const result = this.rowMemo.take(`item\0${JSON.stringify(row.data)}`, () => sanitizeItem('graph_window', row.data, { secrets }));
+        const { section, windows: held } = row.data;
+        const text = textOf.get(row.data[section]);
+        const key = text === undefined ? JSON.stringify(row.data) : `${section}\0${held.join(',')}\0${text}`;
+        const result = this.rowMemo.take(`item\0${key}`, () => sanitizeItem('graph_window', row.data, { secrets }));
         if (result.error) {
           withheld.push({ kind: 'graph_window', externalId, error: result.error });
           continue;