@agentpulselabs/cli-plugin-pulse 1.1.3 → 1.1.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentpulselabs/cli-plugin-pulse",
-  "version": "1.1.3",
+  "version": "1.1.5",
   "description": "AgentPulse CLI plugin for installing and operating the Pulse local connector (replaces the deprecated build-discipline Pulse plugin).",
   "type": "module",
   "license": "MIT",
@@ -21,8 +21,9 @@
   "scripts": {
     "test": "node --import ./test/support/register-loader.mjs --test \"test/*.test.js\" \"test/hub/*.test.js\"",
     "contract:check": "node scripts/sync-contract.mjs --local",
-    "check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cutover.js && node --check src/lib/cutoverWiring.js && node --check src/lib/cutoverRequest.js && node --check src/commands/pulse/cutover.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
+    "check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cutover.js && node --check src/lib/cutoverWiring.js && node --check src/lib/cutoverRequest.js && node --check src/commands/pulse/cutover.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/loopMonitor.js && node --check src/runner/channelWalk.js && node --check src/runner/serial.js && node --check src/runner/directoryCache.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
     "test:t2": "node --import ./test/support/register-loader.mjs --test --test-timeout=120000 \"test/t2/*.test.js\"",
+    "test:perf": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=900000 \"test/perf/*.test.js\"",
     "test:t3": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=240000 \"test/t3/*.test.js\""
   },
   "engines": {

package/src/hub/core/db.mjs

@@ -3,6 +3,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
+import { isBusy } from './ticks.mjs';
 
 export const SCHEMA_VERSION = 1;
 
@@ -231,22 +232,37 @@ export function withTx(db, fn, label = null) {
 }
 
 const pause = (ms) => { if (ms > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); };
+/** How long a busy chunk of writeInChunks waits before each new try; the wait blocks its thread, the collect worker's. */
+const BUSY_PAUSES_MS = [250, 500, 1000, 2000];
 
 /**
  * Runs `steps` (functions) in as many short write transactions as it takes for none to hold the lock past `budgetMs`,
  * pausing `pauseMs` between them so writers on other connections get in. Inside an open transaction it simply joins it.
- * Returns how many transactions it used.
+ * A chunk that finds the database busy past busy_timeout is rolled back and run again from its first step, after
+ * `busyPausesMs`, so another writer's long transaction delays a pass instead of failing it. Returns how many
+ * transactions it used.
  */
-export function writeInChunks(db, steps, { budgetMs = 50, pauseMs = 15, label = 'chunked write', onPause = null } = {}) {
+export function writeInChunks(db, steps, { budgetMs = 50, pauseMs = 15, label = 'chunked write', onPause = null, busyPausesMs = BUSY_PAUSES_MS } = {}) {
   if (TX.get(db)) { for (const step of steps) step(); return 0; }
   let i = 0;
   let chunks = 0;
+  let busy = 0;
   while (i < steps.length) {
     if (chunks) { pause(pauseMs); if (onPause) onPause(); }
     const started = performance.now();
-    withTx(db, () => {
-      do steps[i++](); while (i < steps.length && performance.now() - started < budgetMs);
-    }, `${label} (chunk ${chunks + 1})`);
+    const first = i;
+    try {
+      withTx(db, () => {
+        do steps[i++](); while (i < steps.length && performance.now() - started < budgetMs);
+      }, `${label} (chunk ${chunks + 1})`);
+    } catch (e) {
+      if (!isBusy(e) || busy >= busyPausesMs.length) throw e;
+      txTiming.log(`${label}: database busy, chunk ${chunks + 1} again in ${busyPausesMs[busy]} ms`);
+      pause(busyPausesMs[busy++]);
+      i = first;
+      continue;
+    }
+    busy = 0;
     chunks++;
   }
   return chunks;

package/src/hub/workers/manager.mjs

@@ -20,6 +20,10 @@ const MAX_OUTPUT = 2 * 1024 * 1024;
 /** Answers took 0.7-9.8 min in one chat resumed for two days; a fresh, small chat answers well inside this. */
 export const RUN_TIMEOUT_MS = 5 * 60e3;
 const MAX_ATTEMPTS = 3;
+/** Claims also count a run's own lapsed claim taken back after a stall; past this many, a message no run finishes is given up. */
+const MAX_CLAIMS = 10;
+/** A run that ends while the lease is away (a stalled runner lapses it for a moment) waits this long for it to come back. */
+export const HELD_WAIT_MS = 60e3;
 /** Runs cut off by restarts do not use up attempts; past this many restarts a message is given up, not looped on. */
 export const MAX_RESTARTS = 3;
 /** Runs at once, one chat each. One, as before: a second question waits one run at most, FYIs none. */
@@ -129,10 +133,19 @@ ${voice || ''}`.trim();
 const topicOf = (sender) => (sender === 'user' ? 'user' : `agent/${sender}`);
 const errorText = (error) => String(error && error.message ? error.message : error);
 
+/** Why a cursor-agent run failed (exit, signal, stderr), never its command line: that holds the whole prompt. */
+function runError(error) {
+  if (!error || !String(error.message || '').startsWith('Command failed:')) return errorText(error);
+  const how = error.killed ? `timed out (${error.signal})` : error.code != null ? `exit ${error.code}` : `signal ${error.signal}`;
+  const said = String(error.stderr || error.stdout || '').trim().split('\n').slice(-3).join(' ').slice(0, 300);
+  return `cursor-agent ${how}${said ? `: ${said}` : ''}`;
+}
+
 export function createManager({
   hub, state, run = runFile, onChange = () => {}, parallel = PARALLEL, now = Date.now, log = console.error, ackRetryMs = RETRY_MS,
   held = () => true,
   fyi = null,
+  heldWaitMs = HELD_WAIT_MS,
 }) {
   const configFile = path.join(state, 'manager.json');
   const statusFile = path.join(state, 'manager-status.json');
@@ -143,6 +156,8 @@ export function createManager({
   const { messaging, registry, leases } = hub;
   /** message id → { message, leaseToken, slot, startedAt, abort } */
   const inflight = new Map();
+  /** message id → { count, error } of its runs that failed; claims alone are no attempts. */
+  const failures = new Map();
   const runs = new Set();
   let lastHeartbeat = 0;
   let timer = null;
@@ -222,6 +237,15 @@ export function createManager({
     }, 'manager answer'), { delaysMs: ackRetryMs, label: `manager answer to ${m.id}`, log });
   }
 
+  /** Only the lease holder settles; a stall that lapsed the lease here gets it back moments later. */
+  async function heldAgain(signal) {
+    for (let waited = 0; !held(); waited += 250) {
+      if (signal.aborted || waited >= heldWaitMs) return false;
+      await new Promise((resolve) => setTimeout(resolve, 250));
+    }
+    return !signal.aborted;
+  }
+
   async function answer(f) {
     const { message: m, slot, abort } = f;
     const chat = await chatFor(slot, abort.signal);
@@ -236,6 +260,10 @@ export function createManager({
     if (envelope && envelope.is_error) throw new Error(`cursor-agent failed: ${String(textOf(envelope) || 'error').slice(0, 300)}`);
     const actions = parseManagerActions(stdout);
     if (!actions.reply) throw new Error('manager returned no reply');
+    if (!(await heldAgain(abort.signal))) {
+      if (!abort.signal.aborted) log(`manager answer to ${m.id} held back: the manager lease is not held here`);
+      return null;
+    }
     await settle(m, actions);
     recordTurn(slot, chat.chatId, m, prompt, actions.reply);
     if (registry.fact('manager-restart', m.id)) registry.dropFact('manager-restart', m.id);
@@ -252,11 +280,16 @@ export function createManager({
     const done = answer(f).then((replied) => ({ replied }), (error) => ({ error })).then(({ replied, error }) => {
       inflight.delete(m.id);
       runs.delete(done);
-      if (f.abort.signal.aborted) return;
-      if (error) log(`manager failed for ${m.id}: ${errorText(error)}`);
+      if (f.abort.signal.aborted) { log(`manager run for ${m.id} was stopped before it finished; it is not counted as a try`); return; }
+      const why = error ? runError(error).slice(0, 500) : null;
+      if (error) {
+        const failed = { count: ((failures.get(m.id) || {}).count || 0) + 1, error: why };
+        failures.set(m.id, failed);
+        log(`manager failed for ${m.id} (try ${failed.count} of ${MAX_ATTEMPTS}): ${why}`);
+      } else if (replied) failures.delete(m.id);
       status({
         ...(replied || {}), state: error ? 'error' : inflight.size ? 'working' : 'online', messageId: error ? m.id : null,
-        error: error ? errorText(error).slice(0, 500) : null, inflight: inflightView(),
+        error: why, inflight: inflightView(), ...(error ? { lastFailure: { id: m.id, from: m.from, error: why, at: now() } } : {}),
       });
     }).catch((e) => log(`manager status after ${m.id} failed: ${errorText(e)}`));
     runs.add(done);
@@ -315,9 +348,14 @@ export function createManager({
       const own = inflight.get(message.id);
       if (own) { own.leaseToken = message.leaseToken; continue; }
       if (isFyi(message)) { if (!(fyi && fyi(message))) messaging.ack('manager', message.id, {}, registry.directory()); continue; }
-      if (message.attempts > MAX_ATTEMPTS) {
-        const last = readJson(statusFile, {}).error || 'unknown error';
-        messaging.ack('manager', message.id, { reply: `I could not answer this after ${MAX_ATTEMPTS} tries, so I stopped retrying. Last error: ${last}`, fromName: 'Manager' }, registry.directory());
+      const failed = failures.get(message.id);
+      if ((failed && failed.count >= MAX_ATTEMPTS) || message.attempts > MAX_CLAIMS) {
+        const tries = failed ? failed.count : message.attempts - 1;
+        const last = failed ? failed.error : `none of its ${tries} runs finished (each was cut off by a restart or a lost lease)`;
+        messaging.ack('manager', message.id, { reply: `I could not answer this after ${tries} tries, so I stopped retrying. Last error: ${last}`, fromName: 'Manager' }, registry.directory());
+        failures.delete(message.id);
+        log(`manager gave up on ${message.id} after ${tries} tries: ${last}`);
+        status({ gaveUp: { id: message.id, from: message.from, tries, error: last, at: t } });
         continue;
       }
       dispatch(message);

package/src/lib/contract.sha256

@@ -1,2 +1,2 @@
-ce1142e1e133c8aff45a999d0d4b67faa6177f420ddcd1b449af81f8e806d8d0  connector-contract.json
+c9db92b38d0c978d4fa18d8a731fadbf3013a19659b524d615583ad099a8fafc  connector-contract.json
 7b7a59c08797c100b46436123dfcb09b517730357fa90cd0890da9b80308faf3  mirror-contract.json

package/src/lib/daemonRuntime.js

@@ -152,6 +152,7 @@ export class DaemonRuntime {
           attemptNo: attempt.attemptNo,
           fenceToken: attempt.fenceToken,
           events,
+          itemRefusals: true,
         });
       } catch (error) {
         if (machineWideError(error)) throw error;
@@ -164,15 +165,32 @@ export class DaemonRuntime {
         this.log('events_discarded', { ...trace, discarded });
         continue;
       }
-      if (Array.isArray(response.events)) {
-        await this.ledger.acknowledgeEvents(
-          response.events.map((event) => ({
-            sourceEventId: event.sourceEventId,
-            cloudCursor: event.cloudCursor,
-          }))
-        );
+      await this.#settleUpload(attempt, events, response);
+    }
+  }
+
+  /**
+   * Apply the cloud's answer to one uploaded batch. A stored event comes back with its `cursor` (`cloudCursor` is read
+   * when that is the only one) and leaves the outbox. A refused one ({ refused }) is dropped on its own, so the rest of
+   * the batch is kept. An event the answer leaves out stays pending and is sent again.
+   */
+  async #settleUpload(attempt, events, response) {
+    if (!Array.isArray(response?.events)) return;
+    const sent = new Set(events.map((event) => event.sourceEventId));
+    const accepted = [];
+    const refused = new Map();
+    for (const item of response.events) {
+      if (!sent.has(item?.sourceEventId)) continue;
+      if (item.refused) refused.set(item.sourceEventId, item.refused.code || 'HUB_EVENT_REFUSED');
+      else if (Number.isSafeInteger(item.cursor ?? item.cloudCursor)) {
+        accepted.push({ sourceEventId: item.sourceEventId, cursor: item.cursor ?? item.cloudCursor });
       }
     }
+    if (accepted.length) await this.ledger.acknowledgeEvents(accepted);
+    if (!refused.size) return;
+    const codes = [...new Set(refused.values())].sort().join(',');
+    const { discarded } = await this.ledger.discardEvents(attempt.jobId, codes, { sourceEventIds: [...refused.keys()] });
+    this.log('events_refused', { jobId: attempt.jobId, type: attempt.jobType, attemptNo: attempt.attemptNo, code: codes, discarded });
   }
 
   #compose(grant) {

package/src/lib/hubDbReader.js

@@ -129,4 +129,10 @@ export class HubSnapshot {
     const statement = this.db.prepare('SELECT kind, key, payload, updated_at FROM facts WHERE kind = ? ORDER BY key');
     return kinds.flatMap((kind) => statement.all(kind).map(plain));
   }
+
+  /** The `{ kind, key }` of the same facts in the same order, without their payloads. */
+  factKeys(kinds) {
+    const statement = this.db.prepare('SELECT kind, key FROM facts WHERE kind = ? ORDER BY key');
+    return kinds.flatMap((kind) => statement.all(kind).map(plain));
+  }
 }

package/src/lib/hubScan.js

@@ -17,6 +17,7 @@ export const SKIPPED_FACT_KINDS = Object.freeze({
   run: 'machine_runtime_state',
   wake: 'machine_runtime_state',
   manager: 'machine_runtime_state',
+  'manager-restart': 'machine_runtime_state',
   totals: 'phase1_deviation_totals',
 });
 export const REMOVABLE_KINDS = Object.freeze(['delivery', 'stream', 'agent-row', 'request', 'status_override']);

package/src/lib/ledger.js

@@ -226,6 +226,7 @@ export class DurableLedger {
     return clone(this.data.outbox.slice(0, Math.max(0, Math.min(100, limit))));
   }
 
+  /** Events the cloud stored, as { sourceEventId, cursor } with the cloud cursor it gave each, the wire's names. */
   async acknowledgeEvents(acknowledgements) {
     return this.#mutate((state) => {
       if (!Array.isArray(acknowledgements) || acknowledgements.length > 100) {
@@ -237,15 +238,15 @@ export class DurableLedger {
         if (
           !pendingIds.has(item.sourceEventId) ||
           seen.has(item.sourceEventId) ||
-          !Number.isSafeInteger(item.cloudCursor) ||
-          item.cloudCursor < 1
+          !Number.isSafeInteger(item.cursor) ||
+          item.cursor < 1
         ) {
           throw new HubError('HUB_EVENT_ACK_INVALID', 'Event acknowledgement does not match the sent batch');
         }
         seen.add(item.sourceEventId);
       }
       const accepted = new Map(
-        acknowledgements.map((item) => [item.sourceEventId, Number(item.cloudCursor)])
+        acknowledgements.map((item) => [item.sourceEventId, Number(item.cursor)])
       );
       for (const event of state.outbox) {
         if (!accepted.has(event.sourceEventId) || !TERMINAL_STATES.has(event.terminalState)) continue;
@@ -465,19 +466,27 @@ export class DurableLedger {
   }
 
   /**
-   * Drop a job's pending events the cloud refused for good (its fence is gone or the batch is invalid). They never
-   * count toward acceptedLocalCursor, and a terminal attempt has nothing left to upload.
+   * Drop a job's pending events the cloud refused for good: all of them (its fence is gone or the batch is invalid),
+   * or only sourceEventIds (the cloud refused those items and stored the rest). They never count toward
+   * acceptedLocalCursor. A terminal attempt has nothing left to upload once its terminal event is gone.
    */
-  async discardEvents(jobId, code) {
+  async discardEvents(jobId, code, { sourceEventIds = null } = {}) {
+    const only = sourceEventIds ? new Set(sourceEventIds) : null;
     return this.#mutate((state) => {
-      const discarded = state.outbox.filter((event) => event.jobId === jobId).length;
-      state.outbox = state.outbox.filter((event) => event.jobId !== jobId);
+      const dropped = (event) => event.jobId === jobId && (!only || only.has(event.sourceEventId));
+      const gone = state.outbox.filter(dropped);
+      state.outbox = state.outbox.filter((event) => !dropped(event));
       const attempt = state.attempts[jobId];
-      if (attempt && discarded) {
-        attempt.eventsDiscarded = { count: discarded, code, at: new Date().toISOString() };
-        if (TERMINAL_STATES.has(attempt.state)) attempt.terminalUploaded = true;
+      if (attempt && gone.length) {
+        attempt.eventsDiscarded = {
+          count: (only ? attempt.eventsDiscarded?.count || 0 : 0) + gone.length,
+          code,
+          at: new Date().toISOString(),
+        };
+        const terminalGone = !only || gone.some((event) => event.terminalState === attempt.state);
+        if (TERMINAL_STATES.has(attempt.state) && terminalGone) attempt.terminalUploaded = true;
       }
-      return { discarded };
+      return { discarded: gone.length };
     });
   }
 

package/src/lib/lifecycle.js

@@ -3,10 +3,12 @@ import { HubError } from './errors.js';
 import { validateJobResult } from './intents.js';
 import { redactText } from './redaction.js';
 
+// Hex, so the id always starts with a letter or digit: Pulse clouds before 1.1.3 refuse a sourceEventId starting with
+// '-' or '_', which a base64url digest does about 3% of the time.
 function eventId(tuple, type) {
   return createHash('sha256')
     .update(`${tuple.jobId}\0${tuple.attemptNo}\0${tuple.fenceToken}\0${type}`)
-    .digest('base64url');
+    .digest('hex');
 }
 
 export class AttemptCoordinator {