@agentpulselabs/cli-plugin-pulse 0.2.0 → 1.1.2

package/src/lib/mirrorSync.js

@@ -0,0 +1,353 @@
+import { createHash } from 'node:crypto';
+import { stat, unlink } from 'node:fs/promises';
+import path from 'node:path';
+import { HubError } from './errors.js';
+import { CONTRACT_VERSION, MIRROR_LIMITS } from './mirrorContract.js';
+import { readOwnerOnlyJson, writeOwnerOnlyJson } from './storage.js';
+
+const STATE_VERSION = 1;
+const RECENT_PASSES = 20;
+const MAX_PRIORITY_CALLS = 10;
+const BATCH_HEADROOM_BYTES = 4 * 1024;
+// Read whole on every pass that reads them at all, so a targeted pass may retire their missing items too.
+const WHOLE_FAMILIES = ['graph_window', 'brief'];
+export const RESEND_REQUEST_FILE = 'mirror-resend.request';
+// Contract keys already acted on; a key seen before never re-sends again, so a rolling deploy cannot loop.
+const MAX_CONTRACT_KEYS = 20;
+
+/**
+ * Asks the daemon to send every rejected item again on its next pass, through a file it takes once, so this never
+ * writes the state file the daemon holds in memory. Returns how many items are rejected now.
+ */
+export async function requestResend(directory, { stateFile = path.join(directory, 'mirror-state.json') } = {}) {
+  const stored = await readOwnerOnlyJson(stateFile, null);
+  const rejected = Object.values(stored?.items || {}).filter((entry) => entry?.r).length;
+  await writeOwnerOnlyJson(path.join(directory, RESEND_REQUEST_FILE), { requestedAt: Date.now() });
+  return { rejected };
+}
+
+async function takeResendRequest(directory) {
+  const file = path.join(directory, RESEND_REQUEST_FILE);
+  try {
+    await stat(file);
+    await unlink(file);
+    return true;
+  } catch (error) {
+    if (error.code === 'ENOENT') return false;
+    throw error;
+  }
+}
+
+function canonical(value) {
+  if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
+  if (value && typeof value === 'object') {
+    return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`;
+  }
+  return JSON.stringify(value);
+}
+
+export function itemHash(data) {
+  return createHash('sha256').update(canonical(data)).digest('hex');
+}
+
+function emptyState() {
+  return {
+    version: STATE_VERSION, items: {}, withheld: {}, graphDropped: {}, channelLastAt: {}, lastSyncAt: null, lastFullAt: null, lastCompleteAt: null, lastResult: null, lastPass: null, recentPasses: [], lastError: null, lastAbsent: {}, contractKeys: null,
+    redactedTotal: 0, redactedLastAt: null,
+  };
+}
+
+/**
+ * Where one batch's time went. networkMs is what is left once the lane wait, reading credentials, the backoff and the
+ * server's own time are taken out: the bytes on the wire both ways, TLS and the platform in front of the app.
+ */
+function batchSplit(items, ms, trace) {
+  const laneWaitMs = trace.laneWaitMs ?? 0;
+  const credentialsMs = trace.credentialsMs ?? 0;
+  const backoffMs = trace.backoffMs ?? 0;
+  const serverMs = trace.server?.totalMs ?? null;
+  return {
+    items,
+    ms,
+    laneWaitMs,
+    laneCapped: trace.laneCapped ?? false,
+    heldBy: trace.heldBy ?? [],
+    credentialsMs,
+    attempts: trace.attempts ?? 1,
+    statuses: trace.statuses ?? [],
+    backoffMs,
+    bytes: trace.bytes ?? null,
+    sentBytes: trace.sentBytes ?? null,
+    serverMs,
+    dbMs: trace.server?.dbMs ?? null,
+    lockMs: trace.server?.lockMs ?? null,
+    networkMs: Math.max(0, ms - laneWaitMs - credentialsMs - backoffMs - (serverMs ?? 0)),
+  };
+}
+
+function kindOf(key) {
+  return key.slice(0, key.indexOf('\0'));
+}
+
+function familyOf(key) {
+  const kind = kindOf(key);
+  return (kind === 'channel' || kind === 'message') && key.startsWith('bus:', kind.length + 1) ? 'bus' : kind;
+}
+
+export function chunkItems(items, { maxItems = MIRROR_LIMITS.maxItems, maxBytes = MIRROR_LIMITS.bodyBytes - BATCH_HEADROOM_BYTES } = {}) {
+  const batches = [];
+  let batch = [];
+  let bytes = 0;
+  for (const item of items) {
+    const size = Buffer.byteLength(JSON.stringify(item), 'utf8') + 1;
+    if (batch.length && (batch.length >= maxItems || bytes + size > maxBytes)) {
+      batches.push(batch);
+      batch = [];
+      bytes = 0;
+    }
+    batch.push(item);
+    bytes += size;
+  }
+  if (batch.length) batches.push(batch);
+  return batches;
+}
+
+export class MirrorSync {
+  constructor({ source, transport, stateFile, now = Date.now, maxBatchItems = MIRROR_LIMITS.maxItems }) {
+    this.source = source;
+    this.transport = transport;
+    this.stateFile = stateFile;
+    this.now = now;
+    this.maxBatchItems = maxBatchItems;
+    this.state = null;
+    this.loading = null;
+    this.saving = Promise.resolve();
+  }
+
+  // A windows pass and a change pass run at once (see MirrorService), so both share one loaded state, and saves are
+  // written one after another, each from the state as it is then, so an older snapshot never lands last.
+  #load() {
+    this.loading ??= readOwnerOnlyJson(this.stateFile, null).then((stored) => {
+      this.state = stored?.version === STATE_VERSION ? { ...emptyState(), ...stored } : emptyState();
+      return this.state;
+    }, (error) => {
+      this.loading = null;
+      throw error;
+    });
+    return this.loading;
+  }
+
+  #save() {
+    const write = this.saving.then(() => writeOwnerOnlyJson(this.stateFile, this.state));
+    this.saving = write.catch(() => {});
+    return write;
+  }
+
+  async status() {
+    const state = await this.#load();
+    const mirrored = {};
+    const dropped = {};
+    let rejected = 0;
+    const drop = (key, error) => {
+      const kind = kindOf(key);
+      dropped[kind] ||= {};
+      dropped[kind][error] = (dropped[kind][error] || 0) + 1;
+    };
+    for (const [key, entry] of Object.entries(state.items)) {
+      if (entry.r) {
+        drop(key, entry.r);
+        rejected += 1;
+      } else mirrored[kindOf(key)] = (mirrored[kindOf(key)] || 0) + 1;
+    }
+    for (const [key, error] of Object.entries(state.withheld)) drop(key, error);
+    return {
+      mirrored,
+      dropped,
+      rejected,
+      graphDropped: state.graphDropped,
+      lastSyncAt: state.lastSyncAt,
+      lastFullAt: state.lastFullAt,
+      lastCompleteAt: state.lastCompleteAt,
+      lastResult: state.lastResult,
+      lastPass: state.lastPass,
+      recentPasses: state.recentPasses,
+      lastError: state.lastError,
+      lastAbsent: state.lastAbsent,
+      redactedTotal: state.redactedTotal,
+      redactedLastAt: state.redactedLastAt,
+    };
+  }
+
+  #recordWithheld(state, collected, full) {
+    const current = new Map(collected.withheld.map((entry) => [`${entry.kind}\0${entry.externalId}`, entry.error]));
+    for (const key of collected.items.keys()) delete state.withheld[key];
+    if (full) {
+      for (const key of Object.keys(state.withheld)) {
+        if (!current.has(key) && collected.complete.has(familyOf(key))) delete state.withheld[key];
+      }
+    }
+    for (const [key, error] of current) {
+      if (error === 'content_withheld' && state.withheld[key] !== error) {
+        state.redactedTotal = (state.redactedTotal || 0) + 1;
+        state.redactedLastAt = this.now();
+      }
+      state.withheld[key] = error;
+    }
+  }
+
+  async recordError(error, { retryAt } = {}) {
+    const state = await this.#load();
+    state.lastError = { code: error.code || 'HUB_MIRROR_FAILED', at: this.now(), ...(retryAt ? { retryAt } : {}) };
+    await this.#save();
+  }
+
+  #tombstoneKeys(state, collected, full) {
+    const whole = full ? collected.complete : new Set(WHOLE_FAMILIES.filter((family) => collected.complete.has(family)));
+    if (!whole.size) return [];
+    const keys = [];
+    const listed = full && collected.listedChannels ? new Set(collected.listedChannels) : null;
+    const refetched = new Set(collected.refetchedChannels || []);
+    for (const [key, entry] of Object.entries(state.items)) {
+      if (collected.items.has(key)) continue;
+      const family = familyOf(key);
+      if (whole.has(family)) {
+        keys.push(key);
+      } else if (family === 'message' && listed && (!listed.has(entry.c) || refetched.has(entry.c))) {
+        keys.push(key);
+      }
+    }
+    return keys;
+  }
+
+  /**
+   * A pass whose signal aborts (a change preempting a full pass) stops between reads or at the next batch boundary.
+   * The batches it uploaded stay recorded; the rest are sent by a later pass. It is no reconcile and no error.
+   */
+  async #preempted(state, { full, sources, startedAt, timing }) {
+    const pass = { full, sources: full ? null : [...(sources || [])].sort(), preempted: true, ms: this.now() - startedAt, ...timing, priority: this.#priorityCalls(startedAt) };
+    state.recentPasses = [...(state.recentPasses || []), { at: this.now(), ...pass }].slice(-RECENT_PASSES);
+    await this.#save();
+    return new HubError('HUB_MIRROR_PREEMPTED', 'A change preempted this mirror pass');
+  }
+
+  /**
+   * A rejected item keeps its hash, so it is not sent again until it changes. It loses the hash, and the next pass that
+   * reads it sends it once, when this connector's contract or the cloud's accepted range is one this state has never
+   * seen (an upgrade on either side), or when `pulse mirror resend-rejected` left a request.
+   */
+  async #releaseRejected(state) {
+    const key = `mirror/${CONTRACT_VERSION}|${this.transport?.contractAccepts ?? ''}`;
+    const seen = Array.isArray(state.contractKeys) ? state.contractKeys : [];
+    const fresh = !seen.includes(key);
+    if (fresh) state.contractKeys = [...seen, key].slice(-MAX_CONTRACT_KEYS);
+    const requested = await takeResendRequest(path.dirname(this.stateFile));
+    if (!fresh && !requested) return;
+    for (const [itemKey, entry] of Object.entries(state.items)) {
+      if (entry.r) state.items[itemKey] = { ...entry, h: null };
+    }
+  }
+
+  /** The heartbeat, session, renew, ack and events calls that overlapped a pass: what a batch's lane wait was behind. */
+  #priorityCalls(startedAt) {
+    return (this.transport.priorityCalls?.({ since: startedAt }) ?? []).slice(-MAX_PRIORITY_CALLS);
+  }
+
+  async run({ full = false, sources, signal } = {}) {
+    const state = await this.#load();
+    const startedAt = this.now();
+    await this.#releaseRejected(state);
+    let collected;
+    try {
+      collected = await this.source.collect({ full, sources, channelLastAt: new Map(Object.entries(state.channelLastAt)), ...(signal ? { signal } : {}) });
+    } catch (error) {
+      if (error.code === 'HUB_MIRROR_PREEMPTED') throw await this.#preempted(state, { full, sources, startedAt, timing: { collectMs: this.now() - startedAt } });
+      throw error;
+    }
+    const now = this.now();
+    const timing = { collectMs: now - startedAt, uploadMs: 0, batches: 0, slowestBatchMs: 0, bytes: 0, sentBytes: 0, slowestBatch: null };
+    if (signal?.aborted) throw await this.#preempted(state, { full, sources, startedAt, timing });
+    const pending = new Map();
+    const outgoing = [];
+    for (const [key, item] of collected.items) {
+      const hash = itemHash(item.data);
+      const previous = state.items[key];
+      if (previous && previous.h === hash) continue;
+      const updatedAt = Math.max(item.updatedAt, previous?.u ?? 0);
+      const entry = { h: hash, u: updatedAt };
+      if (item.kind === 'message') entry.c = item.data.channelId;
+      pending.set(key, entry);
+      outgoing.push({ kind: item.kind, externalId: item.externalId, updatedAt, data: item.data });
+    }
+    for (const key of this.#tombstoneKeys(state, collected, full)) {
+      const externalId = key.slice(key.indexOf('\0') + 1);
+      pending.set(key, null);
+      outgoing.push({ kind: kindOf(key), externalId, updatedAt: Math.max(now, state.items[key].u + 1), deleted: true });
+    }
+
+    const result = {
+      pushed: 0, tombstoned: 0, unchanged: 0, ignored: 0, redacted: 0, rejected: [], withheld: collected.withheld.length, readErrors: collected.errors, briefsPending: collected.briefsPending === true,
+    };
+    try {
+      for (const batch of chunkItems(outgoing, { maxItems: this.maxBatchItems })) {
+        if (signal?.aborted) throw await this.#preempted(state, { full, sources, startedAt, timing });
+        const sentAt = this.now();
+        const trace = {};
+        const response = await this.transport.mirror({ items: batch }, { trace });
+        const batchMs = trace.ms ?? this.now() - sentAt;
+        timing.uploadMs += batchMs;
+        timing.batches += 1;
+        timing.bytes += trace.bytes || 0;
+        timing.sentBytes += trace.sentBytes || 0;
+        if (batchMs >= timing.slowestBatchMs) {
+          timing.slowestBatchMs = batchMs;
+          timing.slowestBatch = batchSplit(batch.length, batchMs, trace);
+        }
+        const rejected = new Map((response.rejected || []).map((entry) => [`${entry.kind}\0${entry.externalId}`, entry.error]));
+        for (const item of batch) {
+          const key = `${item.kind}\0${item.externalId}`;
+          const entry = pending.get(key);
+          if (entry === null) {
+            delete state.items[key];
+            result.tombstoned += 1;
+          } else if (rejected.has(key)) {
+            state.items[key] = { ...entry, r: rejected.get(key) };
+            result.rejected.push({ kind: item.kind, externalId: item.externalId, error: rejected.get(key) });
+          } else {
+            state.items[key] = entry;
+            result.pushed += 1;
+          }
+        }
+        result.unchanged += response.unchanged || 0;
+        result.ignored += response.ignored || 0;
+        // The cloud counts only the items it wrote; a batch it wrote nothing of (a retry) adds nothing whatever it says.
+        const redacted = response.accepted > 0 ? response.redacted || 0 : 0;
+        result.redacted += redacted;
+        // Kept per batch, so a pass that fails later still counts what the cloud already redacted from this machine.
+        if (redacted > 0) {
+          state.redactedTotal = (state.redactedTotal || 0) + redacted;
+          state.redactedLastAt = this.now();
+        }
+      }
+    } catch (error) {
+      if (error.code === 'HUB_MIRROR_PREEMPTED') throw error;
+      state.lastError = { code: error.code || 'HUB_MIRROR_FAILED', at: now };
+      await this.#save();
+      throw error;
+    }
+    this.#recordWithheld(state, collected, full);
+    state.graphDropped = Object.fromEntries(Object.entries({ ...state.graphDropped, ...collected.graphDropped }).filter(([, sections]) => Object.keys(sections).length));
+    if (full || sources?.has('channels')) state.channelLastAt = Object.fromEntries(collected.channelLastAt);
+    state.lastSyncAt = now;
+    if (full) state.lastFullAt = now;
+    if (full) {
+      state.lastAbsent = collected.absent || {};
+      if (!collected.errors.length) state.lastCompleteAt = now;
+    }
+    state.lastResult = { pushed: result.pushed, tombstoned: result.tombstoned, rejected: result.rejected.length, withheld: result.withheld, ignored: result.ignored, redacted: result.redacted };
+    state.lastPass = { full, sources: full ? null : [...(sources || [])].sort(), ms: this.now() - startedAt, ...timing, priority: this.#priorityCalls(startedAt) };
+    state.recentPasses = [...(state.recentPasses || []), { at: now, ...state.lastPass }].slice(-RECENT_PASSES);
+    state.lastError = collected.errors.length ? { code: collected.errors[0].code, at: now } : null;
+    await this.#save();
+    return result;
+  }
+}

package/src/lib/nodeFloor.js

@@ -0,0 +1,19 @@
+import { HubError } from './errors.js';
+
+export const NODE_FLOOR_MAJOR = 24;
+
+/** Whether this Node can run Pulse; on Node 22 and older, fetch can lose an abort on a long-lived stream (NODE22-01). */
+export function nodeFloor(version = process.versions.node) {
+  if (Number(version.split('.')[0]) >= NODE_FLOOR_MAJOR) return { ok: true, version };
+  return {
+    ok: false,
+    code: 'HUB_NODE_TOO_OLD',
+    version,
+    message: `Pulse needs Node ${NODE_FLOOR_MAJOR} or newer; this is Node ${version}. Install Node ${NODE_FLOOR_MAJOR} (Active LTS), then run \`agentpulse pulse install\` and \`agentpulse pulse start\`.`,
+  };
+}
+
+export function assertNodeFloor(version) {
+  const floor = nodeFloor(version);
+  if (!floor.ok) throw new HubError(floor.code, floor.message);
+}

package/src/lib/oauthLogin.js

@@ -0,0 +1,132 @@
+import { createHash, randomBytes } from 'node:crypto';
+import http from 'node:http';
+import { HubError } from './errors.js';
+import { cleanOrigin, decodeJwtClaims, fetchJson } from './http.js';
+import { LOGIN_SCOPE, OAUTH_CLIENT_ID } from './transportContract.js';
+
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+const DONE_PAGE = '<!doctype html><meta charset="utf-8"><title>AgentPulse Pulse</title>'
+  + '<p>Pulse connector sign-in finished. You can close this tab.</p>';
+
+export function createPkce(random = randomBytes) {
+  const verifier = random(32).toString('base64url');
+  return {
+    verifier,
+    challenge: createHash('sha256').update(verifier, 'ascii').digest('base64url'),
+    method: 'S256',
+  };
+}
+
+export function buildAuthorizeUrl({ platformUrl, redirectUri, state, challenge, scope = LOGIN_SCOPE }) {
+  return `${cleanOrigin(platformUrl, 'Platform URL')}/api/oauth/authorize?${new URLSearchParams({
+    response_type: 'code',
+    client_id: OAUTH_CLIENT_ID,
+    redirect_uri: redirectUri,
+    scope,
+    state,
+    code_challenge: challenge,
+    code_challenge_method: 'S256',
+  })}`;
+}
+
+export function waitForAuthorizationCode({ state, timeoutMs = 300_000 }) {
+  return new Promise((listening, failedToListen) => {
+    let settle;
+    const code = new Promise((resolve, reject) => {
+      settle = { resolve, reject };
+    });
+    let timer;
+    const server = http.createServer((req, res) => {
+      const url = new URL(req.url, 'http://127.0.0.1');
+      if (url.pathname !== '/callback') {
+        res.writeHead(404).end();
+        return;
+      }
+      res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }).end(DONE_PAGE);
+      clearTimeout(timer);
+      server.close();
+      if (url.searchParams.get('error')) {
+        settle.reject(new HubError('HUB_OAUTH_DENIED', 'Sign-in was denied in the browser'));
+      } else if (url.searchParams.get('state') !== state) {
+        settle.reject(new HubError('HUB_OAUTH_STATE_MISMATCH', 'Sign-in callback state did not match; aborting'));
+      } else if (!url.searchParams.get('code')) {
+        settle.reject(new HubError('HUB_OAUTH_DENIED', 'Sign-in callback carried no authorization code'));
+      } else {
+        settle.resolve(url.searchParams.get('code'));
+      }
+    });
+    server.on('error', failedToListen);
+    server.listen(0, '127.0.0.1', () => {
+      timer = setTimeout(() => {
+        server.close();
+        settle.reject(new HubError('HUB_OAUTH_TIMEOUT', 'Timed out waiting for browser sign-in (5 minutes)'));
+      }, timeoutMs);
+      listening({ redirectUri: `http://127.0.0.1:${server.address().port}/callback`, code });
+    });
+  });
+}
+
+export function acceptUserToken(tokenResponse, { platformUrl, now = Date.now } = {}) {
+  const claims = decodeJwtClaims(tokenResponse?.access_token);
+  if (
+    claims?.type !== 'user_delegated' ||
+    !UUID.test(String(claims.sub || '')) ||
+    !UUID.test(String(claims.team_id || ''))
+  ) {
+    throw new HubError(
+      'HUB_OAUTH_TOKEN_INVALID',
+      'The platform did not return a user-delegated token; run `agentpulse pulse login` again',
+    );
+  }
+  const expiresIn = Number(tokenResponse.expires_in);
+  return {
+    platformUrl: cleanOrigin(platformUrl, 'Platform URL'),
+    userId: claims.sub.toLowerCase(),
+    teamId: claims.team_id.toLowerCase(),
+    accessToken: tokenResponse.access_token,
+    refreshToken: typeof tokenResponse.refresh_token === 'string' ? tokenResponse.refresh_token : null,
+    expiresAt: new Date(now() + (Number.isFinite(expiresIn) && expiresIn > 0 ? expiresIn : 3600) * 1000).toISOString(),
+    scopes: String(tokenResponse.scope || '').split(' ').filter(Boolean),
+  };
+}
+
+export async function loginWithBrowser({
+  platformUrl,
+  fetch = globalThis.fetch,
+  openBrowser,
+  onAuthUrl = () => {},
+  scope = LOGIN_SCOPE,
+  timeoutMs = 300_000,
+  now = Date.now,
+  random = randomBytes,
+}) {
+  const origin = cleanOrigin(platformUrl, 'Platform URL');
+  const pkce = createPkce(random);
+  const state = random(16).toString('base64url');
+  const callback = await waitForAuthorizationCode({ state, timeoutMs });
+  const authorizeUrl = buildAuthorizeUrl({
+    platformUrl: origin,
+    redirectUri: callback.redirectUri,
+    state,
+    challenge: pkce.challenge,
+    scope,
+  });
+  onAuthUrl(authorizeUrl);
+  await openBrowser(authorizeUrl);
+  const code = await callback.code;
+  const exchanged = await fetchJson({
+    fetch,
+    url: `${origin}/api/oauth/token`,
+    body: {
+      grant_type: 'authorization_code',
+      client_id: OAUTH_CLIENT_ID,
+      code,
+      redirect_uri: callback.redirectUri,
+      code_verifier: pkce.verifier,
+    },
+  });
+  if (!exchanged.ok) {
+    throw new HubError('HUB_OAUTH_EXCHANGE_FAILED', `Sign-in token exchange failed with HTTP ${exchanged.status}`);
+  }
+  return acceptUserToken(exchanged.body, { platformUrl: origin, now });
+}

package/src/lib/outboxFlusher.js

@@ -0,0 +1,75 @@
+import { fullJitterDelay } from './backoff.js';
+import { HubError } from './errors.js';
+
+// I3: a command gets at most this many retryable refusals before it is set aside, so it can block its group only so long.
+export const MAX_ATTEMPTS = 6;
+// The cloud holds a command behind an earlier refused command of its group; it goes again as soon as that one does.
+const HELD_CODE = 'held_by_group';
+const HELD_RETRY_MS = 1_000;
+
+/**
+ * Sends the runner store's outbox to Pulse, one batch at a time and in enqueue order (design §6.3 flush loop). Each
+ * command carries its ordering group, so the cloud holds a group's later commands behind one it refused for now, and
+ * the store holds them locally while it backs off. Per item: stored or duplicate → acked; retryable → backoff (then
+ * dead after MAX_ATTEMPTS); any other refusal → dead with its code. A refusal of the whole batch is thrown.
+ */
+export class OutboxFlusher {
+  constructor({ store, transport, random = Math.random, maxAttempts = MAX_ATTEMPTS }) {
+    this.store = store;
+    this.transport = transport;
+    this.random = random;
+    this.maxAttempts = maxAttempts;
+  }
+
+  /** One batch; resolves to its tally, or null when nothing is due. */
+  async flushOnce() {
+    const batch = this.store.nextBatch();
+    if (!batch.length) return null;
+    const response = await this.transport.commands({
+      commands: batch.map((row) => ({ type: row.type, idempotencyKey: row.key, payload: row.payload, group: row.group })),
+    });
+    const results = Array.isArray(response?.results) ? response.results : [];
+    const tally = { sent: batch.length, written: 0, duplicate: 0, retried: 0, held: 0, dead: 0 };
+    batch.forEach((row, index) => {
+      const result = results[index];
+      if (!result || result.key !== row.key) {
+        this.store.retry(row.key, { code: 'no_result', delayMs: fullJitterDelay(row.attempts, { random: this.random }) });
+        tally.retried += 1;
+        return;
+      }
+      if (!result.error) {
+        this.store.acked(row.key, { seq: result.seq ?? null, duplicate: Boolean(result.duplicate) });
+        tally[result.duplicate ? 'duplicate' : 'written'] += 1;
+        return;
+      }
+      const { code = 'refused', retryable = false, message = null } = result.error;
+      if (code === HELD_CODE) {
+        this.store.hold(row.key, { code, delayMs: HELD_RETRY_MS });
+        tally.held += 1;
+        return;
+      }
+      if (retryable && row.attempts + 1 < this.maxAttempts) {
+        this.store.retry(row.key, { code, delayMs: fullJitterDelay(row.attempts, { random: this.random }) });
+        tally.retried += 1;
+        return;
+      }
+      this.store.dead(row.key, { code: retryable ? `max_attempts:${code}` : code, message });
+      tally.dead += 1;
+    });
+    return tally;
+  }
+
+  /** Batches until the outbox has nothing due, `maxBatches` were sent or the signal fires; the summed tally. */
+  async drain({ maxBatches = Infinity, signal } = {}) {
+    const total = { batches: 0, sent: 0, written: 0, duplicate: 0, retried: 0, held: 0, dead: 0 };
+    while (total.batches < maxBatches) {
+      if (signal?.aborted) throw new HubError('HUB_SHADOW_STOPPED', 'The shadow feed stopped');
+      const tally = await this.flushOnce();
+      if (!tally) break;
+      total.batches += 1;
+      for (const key of Object.keys(tally)) total[key] += tally[key];
+      if (tally.written + tally.duplicate + tally.dead === 0) break;
+    }
+    return total;
+  }
+}

package/src/lib/ownerLock.js

@@ -0,0 +1,99 @@
+import { lstat, open, readFile, rm } from 'node:fs/promises';
+import path from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { ensureOwnerOnlyDirectory } from './storage.js';
+
+const WRITER_GRACE_MS = 5_000;
+
+async function readIfPresent(file) {
+  try {
+    return await readFile(file, 'utf8');
+  } catch (error) {
+    if (error.code === 'ENOENT') return null;
+    throw error;
+  }
+}
+
+async function currentOwner(file) {
+  let info;
+  try {
+    info = await lstat(file);
+  } catch (error) {
+    if (error.code === 'ENOENT') return { state: 'free' };
+    throw error;
+  }
+  const text = await readIfPresent(file);
+  if (text === null) return { state: 'free' };
+  if (!text.trim()) {
+    return Date.now() - info.mtimeMs < WRITER_GRACE_MS ? { state: 'writing' } : { state: 'invalid', text };
+  }
+  try {
+    const owner = JSON.parse(text);
+    return owner && typeof owner === 'object' && Number.isInteger(owner.pid)
+      ? { state: 'owned', owner, text }
+      : { state: 'invalid', text };
+  } catch {
+    return { state: 'invalid', text };
+  }
+}
+
+async function removeIfUnchanged(file, observedText) {
+  const takeover = `${file}.takeover`;
+  let handle;
+  try {
+    handle = await open(takeover, 'wx', 0o600);
+  } catch (error) {
+    if (error.code !== 'EEXIST') throw error;
+    const info = await lstat(takeover).catch(() => null);
+    if (info && Date.now() - info.mtimeMs > WRITER_GRACE_MS) await rm(takeover, { force: true });
+    return;
+  }
+  try {
+    await handle.close();
+    if ((await readIfPresent(file)) === observedText) await rm(file, { force: true });
+  } finally {
+    await rm(takeover, { force: true });
+  }
+}
+
+export async function acquireOwnerLock({
+  file,
+  identity,
+  identityProvider,
+  timeoutMs = 0,
+  retryMs = 50,
+  heldError,
+}) {
+  await ensureOwnerOnlyDirectory(path.dirname(file));
+  const serialized = `${JSON.stringify(identity)}\n`;
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    try {
+      const handle = await open(file, 'wx', 0o600);
+      try {
+        await handle.writeFile(serialized, 'utf8');
+        await handle.sync();
+      } finally {
+        await handle.close();
+      }
+      return async () => {
+        const text = await readFile(file, 'utf8').catch(() => null);
+        if (text === serialized) await rm(file, { force: true });
+      };
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+    }
+    const { state, owner, text } = await currentOwner(file);
+    const held = state === 'writing' || (state === 'owned' && await identityProvider.matches(owner));
+    if (!held) {
+      if (state !== 'free') {
+        await removeIfUnchanged(file, text);
+        await sleep(Math.min(retryMs, 10));
+      }
+      if (Date.now() >= deadline + WRITER_GRACE_MS) throw heldError(owner);
+      continue;
+    }
+    if (Date.now() >= deadline) throw heldError(owner);
+    await sleep(retryMs);
+  }
+}