@agentpulselabs/cli-plugin-pulse 1.1.3 → 1.1.6

package/src/lib/runnerStore.js

@@ -91,6 +91,8 @@ export class RunnerStore {
     }
   }
 
+  #statements = new Map();
+
   constructor({ db, file, now, maxPending }) {
     this.db = db;
     this.file = file;
@@ -103,20 +105,30 @@ export class RunnerStore {
     this.db.close();
   }
 
+  /** The statement for this SQL, prepared once per connection: the scanners call the store once or more per row. */
+  sql(text) {
+    let statement = this.#statements.get(text);
+    if (!statement) {
+      statement = this.db.prepare(text);
+      this.#statements.set(text, statement);
+    }
+    return statement;
+  }
+
   /**
    * Run `fn` in one immediate transaction: everything it enqueues and every cursor it moves commit together. Inside a
    * transaction already open on this connection (the runner's capture) it joins that one.
    */
   transaction(fn) {
     if (this.depth > 0 || this.db.isTransaction) return fn();
-    this.db.prepare('BEGIN IMMEDIATE').run();
+    this.sql('BEGIN IMMEDIATE').run();
     this.depth += 1;
     try {
       const out = fn();
-      this.db.prepare('COMMIT').run();
+      this.sql('COMMIT').run();
       return out;
     } catch (error) {
-      this.db.prepare('ROLLBACK').run();
+      this.sql('ROLLBACK').run();
       throw error;
     } finally {
       this.depth -= 1;
@@ -124,34 +136,34 @@ export class RunnerStore {
   }
 
   meta(key) {
-    const row = this.db.prepare('SELECT value FROM meta WHERE key = ?').get(key);
+    const row = this.sql('SELECT value FROM meta WHERE key = ?').get(key);
     return row ? JSON.parse(row.value) : null;
   }
 
   setMeta(key, value) {
-    this.db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value')
+    this.sql('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value')
       .run(key, JSON.stringify(value));
   }
 
   cursor(source) {
-    const row = this.db.prepare('SELECT position FROM cursor WHERE source = ?').get(source);
+    const row = this.sql('SELECT position FROM cursor WHERE source = ?').get(source);
     return row ? JSON.parse(row.position) : null;
   }
 
   setCursor(source, position) {
-    this.db.prepare('INSERT INTO cursor (source, position) VALUES (?, ?) ON CONFLICT (source) DO UPDATE SET position = excluded.position')
+    this.sql('INSERT INTO cursor (source, position) VALUES (?, ?) ON CONFLICT (source) DO UPDATE SET position = excluded.position')
       .run(source, JSON.stringify(position));
   }
 
   /** Whether a key was ever enqueued: pending, acknowledged or dead. */
   known(key) {
-    return Boolean(this.db.prepare(
+    return Boolean(this.sql(
       'SELECT 1 AS found FROM outbox WHERE key = ? UNION ALL SELECT 1 FROM sent WHERE key = ? UNION ALL SELECT 1 FROM dead WHERE key = ? LIMIT 1',
     ).get(key, key, key));
   }
 
   pendingCount() {
-    return Number(this.db.prepare('SELECT COUNT(*) AS n FROM outbox').get().n);
+    return Number(this.sql('SELECT COUNT(*) AS n FROM outbox').get().n);
   }
 
   /**
@@ -160,7 +172,7 @@ export class RunnerStore {
    */
   enqueue(entries) {
     return this.transaction(() => {
-      const insert = this.db.prepare(
+      const insert = this.sql(
         `INSERT INTO outbox (key, grp, type, payload, source, enqueued_at)
          SELECT ?, ?, ?, ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM sent WHERE key = ?) AND NOT EXISTS (SELECT 1 FROM dead WHERE key = ?)
          ON CONFLICT (key) DO NOTHING`,
@@ -214,40 +226,40 @@ export class RunnerStore {
   /** The cloud stored (or already had) this key. */
   acked(key, { seq = null, duplicate = false } = {}) {
     this.transaction(() => {
-      const row = this.db.prepare('SELECT source, type FROM outbox WHERE key = ?').get(key);
+      const row = this.sql('SELECT source, type FROM outbox WHERE key = ?').get(key);
       if (!row) return;
-      this.db.prepare('INSERT INTO sent (key, source, type, seq, duplicate, at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (key) DO NOTHING')
+      this.sql('INSERT INTO sent (key, source, type, seq, duplicate, at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (key) DO NOTHING')
         .run(key, row.source, row.type, seq === null ? null : String(seq), duplicate ? 1 : 0, this.now());
-      this.db.prepare('DELETE FROM outbox WHERE key = ?').run(key);
+      this.sql('DELETE FROM outbox WHERE key = ?').run(key);
     });
   }
 
   /** A retryable refusal: the row stays first in its group and is due again after `delayMs`. */
   retry(key, { code, delayMs }) {
-    this.db.prepare('UPDATE outbox SET attempts = attempts + 1, next_at = ?, last_error = ? WHERE key = ?').run(this.now() + delayMs, code, key);
+    this.sql('UPDATE outbox SET attempts = attempts + 1, next_at = ?, last_error = ? WHERE key = ?').run(this.now() + delayMs, code, key);
   }
 
   /** Held behind an earlier command of its group: due again after `delayMs`, without counting an attempt. */
   hold(key, { code, delayMs }) {
-    this.db.prepare('UPDATE outbox SET next_at = ?, last_error = ? WHERE key = ?').run(this.now() + delayMs, code, key);
+    this.sql('UPDATE outbox SET next_at = ?, last_error = ? WHERE key = ?').run(this.now() + delayMs, code, key);
   }
 
   /** A refusal that repeating cannot fix: the row leaves the outbox with its reason. */
   dead(key, { code, message = null }) {
     this.transaction(() => {
-      const row = this.db.prepare('SELECT grp, type, payload, source FROM outbox WHERE key = ?').get(key);
+      const row = this.sql('SELECT grp, type, payload, source FROM outbox WHERE key = ?').get(key);
       if (!row) return;
-      this.db.prepare(
+      this.sql(
         `INSERT INTO dead (key, grp, type, payload, source, code, message, at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
          ON CONFLICT (key) DO UPDATE SET code = excluded.code, message = excluded.message, at = excluded.at`,
       ).run(key, row.grp, row.type, row.payload, row.source, code, message, this.now());
-      this.db.prepare('DELETE FROM outbox WHERE key = ?').run(key);
+      this.sql('DELETE FROM outbox WHERE key = ?').run(key);
     });
   }
 
   /** A command refused before it was queued (it never reaches the cloud), recorded with its reason. */
   refuse(entry, { code, message = null }) {
-    this.db.prepare(
+    this.sql(
       `INSERT INTO dead (key, grp, type, payload, source, code, message, at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        ON CONFLICT (key) DO NOTHING`,
     ).run(entry.key, entry.group, entry.type, JSON.stringify(entry.payload ?? null), entry.source, code, message, this.now());
@@ -255,38 +267,38 @@ export class RunnerStore {
 
   /** The code a key was refused or dead-lettered with, or null. */
   deadCode(key) {
-    return this.db.prepare('SELECT code FROM dead WHERE key = ?').get(key)?.code ?? null;
+    return this.sql('SELECT code FROM dead WHERE key = ?').get(key)?.code ?? null;
   }
 
   /** Every queued command in queue order, payload parsed (a reconcile reads a scratch store's whole scan this way). */
   planned() {
-    return this.db.prepare('SELECT key, grp, type, payload, source FROM outbox ORDER BY ord').all()
+    return this.sql('SELECT key, grp, type, payload, source FROM outbox ORDER BY ord').all()
       .map((row) => ({ key: row.key, group: row.grp, type: row.type, payload: JSON.parse(row.payload), source: row.source }));
   }
 
   /** Every set-aside key with its code (all of them; deadRows is the newest few). */
   deadKeys() {
-    return this.db.prepare('SELECT key, type, source, code FROM dead ORDER BY key').all().map((row) => ({ ...row }));
+    return this.sql('SELECT key, type, source, code FROM dead ORDER BY key').all().map((row) => ({ ...row }));
   }
 
   deadRows({ limit = 20 } = {}) {
-    return this.db.prepare('SELECT key, type, source, code, message, at FROM dead ORDER BY at DESC LIMIT ?').all(limit).map((row) => ({ ...row }));
+    return this.sql('SELECT key, type, source, code, message, at FROM dead ORDER BY at DESC LIMIT ?').all(limit).map((row) => ({ ...row }));
   }
 
   /** Counts per source: pending, sent (written or duplicate) and dead (by code). */
   counts() {
     const out = {};
     const at = (source) => (out[source] ||= { pending: 0, written: 0, duplicate: 0, dead: {} });
-    for (const row of this.db.prepare('SELECT source, COUNT(*) AS n FROM outbox GROUP BY source').all()) at(row.source).pending = Number(row.n);
-    for (const row of this.db.prepare('SELECT source, duplicate, COUNT(*) AS n FROM sent GROUP BY source, duplicate').all()) {
+    for (const row of this.sql('SELECT source, COUNT(*) AS n FROM outbox GROUP BY source').all()) at(row.source).pending = Number(row.n);
+    for (const row of this.sql('SELECT source, duplicate, COUNT(*) AS n FROM sent GROUP BY source, duplicate').all()) {
       at(row.source)[row.duplicate ? 'duplicate' : 'written'] = Number(row.n);
     }
-    for (const row of this.db.prepare('SELECT source, code, COUNT(*) AS n FROM dead GROUP BY source, code').all()) at(row.source).dead[row.code] = Number(row.n);
+    for (const row of this.sql('SELECT source, code, COUNT(*) AS n FROM dead GROUP BY source, code').all()) at(row.source).dead[row.code] = Number(row.n);
     return out;
   }
 
   version(entity) {
-    const row = this.db.prepare('SELECT sha, raw_sha, static_sha, key, at FROM version WHERE entity = ?').get(entity);
+    const row = this.sql('SELECT sha, raw_sha, static_sha, key, at FROM version WHERE entity = ?').get(entity);
     return row ? { sha: row.sha, rawSha: row.raw_sha, staticSha: row.static_sha, key: row.key, at: Number(row.at) } : null;
   }
 
@@ -295,7 +307,7 @@ export class RunnerStore {
    * `<prefix><sep><sha>.<k>` when it returns after another version (A→B→A), so the return is a new event that applies.
    */
   versionKey(entity, prefix, sha, separator = ':') {
-    const seen = this.db.prepare('SELECT count FROM version_seen WHERE entity = ? AND sha = ?').get(entity, sha);
+    const seen = this.sql('SELECT count FROM version_seen WHERE entity = ? AND sha = ?').get(entity, sha);
     const count = seen ? Number(seen.count) : 0;
     return count === 0 ? `${prefix}${separator}${sha}` : `${prefix}${separator}${sha}.${count}`;
   }
@@ -304,12 +316,12 @@ export class RunnerStore {
   setVersion(entity, { sha, rawSha = null, staticSha = null, key }) {
     const last = this.version(entity);
     if (last?.sha !== sha) {
-      this.db.prepare(
+      this.sql(
         `INSERT INTO version_seen (entity, sha, count) VALUES (?, ?, 1)
          ON CONFLICT (entity, sha) DO UPDATE SET count = count + 1`,
       ).run(entity, sha);
     }
-    this.db.prepare(
+    this.sql(
       `INSERT INTO version (entity, sha, raw_sha, static_sha, key, at) VALUES (?, ?, ?, ?, ?, ?)
        ON CONFLICT (entity) DO UPDATE SET sha = excluded.sha, raw_sha = excluded.raw_sha, static_sha = excluded.static_sha,
          key = excluded.key, at = excluded.at`,
@@ -318,67 +330,67 @@ export class RunnerStore {
 
   /** The raw form last read, without changing what was sent (an unchanged row skips redaction next time). */
   touchVersion(entity, rawSha) {
-    this.db.prepare('UPDATE version SET raw_sha = ? WHERE entity = ?').run(rawSha, entity);
+    this.sql('UPDATE version SET raw_sha = ? WHERE entity = ?').run(rawSha, entity);
   }
 
   dropVersion(entity) {
-    this.db.prepare('DELETE FROM version WHERE entity = ?').run(entity);
+    this.sql('DELETE FROM version WHERE entity = ?').run(entity);
   }
 
   entities(prefix) {
-    return this.db.prepare("SELECT entity FROM version WHERE entity >= ? AND entity < ? || char(1114111)").all(prefix, prefix).map((row) => row.entity);
+    return this.sql("SELECT entity FROM version WHERE entity >= ? AND entity < ? || char(1114111)").all(prefix, prefix).map((row) => row.entity);
   }
 
   /** The receipt state last queued for each recipient row still able to change (not acked or expired). */
   openReceipts() {
-    return this.db.prepare("SELECT message_id, recipient, state FROM receipt_state WHERE state NOT IN ('acked', 'expired')").all()
+    return this.sql("SELECT message_id, recipient, state FROM receipt_state WHERE state NOT IN ('acked', 'expired')").all()
       .map((row) => ({ messageId: row.message_id, recipient: row.recipient, state: row.state }));
   }
 
   receiptState(messageId, recipient) {
-    return this.db.prepare('SELECT state FROM receipt_state WHERE message_id = ? AND recipient = ?').get(messageId, recipient)?.state ?? null;
+    return this.sql('SELECT state FROM receipt_state WHERE message_id = ? AND recipient = ?').get(messageId, recipient)?.state ?? null;
   }
 
   setReceiptState(messageId, recipient, state) {
-    this.db.prepare(
+    this.sql(
       `INSERT INTO receipt_state (message_id, recipient, state) VALUES (?, ?, ?)
        ON CONFLICT (message_id, recipient) DO UPDATE SET state = excluded.state`,
     ).run(messageId, recipient, state);
   }
 
   dropReceipts(messageId) {
-    this.db.prepare('DELETE FROM receipt_state WHERE message_id = ?').run(messageId);
+    this.sql('DELETE FROM receipt_state WHERE message_id = ?').run(messageId);
   }
 
   rememberMessage(id, createdAt, group) {
-    this.db.prepare('INSERT INTO recent_message (id, created_at, grp) VALUES (?, ?, ?) ON CONFLICT (id) DO NOTHING').run(id, createdAt, group);
+    this.sql('INSERT INTO recent_message (id, created_at, grp) VALUES (?, ?, ?) ON CONFLICT (id) DO NOTHING').run(id, createdAt, group);
   }
 
   recentMessages(sinceMs) {
-    return this.db.prepare('SELECT id, created_at, grp FROM recent_message WHERE created_at >= ? ORDER BY created_at').all(sinceMs)
+    return this.sql('SELECT id, created_at, grp FROM recent_message WHERE created_at >= ? ORDER BY created_at').all(sinceMs)
       .map((row) => ({ id: row.id, createdAt: Number(row.created_at), group: row.grp }));
   }
 
   forgetMessage(id) {
-    this.db.prepare('DELETE FROM recent_message WHERE id = ?').run(id);
+    this.sql('DELETE FROM recent_message WHERE id = ?').run(id);
   }
 
   pruneRecentMessages(beforeMs) {
-    this.db.prepare('DELETE FROM recent_message WHERE created_at < ?').run(beforeMs);
+    this.sql('DELETE FROM recent_message WHERE created_at < ?').run(beforeMs);
   }
 
   ingested(channel) {
-    return this.db.prepare('SELECT ordinal, sha, post_key, at FROM ingested WHERE channel = ? ORDER BY ordinal').all(channel)
+    return this.sql('SELECT ordinal, sha, post_key, at FROM ingested WHERE channel = ? ORDER BY ordinal').all(channel)
       .map((row) => ({ ordinal: Number(row.ordinal), sha: row.sha, postKey: row.post_key, at: Number(row.at) }));
   }
 
   /** The key a channel's block (its digest and occurrence) was first sent under, or null. */
   ingestedKey(channel, sha) {
-    return this.db.prepare('SELECT post_key FROM ingested WHERE channel = ? AND sha = ?').get(channel, sha)?.post_key ?? null;
+    return this.sql('SELECT post_key FROM ingested WHERE channel = ? AND sha = ?').get(channel, sha)?.post_key ?? null;
   }
 
   setIngested(channel, { ordinal, sha, postKey, at }) {
-    this.db.prepare(
+    this.sql(
       `INSERT INTO ingested (channel, ordinal, sha, post_key, at) VALUES (?, ?, ?, ?, ?)
        ON CONFLICT (channel, ordinal) DO UPDATE SET sha = excluded.sha, post_key = excluded.post_key, at = excluded.at`,
     ).run(channel, ordinal, sha, postKey, at);
@@ -387,18 +399,18 @@ export class RunnerStore {
   /** What one channel file holds now, by position, replacing what was recorded before. */
   replaceIngested(channel, rows) {
     this.transaction(() => {
-      this.db.prepare('DELETE FROM ingested WHERE channel = ?').run(channel);
+      this.sql('DELETE FROM ingested WHERE channel = ?').run(channel);
       for (const row of rows) this.setIngested(channel, row);
     });
   }
 
   presenceSent(actorId) {
-    const row = this.db.prepare('SELECT last_seen, sent_at FROM presence_sent WHERE actor_id = ?').get(actorId);
+    const row = this.sql('SELECT last_seen, sent_at FROM presence_sent WHERE actor_id = ?').get(actorId);
     return row ? { lastSeen: Number(row.last_seen), sentAt: Number(row.sent_at) } : null;
   }
 
   setPresenceSent(actorId, lastSeen) {
-    this.db.prepare(
+    this.sql(
       `INSERT INTO presence_sent (actor_id, last_seen, sent_at) VALUES (?, ?, ?)
        ON CONFLICT (actor_id) DO UPDATE SET last_seen = excluded.last_seen, sent_at = excluded.sent_at`,
     ).run(actorId, lastSeen, this.now());
@@ -406,12 +418,12 @@ export class RunnerStore {
 
   /** Whether the cloud acknowledged `key` as a command of `type`. */
   sentAs(key, type) {
-    return Boolean(this.db.prepare('SELECT 1 AS found FROM sent WHERE key = ? AND type = ?').get(key, type));
+    return Boolean(this.sql('SELECT 1 AS found FROM sent WHERE key = ? AND type = ?').get(key, type));
   }
 
   /** Acknowledged keys older than `beforeMs` go; the cloud's dedupe is the backstop for anything resent after that. */
   pruneSent(beforeMs) {
-    return Number(this.db.prepare('DELETE FROM sent WHERE at < ?').run(beforeMs).changes);
+    return Number(this.sql('DELETE FROM sent WHERE at < ?').run(beforeMs).changes);
   }
 }
 

package/src/runner/bridge.js

@@ -24,6 +24,8 @@ const PRESENCE_CURSOR = 'presence_cursor';
 export const SYNC_FAILING_AFTER = 3;
 /** SCOPE-02: while blocked for a scope or grant, the sync is retried this often with the sign-in read again, so a re-login clears it. */
 export const BLOCK_RETRY_MS = 60_000;
+/** Changes capture refused to send because they hold secret-like content (hubScan): set aside by design, not a fault. */
+const WITHHELD_CODE = 'content_withheld';
 const ERROR_CODES = new Set(['HUB_LOGIN_REQUIRED', 'HUB_SCOPE_DENIED', 'capability_denied', 'cloud_first_off', 'HUB_MACHINE_REVOKED']);
 
 /**
@@ -63,6 +65,7 @@ export class RunnerBridge {
     if (this.running) return;
     this.running = true;
     this.runner.on('wrote', () => this.kick());
+    this.runner.on('loop_stall', (message) => this.log('loop_stall', message.details || {}));
     this.syncing = this.#syncLoop();
     this.flushing = this.#flushLoop();
     if (this.leaseConfig) this.leasing = this.#leaseLoop();
@@ -254,10 +257,12 @@ export class RunnerBridge {
     const now = this.now();
     const pending = this.store.pendingCount();
     const dead = Number(this.store.db.prepare('SELECT COUNT(*) AS n FROM dead').get().n);
+    const withheld = Number(this.store.db.prepare('SELECT COUNT(*) AS n FROM dead WHERE code = ?').get(WITHHELD_CODE).n);
+    const setAside = dead - withheld;
     const quarantined = Number(this.store.db.prepare('SELECT COUNT(*) AS n FROM pulse_quarantine').get()?.n || 0);
     const applied = this.applied();
     const lag = Math.max(0, this.state.head - applied);
-    const outbox = { pending, dead, capacity: OUTBOX_MAX_PENDING };
+    const outbox = { pending, dead, withheld, capacity: OUTBOX_MAX_PENDING };
     const base = { lastSyncAt: this.state.lastSyncAt, applied, head: this.state.head, lag, outbox, quarantined,
       staleSeconds: this.state.lastSyncAt ? (now - this.state.lastSyncAt) / 1000 : null };
     const { block } = this.state.error || {};
@@ -276,8 +281,8 @@ export class RunnerBridge {
       return { ...base, state: 'offline', reason: this.state.lastContactAt ? 'no contact with Pulse' : 'not connected yet' };
     }
     if (lag > LAG_EVENTS || (pending > 0 && this.state.flushError)) return { ...base, state: 'catching_up', reason: `${lag} events and ${pending} changes to go` };
-    if (dead || quarantined || pending > 0.8 * OUTBOX_MAX_PENDING) {
-      return { ...base, state: 'degraded', reason: `${dead} change${dead === 1 ? '' : 's'} and ${quarantined} event${quarantined === 1 ? '' : 's'} set aside` };
+    if (setAside || quarantined || pending > 0.8 * OUTBOX_MAX_PENDING) {
+      return { ...base, state: 'degraded', reason: `${setAside} change${setAside === 1 ? '' : 's'} and ${quarantined} event${quarantined === 1 ? '' : 's'} set aside` };
     }
     if (!this.state.lastSyncAt || now - this.state.lastSyncAt > ONLINE_MS) return { ...base, state: 'catching_up', reason: 'waiting for a sync' };
     return { ...base, state: 'online', reason: 'in sync' };

package/src/runner/busIdentity.js

@@ -204,8 +204,18 @@ export function createBusIdentity(hub, { now: clock = Date.now } = {}) {
     refreshProof: db.prepare("INSERT INTO pulse_proofs (address, at, via, used) VALUES (?, ?, 'listen', 0) ON CONFLICT (address) DO UPDATE SET at = MAX(pulse_proofs.at, excluded.at)"),
     useProof: db.prepare('UPDATE pulse_proofs SET used = 1 WHERE address = ?'),
     remote: db.prepare('SELECT * FROM pulse_remote_presence WHERE actor_id = ?'),
+    queuedFor: db.prepare(`SELECT 1 AS found FROM deliveries d JOIN messages m ON m.id = d.message_id
+      WHERE d.recipient = ? AND d.state = 'queued' AND m.expires_at > ? LIMIT 1`),
+    lapsedFor: db.prepare(`SELECT 1 AS found FROM deliveries d JOIN messages m ON m.id = d.message_id
+      WHERE d.recipient = ? AND d.state = 'leased' AND d.lease_until < ? AND m.expires_at > ? LIMIT 1`),
   };
 
+  /**
+   * Whether messaging.claim would find anything for one of `addresses` now (its own rows: queued, or leased with a lapsed
+   * lease, and not expired), read without the write lock, so an empty `bus listen` poll needs no write transaction.
+   */
+  const claimable = (addresses, now = clock()) => addresses.some((a) => q.queuedFor.get(a, now) || q.lapsedFor.get(a, now, now));
+
   /**
    * Every chat the hub already knows to be a session, before its next hook: hook-seen and transcript-found actors, and
    * presence rows whose last event came from a hook. Without this, right after an upgrade another session could take a
@@ -464,7 +474,7 @@ export function createBusIdentity(hub, { now: clock = Date.now } = {}) {
     return { registrations: shown.map(({ active: _active, ...e }) => e), groups: [...groups.values()], window: { minutes: windowMs / 60e3, all: !!all } };
   }
 
-  return { addressesOf, inboxFor, sessionOf, aliasesOf, activity, known, suggest, reach, sawHook, proved, listened, bindFromHook, notices, who };
+  return { addressesOf, inboxFor, claimable, sessionOf, aliasesOf, activity, known, suggest, reach, sawHook, proved, listened, bindFromHook, notices, who };
 }
 
 /**

package/src/runner/capture.js

@@ -1,3 +1,4 @@
+import { setTimeout as sleep } from 'node:timers/promises';
 import { withTx } from '../hub/core/db.mjs';
 
 /**
@@ -18,6 +19,17 @@ const SCHEMA = [
   'CREATE TABLE IF NOT EXISTS pulse_capture (id INTEGER PRIMARY KEY AUTOINCREMENT, tbl TEXT NOT NULL, op TEXT NOT NULL, k1 TEXT NOT NULL, k2 TEXT)',
 ];
 
+/** One chunk of swept or pumped work holds the write lock and the event loop about this long, then yields. */
+export const CHUNK_MS = 50;
+/**
+ * Between chunks the lock is left free this long (writeInChunks' pause, without blocking the loop); longer while
+ * another connection is writing (it committed since the last pause), because its busy handler sleeps up to 100 ms
+ * between tries and would otherwise rarely land in a short gap.
+ */
+export const CHUNK_PAUSE_MS = 15;
+export const WRITER_PAUSE_MS = 120;
+const JOURNAL_PAGE = 200;
+
 function triggerSql({ table, keys, ops }) {
   return ops.map((op) => {
     const row = op === 'DELETE' ? 'OLD' : 'NEW';
@@ -37,30 +49,40 @@ export function installCapture(db) {
  * Turns every captured hub write into outbox commands inside the transaction that made it.
  *
  * `tx(fn, { guard })` runs fn in one write transaction (the ported core's own withTx calls join it), drains the trigger
- * journal through `mapping.commands(change, db)` into the outbox, then runs `guard`; a guard that throws rolls the
+ * rows fn added through `mapping.commands(change, db)` into the outbox, then runs `guard`; a guard that throws rolls the
  * hub write and its commands back together. `applying(fn)` writes with the triggers off, for rows that came from the
  * cloud: other connections never see the flag set, because SQLite serializes writers and the flag is set and reset
- * inside one write transaction. `pump()` drains what writes outside any `tx` left in the journal.
+ * inside one write transaction. `pump({ budgetMs })` drains what writes on other connections (the collect worker) left
+ * in the journal, oldest first, for at most budgetMs per transaction. `chunks(steps)` runs steps in capture
+ * transactions of about CHUNK_MS each, leaving the event loop and the lock free between them (`yieldLock`).
  */
 export function createCapture({ db, store, mapping, log = () => {} }) {
-  const journal = db.prepare('SELECT id, tbl, op, k1, k2 FROM pulse_capture ORDER BY id');
-  const clear = db.prepare('DELETE FROM pulse_capture WHERE id <= ?');
+  const journal = db.prepare('SELECT id, tbl, op, k1, k2 FROM pulse_capture WHERE id > ? ORDER BY id LIMIT ?');
+  const newest = db.prepare('SELECT COALESCE(MAX(id), 0) AS id FROM pulse_capture');
+  const waiting = db.prepare('SELECT EXISTS (SELECT 1 FROM pulse_capture) AS found');
+  const clear = db.prepare('DELETE FROM pulse_capture WHERE id > ? AND id <= ?');
   const setApplying = db.prepare('UPDATE pulse_ctx SET applying = ? WHERE one = 1');
+  const dataVersion = db.prepare('PRAGMA data_version');
+  let lastVersion = null;
+  const batchOf = () => (mapping.batch ? mapping.batch(db) : (change) => mapping.commands(change, db));
 
-  function drain() {
-    const rows = journal.all();
-    if (!rows.length) return 0;
-    const latest = new Map();
-    for (const row of rows) {
-      const id = `${row.tbl}\0${row.k1}\0${row.k2 ?? ''}`;
-      latest.delete(id);
-      latest.set(id, row);
-    }
+  const entityOf = (row) => `${row.tbl}\0${row.k1}\0${row.k2 ?? ''}`;
+
+  /**
+   * Commands for one page of journal rows, each entity once at its latest row in the page, in row order. Stops after
+   * the row at which `over()` turns true; returns the id of the last row it is done with and what it queued. Rows
+   * before that whose entity comes again later in the page are left to that later row.
+   */
+  function drainPage(rows, commands, over) {
+    const lastAt = new Map(rows.map((row, index) => [entityOf(row), index]));
     const entries = [];
-    for (const { tbl, op, k1, k2 } of latest.values()) {
+    let done = rows.length - 1;
+    for (const [index, row] of rows.entries()) {
+      if (lastAt.get(entityOf(row)) !== index) continue;
+      const { tbl, op, k1, k2 } = row;
       const change = { tbl, op, k1, k2 };
       try {
-        entries.push(...mapping.commands(change, db));
+        entries.push(...commands(change));
       } catch (error) {
         // A full outbox refuses the write itself (507 pulse_backlog), so the caller is told.
         if (error.code === 'pulse_backlog') throw error;
@@ -71,16 +93,36 @@ export function createCapture({ db, store, mapping, log = () => {} }) {
         );
         log('capture_failed', { tbl, op, code: error?.code || 'capture_failed' });
       }
+      if (index < rows.length - 1 && over()) { done = index; break; }
     }
     if (entries.length) store.enqueue(entries);
-    clear.run(rows.at(-1).id);
-    return entries.length;
+    return { last: rows[done].id, queued: entries.length, all: done === rows.length - 1 };
+  }
+
+  /** Drains journal rows after `after`, a page at a time, until none are left or `budgetMs` has passed. */
+  function drain({ after = 0, budgetMs = Infinity } = {}) {
+    const started = performance.now();
+    const over = () => performance.now() - started >= budgetMs;
+    const commands = batchOf();
+    let last = after;
+    let queued = 0;
+    for (;;) {
+      const rows = journal.all(last, JOURNAL_PAGE);
+      if (!rows.length) break;
+      const page = drainPage(rows, commands, over);
+      queued += page.queued;
+      last = page.last;
+      if (!page.all || rows.length < JOURNAL_PAGE || over()) break;
+    }
+    if (last > after) clear.run(after, last);
+    return queued;
   }
 
   function tx(fn, { guard = null, label = 'capture' } = {}) {
     return withTx(db, () => {
+      const mark = newest.get().id;
       const out = fn();
-      drain();
+      drain({ after: mark });
       if (guard) guard();
       return out;
     }, label);
@@ -97,7 +139,28 @@ export function createCapture({ db, store, mapping, log = () => {} }) {
     }, 'apply');
   }
 
-  const pump = () => withTx(db, drain, 'capture pump');
+  const pump = ({ budgetMs = Infinity } = {}) => withTx(db, () => drain({ budgetMs }), 'capture pump');
+  const pending = () => Boolean(waiting.get().found);
+  const yieldLock = () => {
+    const version = dataVersion.get().data_version;
+    const writing = lastVersion !== null && version !== lastVersion;
+    lastVersion = version;
+    return sleep(writing ? WRITER_PAUSE_MS : CHUNK_PAUSE_MS);
+  };
+
+  async function chunks(steps, { budgetMs = CHUNK_MS, label = 'capture chunk' } = {}) {
+    let i = 0;
+    let n = 0;
+    while (i < steps.length) {
+      const started = performance.now();
+      n += 1;
+      tx(() => {
+        do steps[i++](); while (i < steps.length && performance.now() - started < budgetMs);
+      }, { label: `${label} (chunk ${n})` });
+      if (i < steps.length) await yieldLock();
+    }
+    return n;
+  }
 
-  return { tx, applying, pump, drain };
+  return { tx, applying, pump, pending, chunks, yieldLock, drain };
 }