@agentpulselabs/cli-plugin-pulse 1.1.5 → 1.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@agentpulselabs/cli-plugin-pulse",
3
- "version": "1.1.5",
3
+ "version": "1.1.6",
4
4
  "description": "AgentPulse CLI plugin for installing and operating the Pulse local connector (replaces the deprecated build-discipline Pulse plugin).",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -21,7 +21,7 @@
21
21
  "scripts": {
22
22
  "test": "node --import ./test/support/register-loader.mjs --test \"test/*.test.js\" \"test/hub/*.test.js\"",
23
23
  "contract:check": "node scripts/sync-contract.mjs --local",
24
- "check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cutover.js && node --check src/lib/cutoverWiring.js && node --check src/lib/cutoverRequest.js && node --check src/commands/pulse/cutover.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/loopMonitor.js && node --check src/runner/channelWalk.js && node --check src/runner/serial.js && node --check src/runner/directoryCache.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
24
+ "check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cutover.js && node --check src/lib/cutoverWiring.js && node --check src/lib/cutoverRequest.js && node --check src/commands/pulse/cutover.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/loopMonitor.js && node --check src/runner/channelWalk.js && node --check src/runner/serial.js && node --check src/runner/directoryCache.js && node --check src/runner/channelFiles.js && node --check src/runner/fileReads.js && node --check src/runner/fileReadsWorker.js && node --check src/runner/transcriptFeed.js && node --check src/runner/collectWorker.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
25
25
  "test:t2": "node --import ./test/support/register-loader.mjs --test --test-timeout=120000 \"test/t2/*.test.js\"",
26
26
  "test:perf": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=900000 \"test/perf/*.test.js\"",
27
27
  "test:t3": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=240000 \"test/t3/*.test.js\""
@@ -268,6 +268,25 @@ export function writeInChunks(db, steps, { budgetMs = 50, pauseMs = 15, label =
268
268
  return chunks;
269
269
  }
270
270
 
271
+ /**
272
+ * Pulse runner-stalls round 2: when discovery last saved a pass. A pass rewrites only the rows that moved, so a row's
273
+ * updated_at no longer says when a pass last confirmed it; this does. Kept out of the facts, where capture and the
274
+ * change feed would see it on every pass.
275
+ */
276
+ export function recordDiscoveryPass(db, at = Date.now()) {
277
+ runSql(db, 'CREATE TABLE IF NOT EXISTS discovery_pass (id INTEGER PRIMARY KEY CHECK (id = 1), at INTEGER NOT NULL)');
278
+ db.prepare('INSERT INTO discovery_pass (id, at) VALUES (1, ?) ON CONFLICT (id) DO UPDATE SET at = excluded.at').run(at);
279
+ }
280
+
281
+ /** When discovery last saved a pass (recordDiscoveryPass), or 0 before the first. */
282
+ export function lastDiscoveryPass(db) {
283
+ try {
284
+ return Number(db.prepare('SELECT at FROM discovery_pass WHERE id = 1').get()?.at) || 0;
285
+ } catch {
286
+ return 0;
287
+ }
288
+ }
289
+
271
290
  export function jsonCol(v) {
272
291
  if (v == null) return null;
273
292
  return typeof v === 'string' ? v : JSON.stringify(v);
@@ -4,6 +4,7 @@
4
4
  // and again after WAKE_AGAIN_MS if your message is still waiting.
5
5
  import { ADDRESS, managerChats } from './ids.mjs';
6
6
  import { edgeId } from './edgeIds.mjs';
7
+ import { lastDiscoveryPass } from './db.mjs';
7
8
 
8
9
  export const WAKE_AGAIN_MS = 15 * 60e3;
9
10
  /** No hook event for this long: the subagent is not running (long commands can be silent for minutes). */
@@ -59,8 +60,9 @@ const nudgeText = (every) => `Scheduled check-in from the hub (every ${every} mi
59
60
  /**
60
61
  * Agents finished by what the Board shows: a stream whose final review passed, or a done agent row or delivery. Rows
61
62
  * and deliveries are only as fresh as the last collect, so one older than the agent's last sign of life does not count.
63
+ * A row is as fresh as the last saved pass (`passAt`), which rewrites only the rows that moved.
62
64
  */
63
- function doneIds(registry, actors, seen) {
65
+ function doneIds(registry, actors, seen, passAt = 0) {
64
66
  const rows = new Map(registry.facts('agent-row').map((r) => [r.key, r]));
65
67
  const deliveries = new Map(registry.facts('delivery').map((d) => [d.key, d]));
66
68
  const done = new Set();
@@ -69,7 +71,7 @@ function doneIds(registry, actors, seen) {
69
71
  const key = a.stream_key || (row && row.streamKey);
70
72
  const stream = key && registry.fact('stream', key);
71
73
  const d = a.node_type === 'orchestrator' && deliveries.get(a.id);
72
- const current = (f) => f && f.status === 'done' && f.updated_at >= (seen.get(a.id) || 0);
74
+ const current = (f) => f && f.status === 'done' && Math.max(f.updated_at, passAt) >= (seen.get(a.id) || 0);
73
75
  if ((stream && stream.done) || current(row) || current(d)) done.add(a.id);
74
76
  }
75
77
  return done;
@@ -91,7 +93,7 @@ export function scheduledNudge(hub, { everyMin, now = Date.now(), skip = null, r
91
93
  for (const chat of managerChats(registry.fact('manager', 'config'))) exclude.add(chat);
92
94
  if (skip) for (const id of skip(last)) exclude.add(id);
93
95
  if (released) for (const id of released()) pending.delete(id);
94
- const targets = nudgeTargets({ actors, seen, done: doneIds(registry, actors, seen), pending, exclude, now });
96
+ const targets = nudgeTargets({ actors, seen, done: doneIds(registry, actors, seen, lastDiscoveryPass(db)), pending, exclude, now });
95
97
  const dir = targets.length ? registry.directory(now) : [];
96
98
  const sent = [];
97
99
  for (const t of targets) {
@@ -2,7 +2,7 @@
2
2
  // Discovery worker: scans transcripts and channels, writes inferred actors and facts. Does not serve GET.
3
3
  import crypto from 'node:crypto';
4
4
  import { DERIVED_REQUEST_PREFIX, collect } from '../lib/collect.mjs';
5
- import { writeInChunks } from '../core/db.mjs';
5
+ import { recordDiscoveryPass, writeInChunks } from '../core/db.mjs';
6
6
  import { nodeTypeOf } from '../core/ids.mjs';
7
7
  import { projectGraph } from '../projections/graph.mjs';
8
8
 
@@ -16,6 +16,15 @@ export const DISCOVERY_REUSE_MS = 30e3;
16
16
  export const CANONICAL_HOURS = 168;
17
17
 
18
18
  const lastWritten = new WeakMap();
19
+ /**
20
+ * Pulse runner-stalls: per db, every row the last saved pass wrote, as hashes of what it wrote and of the row it left.
21
+ * A row whose input and current row both still match is not written again, so a pass where little moved takes the
22
+ * write lock, and gives capture journal rows, only for what did.
23
+ */
24
+ const lastRows = new WeakMap();
25
+ const rowHash = (text) => crypto.createHash('sha1').update(String(text)).digest('base64');
26
+ /** The message ids conversations.importTranscriptTurn and forgetOpening use for a session's opening turn. */
27
+ const openingIds = (id) => { const base = `t-${`${id}-open`.slice(0, 40)}`; return [`${base}-p`, `${base}-r`]; };
19
28
 
20
29
  /** What a pass would write, minus what moves on every pass by itself (generatedAt, the manager node's window). */
21
30
  function discoverySignature(registry, graph) {
@@ -26,7 +35,7 @@ function discoverySignature(registry, graph) {
26
35
  }
27
36
 
28
37
  /**
29
- * Saves a discovery pass. Returns false, writing nothing, when it matches the last pass saved through this db. A fact
38
+ * Saves a discovery pass. Returns false, writing no rows (only the pass time), when it matches the last pass saved through this db. A fact
30
39
  * kind written here must be listed in DISCOVERY_FACTS (core/changes.mjs) or each pass wakes the change feed.
31
40
  *
32
41
  * A full pass is ~1.5k actor and ~1.1k fact writes (0.3-0.4 s on the live hub.db), so it goes in short transactions
@@ -36,18 +45,39 @@ function discoverySignature(registry, graph) {
36
45
  export function persistDiscovery(hub, graph, chunks = {}) {
37
46
  const { registry, conversations, db } = hub;
38
47
  const signature = discoverySignature(registry, graph);
39
- if (lastWritten.get(db) === signature) return false;
48
+ if (lastWritten.get(db) === signature) {
49
+ // Nothing moved, but this pass still confirmed every row (core/wake.mjs doneIds reads when).
50
+ writeInChunks(db, [() => recordDiscoveryPass(db)], { label: 'discovery pass', ...chunks });
51
+ return false;
52
+ }
40
53
  const now = graph.generatedAt || Date.now();
54
+ const rows = lastRows.get(db) || new Map();
55
+ const kept = new Map();
41
56
  const steps = [];
42
- const step = (fn) => steps.push(fn);
43
- for (const n of graph.nodes || []) step(() => {
57
+ /** A keyed step is skipped while `input` and the row `read` returns are what the last saved pass left. */
58
+ const step = (fn, key = null, input = null, read = null) => {
59
+ if (key === null) { steps.push(fn); return; }
60
+ const want = rowHash(JSON.stringify(input));
61
+ const was = rows.get(key);
62
+ if (was && was.input === want && was.row === rowHash(read())) { kept.set(key, was); return; }
63
+ steps.push(() => { fn(); kept.set(key, { input: want, row: rowHash(read()) }); });
64
+ };
65
+ const actorRow = db.prepare('SELECT * FROM actors WHERE id = ?');
66
+ const factRow = db.prepare('SELECT payload FROM facts WHERE kind = ? AND key = ?');
67
+ const factKeys = db.prepare("SELECT kind, key FROM facts WHERE kind IN ('delivery', 'stream', 'agent-row') ORDER BY kind, key");
68
+ const restRows = db.prepare("SELECT kind, key, payload FROM facts WHERE kind IN ('request', 'totals') ORDER BY kind, key");
69
+ const openingRows = db.prepare('SELECT m.id, d.recipient FROM messages m LEFT JOIN deliveries d ON d.message_id = m.id WHERE m.id IN (?, ?) ORDER BY m.id');
70
+ const readActor = (id) => () => JSON.stringify(actorRow.get(id) ?? null);
71
+ const readFact = (kind, key) => () => factRow.get(kind, key)?.payload ?? '';
72
+ const readOpening = (id) => () => JSON.stringify(openingRows.all(...openingIds(id)));
73
+ for (const n of graph.nodes || []) {
44
74
  const parentId = n.parent || n.launchedBy || null;
45
75
  const synthetic = !!n.synthetic;
46
76
  const nodeType = n.node_type || nodeTypeOf({
47
77
  id: n.id, parentId, synthetic, role: n.role,
48
78
  });
49
79
  const persona = n.persona || {};
50
- registry.upsertInferred({
80
+ const patch = {
51
81
  id: n.id,
52
82
  node_type: nodeType,
53
83
  role: n.role || (nodeType === 'orchestrator' ? 'ARCHITECT' : null),
@@ -65,22 +95,29 @@ export function persistDiscovery(hub, graph, chunks = {}) {
65
95
  started_at: n.startedAt, last_at: n.lastAt, tokens_est: n.tokensEst,
66
96
  msgs_in: n.msgsIn, msgs_out: n.msgsOut, repo: n.repo || n.project, channel: n.channel,
67
97
  synthetic, extra: { streams: n.streams, status: n.status, kind: n.kind },
68
- });
98
+ };
99
+ step(() => registry.upsertInferred(patch), `actor:${n.id}`, patch, readActor(n.id));
69
100
  // Presence is owned by the hook heartbeat. Discovery only records last_at on the actor.
70
- });
101
+ }
71
102
  const deliveries = graph.deliveries || [];
72
103
  const accepted = new Map(registry.facts('accepted').map((a) => [a.key, a.at || a.updated_at]));
73
- for (const d of deliveries) step(() => registry.setFact('delivery', d.id, accepted.has(d.id) ? { ...d, acceptedAt: accepted.get(d.id) } : d));
74
- for (const d of deliveries) for (const s of d.streams || []) step(() => registry.setFact('stream', s.key, { ...s, deliveryId: d.id }));
75
- for (const a of graph.agents || []) step(() => registry.setFact('agent-row', a.id, a));
104
+ const setFact = (kind, key, payload) => step(() => registry.setFact(kind, key, payload), `fact:${kind}:${key}`, payload, readFact(kind, key));
105
+ for (const d of deliveries) setFact('delivery', d.id, accepted.has(d.id) ? { ...d, acceptedAt: accepted.get(d.id) } : d);
106
+ for (const d of deliveries) for (const s of d.streams || []) setFact('stream', s.key, { ...s, deliveryId: d.id });
107
+ for (const a of graph.agents || []) setFact('agent-row', a.id, a);
108
+ const keys = {
109
+ delivery: deliveries.map((d) => d.id),
110
+ stream: deliveries.flatMap((d) => (d.streams || []).map((s) => s.key)),
111
+ 'agent-row': (graph.agents || []).map((a) => a.id),
112
+ };
76
113
  step(() => {
77
- registry.keepFacts('delivery', deliveries.map((d) => d.id));
78
- registry.keepFacts('stream', deliveries.flatMap((d) => (d.streams || []).map((s) => s.key)));
79
- registry.keepFacts('agent-row', (graph.agents || []).map((a) => a.id));
80
- });
81
- for (const n of graph.nodes || []) if (n.kind === 'agent') step(() => conversations.forgetOpening(n.id));
114
+ registry.keepFacts('delivery', keys.delivery);
115
+ registry.keepFacts('stream', keys.stream);
116
+ registry.keepFacts('agent-row', keys['agent-row']);
117
+ }, 'keep', keys, () => JSON.stringify(factKeys.all()));
118
+ for (const n of graph.nodes || []) if (n.kind === 'agent') step(() => conversations.forgetOpening(n.id), `forget:${n.id}`, null, readOpening(n.id));
82
119
  const requests = (graph.requests || []).map((r) => [r.id || `r-${r.at}`, r]);
83
- for (const [key, r] of requests) step(() => registry.setFact('request', key, r));
120
+ for (const [key, r] of requests) setFact('request', key, r);
84
121
  step(() => {
85
122
  // Only post-derived requests are discovery's to drop; filed ones (hub.mjs request) live as facts alone.
86
123
  const current = new Set(requests.map(([key]) => key));
@@ -90,18 +127,21 @@ export function persistDiscovery(hub, graph, chunks = {}) {
90
127
  // The projection counts each window's totals from this one basis; per-window facts older passes wrote are dropped.
91
128
  if (graph.totalsBasis) registry.setFact('totals', 'basis', graph.totalsBasis);
92
129
  registry.keepFacts('totals', graph.totalsBasis ? ['basis'] : []);
93
- });
130
+ }, 'requests', [[...requests.map(([key]) => key)].sort(), graph.totalsBasis ?? null], () => JSON.stringify(restRows.all()));
94
131
  for (const n of graph.nodes || []) {
95
132
  if (n.kind !== 'session' || n.parent || n.launchedBy) continue;
96
133
  if (n.promptTitle || n.title) {
134
+ // A transcript turn is inserted once and never updated, so a pass's own clock is no part of its input.
97
135
  step(() => conversations.importTranscriptTurn({
98
136
  sessionId: n.id, turnId: `${n.id}-open`, at: n.startedAt || now,
99
137
  prompt: n.promptTitle || n.title, reply: null,
100
- }));
138
+ }), `opening:${n.id}`, [n.startedAt || null, n.promptTitle || n.title], readOpening(n.id));
101
139
  }
102
140
  }
141
+ step(() => recordDiscoveryPass(db));
103
142
  writeInChunks(db, steps, { label: 'discovery', ...chunks });
104
143
  lastWritten.set(db, signature);
144
+ lastRows.set(db, kept);
105
145
  return true;
106
146
  }
107
147
 
@@ -150,6 +150,7 @@ export class MirrorSource {
150
150
  this.redaction = new UploadRedactor({ workspace, home, approvedRoots });
151
151
  this.windowCache = new Map();
152
152
  this.rowMemo = new RowMemo();
153
+ this.itemMemo = new RowMemo();
153
154
  this.briefs = new Map();
154
155
  this.taskKeys = null;
155
156
  }
@@ -388,12 +389,18 @@ export class MirrorSource {
388
389
  checkpoint();
389
390
  const secrets = [await this.reader.tokenValue().catch(() => null), ...this.heldSecrets()].filter(Boolean);
390
391
  const redactor = this.redaction.redactor(secrets);
392
+ const basis = this.redaction.basis(secrets);
391
393
  const items = new Map();
394
+ // An item's redacted, checked form depends on its kind and data and on the redaction basis, so one that did not
395
+ // move since the last pass is not redacted and checked again.
396
+ this.itemMemo.begin(basis);
392
397
  for (const entry of raw) {
393
398
  if (typeof entry.externalId !== 'string' || !entry.externalId) continue;
394
399
  const externalId = externalIdFor(entry.externalId);
395
- const redacted = redactor.redactValue(entry.data, { keep: entry.kind === 'rule_pack' ? ['file'] : [] });
396
- const result = sanitizeItem(entry.kind, entry.kind === 'brief' ? markTruncated(redacted) : redacted, { secrets });
400
+ const result = this.itemMemo.take(`${entry.kind}\0${JSON.stringify(entry.data)}`, () => {
401
+ const redacted = redactor.redactValue(entry.data, { keep: entry.kind === 'rule_pack' ? ['file'] : [] });
402
+ return sanitizeItem(entry.kind, entry.kind === 'brief' ? markTruncated(redacted) : redacted, { secrets });
403
+ });
397
404
  if (result.error) {
398
405
  withheld.push({ kind: entry.kind, externalId, error: result.error });
399
406
  continue;
@@ -405,19 +412,25 @@ export class MirrorSource {
405
412
  const graphDropped = {};
406
413
  if (windows.length === WINDOW_HOURS.length) {
407
414
  // A row's redacted, checked form depends on the roots the redactor knows and the secrets, and on nothing else.
408
- this.rowMemo.begin(this.redaction.basis(secrets));
415
+ this.rowMemo.begin(basis);
409
416
  const options = { redact: (value) => redactor.redactValue(value), secrets, memo: this.rowMemo };
410
- const built = mergeWindows(windows.map(({ hours, body, fresh }) => {
417
+ const perWindow = windows.map(({ hours, body, fresh }) => {
411
418
  if (!fresh) return this.windowCache.get(hours).rows;
412
419
  const rows = windowRows(hours, body, options);
413
420
  this.windowCache.set(hours, { body, rows });
414
421
  return rows;
415
- }));
422
+ });
423
+ // Each checked row's canonical text, so an item is keyed by its windows and that text, not stringified again.
424
+ const textOf = new Map(perWindow.flatMap(({ rows }) => rows.map(({ row, text }) => [row, text])));
425
+ const built = mergeWindows(perWindow);
416
426
  Object.assign(graphDropped, built.dropped);
417
427
  const updatedAt = firstTime(Math.max(...windows.map(({ body }) => Number(body.generatedAt) || 0)), fallback);
418
428
  for (const row of built.items) {
419
429
  const externalId = externalIdFor(row.externalId);
420
- const result = this.rowMemo.take(`item\0${JSON.stringify(row.data)}`, () => sanitizeItem('graph_window', row.data, { secrets }));
430
+ const { section, windows: held } = row.data;
431
+ const text = textOf.get(row.data[section]);
432
+ const key = text === undefined ? JSON.stringify(row.data) : `${section}\0${held.join(',')}\0${text}`;
433
+ const result = this.rowMemo.take(`item\0${key}`, () => sanitizeItem('graph_window', row.data, { secrets }));
421
434
  if (result.error) {
422
435
  withheld.push({ kind: 'graph_window', externalId, error: result.error });
423
436
  continue;
@@ -204,8 +204,18 @@ export function createBusIdentity(hub, { now: clock = Date.now } = {}) {
204
204
  refreshProof: db.prepare("INSERT INTO pulse_proofs (address, at, via, used) VALUES (?, ?, 'listen', 0) ON CONFLICT (address) DO UPDATE SET at = MAX(pulse_proofs.at, excluded.at)"),
205
205
  useProof: db.prepare('UPDATE pulse_proofs SET used = 1 WHERE address = ?'),
206
206
  remote: db.prepare('SELECT * FROM pulse_remote_presence WHERE actor_id = ?'),
207
+ queuedFor: db.prepare(`SELECT 1 AS found FROM deliveries d JOIN messages m ON m.id = d.message_id
208
+ WHERE d.recipient = ? AND d.state = 'queued' AND m.expires_at > ? LIMIT 1`),
209
+ lapsedFor: db.prepare(`SELECT 1 AS found FROM deliveries d JOIN messages m ON m.id = d.message_id
210
+ WHERE d.recipient = ? AND d.state = 'leased' AND d.lease_until < ? AND m.expires_at > ? LIMIT 1`),
207
211
  };
208
212
 
213
+ /**
214
+ * Whether messaging.claim would find anything for one of `addresses` now (its own rows: queued, or leased with a lapsed
215
+ * lease, and not expired), read without the write lock, so an empty `bus listen` poll needs no write transaction.
216
+ */
217
+ const claimable = (addresses, now = clock()) => addresses.some((a) => q.queuedFor.get(a, now) || q.lapsedFor.get(a, now, now));
218
+
209
219
  /**
210
220
  * Every chat the hub already knows to be a session, before its next hook: hook-seen and transcript-found actors, and
211
221
  * presence rows whose last event came from a hook. Without this, right after an upgrade another session could take a
@@ -464,7 +474,7 @@ export function createBusIdentity(hub, { now: clock = Date.now } = {}) {
464
474
  return { registrations: shown.map(({ active: _active, ...e }) => e), groups: [...groups.values()], window: { minutes: windowMs / 60e3, all: !!all } };
465
475
  }
466
476
 
467
- return { addressesOf, inboxFor, sessionOf, aliasesOf, activity, known, suggest, reach, sawHook, proved, listened, bindFromHook, notices, who };
477
+ return { addressesOf, inboxFor, claimable, sessionOf, aliasesOf, activity, known, suggest, reach, sawHook, proved, listened, bindFromHook, notices, who };
468
478
  }
469
479
 
470
480
  /**
@@ -0,0 +1,39 @@
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import { readTaskBrief } from '../hub/lib/brief.mjs';
4
+ import { findAgentQuery } from '../hub/lib/transcripts.mjs';
5
+
6
+ const BRIEF_CACHE_MS = 30e3;
7
+ const BRIEF_CACHE_MAX = 200;
8
+ const briefCache = new Map();
9
+
10
+ /** A task channel's file under one of the roots, or null; a path that leaves its root through a link is refused. */
11
+ export function channelFileFor(key, roots) {
12
+ const m = String(key || '').match(/^([A-Za-z0-9._-]+)\/([A-Za-z0-9._-]+)$/);
13
+ if (!m || m[1].startsWith('.') || m[2].startsWith('.')) return null;
14
+ for (const root of roots) {
15
+ const file = path.join(root, m[1], '.agents', m[2], 'channel.md');
16
+ try {
17
+ const st = fs.lstatSync(file);
18
+ if (!st.isFile()) continue;
19
+ const realRoot = fs.realpathSync(root);
20
+ if (fs.realpathSync(file) !== path.join(realRoot, m[1], '.agents', m[2], 'channel.md')) continue;
21
+ return file;
22
+ } catch { /* not here */ }
23
+ }
24
+ return null;
25
+ }
26
+
27
+ /** A task's brief: brief.md and the spec, else the longest opening prompt that names the channel (a transcript scan). */
28
+ export function taskBriefFor(key, roots) {
29
+ const hit = briefCache.get(key);
30
+ if (hit && Date.now() - hit.at < BRIEF_CACHE_MS) return hit.body;
31
+ const file = channelFileFor(key, roots);
32
+ if (!file) return null;
33
+ const fromFiles = readTaskBrief(file);
34
+ const thin = !fromFiles || !fromFiles.sections.some((s) => s.body && s.body.trim());
35
+ const body = thin ? (readTaskBrief(file, { query: findAgentQuery(key) }) || fromFiles || { title: null, source: null, sections: [] }) : fromFiles;
36
+ if (briefCache.size > BRIEF_CACHE_MAX) briefCache.clear();
37
+ briefCache.set(key, { at: Date.now(), body });
38
+ return body;
39
+ }
@@ -0,0 +1,92 @@
1
+ // The runner's discovery worker: the hub's lib/collect-worker.mjs, except that once a pass is saved a graph read never
2
+ // waits for the next one. A read answers from the rows the last pass saved and, when its epoch is newer or that pass is
3
+ // older than DISCOVERY_REUSE_MS, asks for a new pass, which runs and writes its rows after the reads that came with it.
4
+ // The runner's first read still waits for the first pass. `{ type: 'discovered' }` tells the facade a pass was saved.
5
+ import path from 'node:path';
6
+ import { parentPort } from 'node:worker_threads';
7
+ import { openHub } from '../hub/core/hub.mjs';
8
+ import { hubHome, runnerDb } from '../hub/home.mjs';
9
+ import { collect } from '../hub/lib/collect.mjs';
10
+ import { projectGraph } from '../hub/projections/graph.mjs';
11
+ import { CANONICAL_HOURS, DISCOVERY_REUSE_MS, persistDiscovery } from '../hub/workers/collector.mjs';
12
+
13
+ /** A pass starts once reads have stopped this long, so a mirror pass's window reads are all answered before it. */
14
+ const SETTLE_MS = 100;
15
+ /** Reads that never stop for SETTLE_MS still let a wanted pass start this soon after it was first wanted. */
16
+ const SETTLE_MAX_MS = 2000;
17
+
18
+ const hub = openHub(path.join(hubHome(), 'state'), runnerDb());
19
+ let saved = null;
20
+ let wanted = null;
21
+ let timer = null;
22
+ let wantedSince = null;
23
+
24
+ const discover = (epoch, aliases) => {
25
+ persistDiscovery(hub, collect({ hours: CANONICAL_HOURS, runs: hub.registry.facts('run'), aliases: aliases || [] }));
26
+ saved = { epoch, at: Date.now() };
27
+ };
28
+ const stale = (epoch) => !saved || epoch == null || saved.epoch !== epoch || Date.now() - saved.at >= DISCOVERY_REUSE_MS;
29
+
30
+ const passSoon = () => {
31
+ clearTimeout(timer);
32
+ wantedSince ??= Date.now();
33
+ timer = setTimeout(() => {
34
+ timer = null;
35
+ wantedSince = null;
36
+ const next = wanted;
37
+ wanted = null;
38
+ if (!next || !stale(next.epoch)) return;
39
+ try {
40
+ discover(next.epoch, next.aliases);
41
+ parentPort.postMessage({ type: 'discovered' });
42
+ } catch (e) {
43
+ console.error(`discovery failed: ${(e && e.stack) || e}`);
44
+ }
45
+ }, Math.min(SETTLE_MS, Math.max(0, wantedSince + SETTLE_MAX_MS - Date.now())));
46
+ };
47
+
48
+ /** Registry reads the windows of one burst all make: read once per burst; the projections copy what they change. */
49
+ const SHARED_READS = ['all', 'get', 'presence', 'presences', 'fact', 'facts', 'directory', 'registrations', 'eventsSince'];
50
+ function burstView() {
51
+ const memo = new Map();
52
+ const registry = Object.create(hub.registry);
53
+ for (const name of SHARED_READS) {
54
+ const read = hub.registry[name];
55
+ registry[name] = (...args) => {
56
+ const key = `${name}\0${JSON.stringify(args)}`;
57
+ if (!memo.has(key)) memo.set(key, read.apply(hub.registry, args));
58
+ return memo.get(key);
59
+ };
60
+ }
61
+ return { ...hub, registry };
62
+ }
63
+
64
+ /**
65
+ * Reads that queued while the worker was busy (a mirror pass reads every window at once) are projected together, at
66
+ * one clock and over one set of registry reads.
67
+ */
68
+ let burst = [];
69
+ function project() {
70
+ const reads = burst;
71
+ burst = [];
72
+ const now = Date.now();
73
+ let view = saved ? burstView() : null;
74
+ for (const { id, hours, managerView, aliases, discoveryEpoch } of reads) {
75
+ try {
76
+ if (!saved) { discover(discoveryEpoch, aliases); view = burstView(); }
77
+ else if (stale(discoveryEpoch)) wanted = { epoch: discoveryEpoch, aliases };
78
+ // As JSON text the body crosses to the main thread as one string, not an object graph it must clone and stringify.
79
+ parentPort.postMessage({ id, text: JSON.stringify(projectGraph(view, { hours, managerView: managerView || {}, now })) });
80
+ } catch (e) {
81
+ parentPort.postMessage({ id, error: String((e && e.stack) || e) });
82
+ }
83
+ }
84
+ // Armed only now: a timer that came due while this burst projected would run before the reads queued behind it.
85
+ if (wanted) passSoon();
86
+ }
87
+
88
+ parentPort.on('message', (read) => {
89
+ burst.push(read);
90
+ if (burst.length === 1) setImmediate(project);
91
+ if (timer) passSoon();
92
+ });
@@ -23,12 +23,9 @@ import { isBusy } from '../hub/core/ticks.mjs';
23
23
  import { handleHook } from '../hub/hooks/hub-bridge.mjs';
24
24
  import { defaultDrivers, diagnoseClaude } from '../hub/runners/drivers.mjs';
25
25
  import { createRunner } from '../hub/workers/runner.mjs';
26
- import { readTaskBrief } from '../hub/lib/brief.mjs';
27
26
  import { decisionPost, parsePosts } from '../hub/lib/channels.mjs';
28
27
  import { STEP_IDS, auditPost, movePost, overrideHolds, overrideKey, overrideText, statusRequest, stepHolds, withOverride, workMoved } from '../hub/lib/status.mjs';
29
- import { askDetail, channelParticipant, hasChatInbox, listChannels, participantPost, participantThread, postToProject, projectMessages, taskMessages } from '../hub/lib/chat.mjs';
30
- import { findAgentQuery } from '../hub/lib/transcripts.mjs';
31
- import { watchTranscripts } from '../hub/lib/transcript-feed.mjs';
28
+ import { channelParticipant, hasChatInbox, participantPost, postToProject } from '../hub/lib/chat.mjs';
32
29
  import { HUB, acceptDelivery, keepAwakeStatus, readSettings, validSettings, writeSettings } from '../hub/lib/store.mjs';
33
30
  import { createManager } from '../hub/workers/manager.mjs';
34
31
  import { NOBODY, aliasesFor } from '../hub/lib/me.mjs';
@@ -37,8 +34,10 @@ import { projectDeliveries } from '../hub/projections/deliveries.mjs';
37
34
  import { projectGraph } from '../hub/projections/graph.mjs';
38
35
  import { execFileBounded } from '../lib/subprocess.js';
39
36
  import { createBusIdentity, routedMessaging } from './busIdentity.js';
37
+ import { channelFileFor as channelFileIn } from './channelFiles.js';
40
38
  import { createChannelDiscipline } from './channelDiscipline.js';
41
39
  import { createDeliveryStates } from './deliveryStates.js';
40
+ import { createFileReads } from './fileReads.js';
42
41
  import { UNMONITORED } from './loopMonitor.js';
43
42
  import { serial } from './serial.js';
44
43
  import { statusPage } from './statusPage.js';
@@ -48,6 +47,8 @@ const CHANNEL_ROOTS = ['AgentPulseApps', 'Google_Cloud'].map((d) => path.join(WO
48
47
  const HOURS = [1, 3, 6, 12, 24, 72, 168];
49
48
  const CACHE_MS = 4000;
50
49
  const CHANGE_DEBOUNCE_MS = 600;
50
+ /** A change feed news of transcripts alone starts a new discovery pass at most this often. */
51
+ const TRANSCRIPT_RESCAN_MS = 15e3;
51
52
  /** Steady writes (heartbeats every few hundred ms) keep restarting the debounce; a change still goes out this soon. */
52
53
  const CHANGE_MAX_WAIT_MS = 2000;
53
54
  /** A graph read re-registers the user at most this often: often enough to stay online, rarely enough not to wake the change feed on every read. */
@@ -66,10 +67,12 @@ const LOCAL_HOST = /^(127\.0\.0\.1|localhost)(:\d+)?$/;
66
67
  const LOCAL_ORIGIN = /^http:\/\/(127\.0\.0\.1|localhost)(:\d+)?$/;
67
68
  const NO_FRAME = { 'x-frame-options': 'DENY', 'content-security-policy': "frame-ancestors 'none'", 'x-content-type-options': 'nosniff' };
68
69
 
69
- const sendJson = (res, body, code = 200, extra = {}) => {
70
+ /** sendJson for a body already serialized (by the file reads worker): the same headers and bytes. */
71
+ const sendJsonText = (res, text, code = 200, extra = {}) => {
70
72
  res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store', ...NO_FRAME, ...extra });
71
- res.end(JSON.stringify(body));
73
+ res.end(text);
72
74
  };
75
+ const sendJson = (res, body, code = 200, extra = {}) => sendJsonText(res, JSON.stringify(body), code, extra);
73
76
 
74
77
  function trustedWrite(req) {
75
78
  const origin = req.headers.origin;
@@ -139,37 +142,7 @@ export function plainCapture(db) {
139
142
  };
140
143
  }
141
144
 
142
- const BRIEF_CACHE_MS = 30e3;
143
- const briefCache = new Map();
144
-
145
- export function channelFileFor(key, roots = CHANNEL_ROOTS) {
146
- const m = String(key || '').match(/^([A-Za-z0-9._-]+)\/([A-Za-z0-9._-]+)$/);
147
- if (!m || m[1].startsWith('.') || m[2].startsWith('.')) return null;
148
- for (const root of roots) {
149
- const file = path.join(root, m[1], '.agents', m[2], 'channel.md');
150
- try {
151
- const st = fs.lstatSync(file);
152
- if (!st.isFile()) continue;
153
- const realRoot = fs.realpathSync(root);
154
- if (fs.realpathSync(file) !== path.join(realRoot, m[1], '.agents', m[2], 'channel.md')) continue;
155
- return file;
156
- } catch { /* not here */ }
157
- }
158
- return null;
159
- }
160
-
161
- function taskBriefFor(key, roots) {
162
- const hit = briefCache.get(key);
163
- if (hit && Date.now() - hit.at < BRIEF_CACHE_MS) return hit.body;
164
- const file = channelFileFor(key, roots);
165
- if (!file) return null;
166
- const fromFiles = readTaskBrief(file);
167
- const thin = !fromFiles || !fromFiles.sections.some((s) => s.body && s.body.trim());
168
- const body = thin ? (readTaskBrief(file, { query: findAgentQuery(key) }) || fromFiles || { title: null, source: null, sections: [] }) : fromFiles;
169
- if (briefCache.size > 200) briefCache.clear();
170
- briefCache.set(key, { at: Date.now(), body });
171
- return body;
172
- }
145
+ export const channelFileFor = (key, roots = CHANNEL_ROOTS) => channelFileIn(key, roots);
173
146
 
174
147
  /** How many posts a channel holds now: the position of the one just appended. */
175
148
  const postsIn = (file) => parsePosts(fs.readFileSync(file, 'utf8'), Date.now()).length;
@@ -185,14 +158,15 @@ export function appendNoFollow(file, text) {
185
158
  } finally { fs.closeSync(fd); }
186
159
  }
187
160
 
188
- function collectWorker() {
189
- const worker = new Worker(new URL('../hub/lib/collect-worker.mjs', import.meta.url));
161
+ function collectWorker({ onDiscovered = () => {} } = {}) {
162
+ const worker = new Worker(new URL('./collectWorker.js', import.meta.url));
190
163
  const waiting = new Map();
191
164
  let seq = 0;
192
- worker.on('message', ({ id, body, error }) => {
165
+ worker.on('message', ({ type, id, text, error }) => {
166
+ if (type === 'discovered') { onDiscovered(); return; }
193
167
  const w = waiting.get(id);
194
168
  waiting.delete(id);
195
- if (w) { if (error) w.reject(new Error(error)); else w.resolve(body); }
169
+ if (w) { if (error) w.reject(new Error(error)); else w.resolve(text); }
196
170
  });
197
171
  worker.on('error', (e) => { console.error(`collector crashed: ${e && e.stack ? e.stack : e}`); process.exit(1); });
198
172
  return (hours, managerView, aliases, { discoveryEpoch } = {}) => {
@@ -203,12 +177,23 @@ function collectWorker() {
203
177
  };
204
178
  }
205
179
 
206
- function collector(run) {
180
+ function collector(run, { rescanMs = TRANSCRIPT_RESCAN_MS } = {}) {
207
181
  const cache = new Map();
208
182
  /** Bumped by clear() and reproject(): a collect started before a change must not cache its older body. */
209
183
  let generation = 0;
210
184
  /** Bumped by clear() only: the collector may reuse a discovery pass from the same epoch (workers/collector.mjs). */
211
185
  let discoveryEpoch = 0;
186
+ let epochAt = -Infinity;
187
+ let rescan = null;
188
+ const clear = () => {
189
+ clearTimeout(rescan);
190
+ rescan = null;
191
+ generation++;
192
+ discoveryEpoch++;
193
+ epochAt = Date.now();
194
+ cache.clear();
195
+ };
196
+ const reproject = () => { generation++; cache.clear(); };
212
197
  return {
213
198
  graph(hours, managerView, aliases = []) {
214
199
  const hit = cache.get(hours);
@@ -221,9 +206,20 @@ function collector(run) {
221
206
  cache.set(hours, { ...(hit || {}), running });
222
207
  return running;
223
208
  },
224
- clear() { generation++; discoveryEpoch++; cache.clear(); },
209
+ clear,
225
210
  /** Presence moved and nothing else: refresh the bodies without a new discovery pass. */
226
- reproject() { generation++; cache.clear(); },
211
+ reproject,
212
+ /**
213
+ * Only Cursor transcripts moved, which they do every second while agents work: a new discovery pass at most every
214
+ * `rescanMs`, the last change still picked up by a pass at the end of the wait; reads meanwhile reproject.
215
+ */
216
+ transcriptsMoved() {
217
+ const wait = epochAt + rescanMs - Date.now();
218
+ if (wait <= 0) return clear();
219
+ reproject();
220
+ if (!rescan) { rescan = setTimeout(clear, wait); rescan.unref(); }
221
+ return undefined;
222
+ },
227
223
  };
228
224
  }
229
225
 
@@ -234,7 +230,7 @@ function watchRecursive(dir, onFile, onError) {
234
230
  return fs.watch(dir, { recursive: true }, (e, f) => { if (f) onFile(String(f)); }).on('error', onError);
235
231
  }
236
232
 
237
- function changeFeed(onChange, { stateDir, dbFile, channelRoots, transcriptsDir, hubChanged, watchDir }) {
233
+ function changeFeed(onChange, { stateDir, dbFile, channelRoots, hubChanged, watchDir }) {
238
234
  const clients = new Set();
239
235
  const pending = new Set();
240
236
  let timer = null;
@@ -267,7 +263,6 @@ function changeFeed(onChange, { stateDir, dbFile, channelRoots, transcriptsDir,
267
263
  for (const root of channelRoots) watch(root, (f) => (/\.agents[\\/][^\\/]+[\\/](channel|brief)\.md$/.test(f) ? 'channels' : null));
268
264
  watch(stateDir, (f) => (/(events\.jsonl|requests|chat[\\/]|inbox[\\/]|bus[\\/]|accepted\.json|personas\.json)/.test(f) ? 'hub' : null));
269
265
  if (dbFile) watch(path.dirname(dbFile), (f) => (f.startsWith(dbName) ? DB_WRITE : null));
270
- if (transcriptsDir) watchTranscripts(transcriptsDir, () => notify('agents'));
271
266
  const keepAlive = setInterval(() => { for (const res of clients) res.write(': keep-alive\n\n'); }, 25e3);
272
267
  keepAlive.unref();
273
268
  return {
@@ -325,11 +320,23 @@ export function serve({
325
320
  const m = identity ? identity.current() : NOBODY;
326
321
  return m === NOBODY ? NOBODY : { ...m, aliases: aliasesFor(m, personaUsernames()) };
327
322
  };
328
- const graphs = enableCollector ? collector(collect || collectWorker()) : { graph: async () => ({ nodes: [], links: [], events: [], deliveries: [], agents: [], totals: {} }), clear() {}, reproject() {} };
323
+ // A discovery pass the worker saved after answering: later reads project its rows, and change-feed clients read again.
324
+ const graphs = enableCollector ? collector(collect || collectWorker({ onDiscovered: () => { graphs.reproject(); feed.notify('presence'); } })) : { graph: async () => ({ nodes: [], links: [], events: [], deliveries: [], agents: [], totals: {} }), clear() {}, reproject() {}, transcriptsMoved() {} };
329
325
  const caches = { clear: () => graphs.clear() };
326
+ const onFeedChange = (kinds) => {
327
+ if (kinds.every((k) => k === 'presence')) return graphs.reproject();
328
+ if (kinds.every((k) => k === 'presence' || k === 'agents')) return graphs.transcriptsMoved();
329
+ return graphs.clear();
330
+ };
330
331
  const feed = enableCollector
331
- ? changeFeed((kinds) => (kinds.every((k) => k === 'presence') ? graphs.reproject() : graphs.clear()), { stateDir, dbFile, channelRoots, transcriptsDir, hubChanged: changeTracker(hub.db), watchDir })
332
+ ? changeFeed(onFeedChange, { stateDir, dbFile, channelRoots, hubChanged: changeTracker(hub.db), watchDir })
332
333
  : { attach(req, res) { res.writeHead(200); res.end(); }, dbWrite() {}, notify() {}, close() {} };
334
+ // Channel, brief and transcript files are read and parsed off the event loop (fileReadsWorker.js).
335
+ const fileReads = createFileReads({ roots: channelRoots, transcriptsDir: enableCollector ? transcriptsDir : null, onTranscripts: () => feed.notify('agents') });
336
+ /** Answers a route from the file reads worker; `fallback` answers what the worker leaves to the hub (null). */
337
+ const fromFiles = (res, op, args, fallback = null) => fileReads.read(op, args, { text: true })
338
+ .then((answer) => (answer ? sendJsonText(res, ...answer) : fallback()))
339
+ .catch((e) => { console.error(`file read failed: ${op}: ${e.message}`); if (!res.headersSent) sendJson(res, { error: 'server error' }, 500); });
333
340
  if (identity) {
334
341
  identity.listen(() => { graphs.clear(); feed.notify('hub'); });
335
342
  identity.refresh();
@@ -526,7 +533,12 @@ export function serve({
526
533
  capture.tx(() => hub.registry.register({ id: 'user', role: 'USER', name: 'You', topics: ['user'], node_type: 'user' }), { label: 'graph user' });
527
534
  wrote();
528
535
  }
529
- return graphs.graph(hours, managerView(), me().aliases).then((body) => sendJson(res, { ...body, settings: { ...settingsOf(hub), me: me() } }))
536
+ return graphs.graph(hours, managerView(), me().aliases).then((body) => {
537
+ const settings = { ...settingsOf(hub), me: me() };
538
+ // The collect worker answers JSON text of an object without `settings`; appended last, the bytes are those of the spread.
539
+ if (typeof body === 'string' && body.length > 2 && body.endsWith('}')) return sendJsonText(res, `${body.slice(0, -1)},"settings":${JSON.stringify(settings)}}`);
540
+ return sendJson(res, { ...(typeof body === 'string' ? JSON.parse(body) : body), settings });
541
+ })
530
542
  .catch((e) => { console.error(`collect failed: ${e.message}`); if (!res.headersSent) sendJson(res, { error: 'server error' }, 500); });
531
543
  }
532
544
  if (p === '/api/settings' && req.method === 'GET') return sendJson(res, { ...settingsOf(hub), me: me() });
@@ -552,8 +564,7 @@ export function serve({
552
564
  if (p === '/api/task/brief' && req.method === 'GET') {
553
565
  const key = String(url.searchParams.get('key') || '');
554
566
  if (!/^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/.test(key)) return sendJson(res, { error: 'no such channel' }, 404);
555
- const brief = taskBriefFor(key, channelRoots);
556
- return brief ? sendJson(res, brief) : sendJson(res, { error: 'no such channel' }, 404);
567
+ return fromFiles(res, 'brief', { key });
557
568
  }
558
569
  if (p === '/api/runs' && req.method === 'GET') return sendJson(res, { runs: runner.list(), drivers: runner.drivers() });
559
570
  if (p === '/api/runs/diagnose' && req.method === 'GET') {
@@ -562,12 +573,10 @@ export function serve({
562
573
  }
563
574
  if (p === '/api/task/ask' && req.method === 'GET') {
564
575
  const key = String(url.searchParams.get('key') || '');
565
- const file = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/.test(key) ? channelFileFor(key, channelRoots) : null;
566
- if (!file) return sendJson(res, { error: 'no such channel' }, 404);
576
+ if (!/^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/.test(key)) return sendJson(res, { error: 'no such channel' }, 404);
567
577
  const ref = String(url.searchParams.get('ref') || '');
568
578
  const at = Number(url.searchParams.get('at')) || null;
569
- const detail = askDetail(file, { ref: /^[A-Z]?\d+$/.test(ref) ? ref : null, at, aliases: me().aliases });
570
- return detail ? sendJson(res, detail) : sendJson(res, { error: 'no open ask in this channel' }, 404);
579
+ return fromFiles(res, 'ask', { key, ref: /^[A-Z]?\d+$/.test(ref) ? ref : null, at, aliases: me().aliases });
571
580
  }
572
581
  if (p === '/api/inbox' && req.method === 'GET') {
573
582
  const since = Math.max(Date.now() - 24 * 3600e3, Number(url.searchParams.get('since')) || 0);
@@ -577,29 +586,16 @@ export function serve({
577
586
  return sendJson(res, { messages, at: Date.now() });
578
587
  }
579
588
  if (p === '/api/chat/channels' && req.method === 'GET') {
580
- const dms = hub.conversations.dmPartners();
581
- return sendJson(res, { ...listChannels(channelRoots, Date.now() - CHAT_CHANNELS_MS), dms });
589
+ return fileReads.read('channels', { sinceMs: Date.now() - CHAT_CHANNELS_MS })
590
+ .then(([listed]) => sendJson(res, { ...listed, dms: hub.conversations.dmPartners() }))
591
+ .catch((e) => { console.error(`file read failed: channels: ${e.message}`); if (!res.headersSent) sendJson(res, { error: 'server error' }, 500); });
582
592
  }
583
593
  if (p === '/api/chat/messages' && req.method === 'GET') {
584
594
  const ch = String(url.searchParams.get('ch') || '');
585
- if (ch.startsWith('task:')) {
586
- const file = channelFileFor(ch.slice(5), channelRoots);
587
- return file ? sendJson(res, { messages: taskMessages(file) }) : sendJson(res, { error: 'no such channel' }, 404);
588
- }
589
- if (ch.startsWith('project:')) {
590
- const messages = projectMessages(channelRoots, ch.slice(8), Date.now() - CHAT_HISTORY_MS);
591
- return messages ? sendJson(res, { messages }) : sendJson(res, { error: 'no such channel' }, 404);
592
- }
593
- if (ch.startsWith('dm:')) {
594
- const part = channelParticipant(ch.slice(3));
595
- if (part) {
596
- const file = channelFileFor(part.key, channelRoots);
597
- return file ? sendJson(res, { messages: participantThread(file, part.role) }) : sendJson(res, { error: 'no such person' }, 404);
598
- }
595
+ return fromFiles(res, 'messages', { ch, historyMs: CHAT_HISTORY_MS }, () => {
599
596
  const dm = hub.conversations.thread(ch.slice(3));
600
597
  return dm ? sendJson(res, dm) : sendJson(res, { error: 'no such person' }, 404);
601
- }
602
- return sendJson(res, { error: 'no such channel' }, 404);
598
+ });
603
599
  }
604
600
  if (p === '/api/deliveries' && req.method === 'GET') return sendJson(res, projectDeliveries(hub));
605
601
  if (p === '/api/legacy/map' && req.method === 'GET') {
@@ -636,9 +632,17 @@ export function serve({
636
632
  const pollStarted = Date.now();
637
633
  let beat = pollStarted;
638
634
  const inbox = routedMessaging(hub.messaging, box.addresses);
635
+ const claimsFor = box.addresses.length > 1 && box.addresses[0] === bound.principal ? box.addresses : [bound.principal];
639
636
  // Re-polls run from a timer, outside readBody's try/catch: a busy database waits for the next poll, anything else answers 503.
640
637
  const poll = () => {
641
638
  if (!connected(req, res)) return undefined;
639
+ let idle = false;
640
+ try { idle = !(body.heartbeat === true && Date.now() - beat >= LISTEN_HEARTBEAT_MS) && !bus.claimable(claimsFor); } catch { /* the claim below says why */ }
641
+ if (idle) {
642
+ if (Date.now() - pollStarted >= timeout) return sendJson(res, { messages: [], at: Date.now() });
643
+ setTimeout(poll, 250).unref();
644
+ return undefined;
645
+ }
642
646
  let messages = [];
643
647
  try {
644
648
  messages = commit(req, res, pollStarted, Infinity, () => {
@@ -924,6 +928,7 @@ export function serve({
924
928
  });
925
929
  server.on('close', () => {
926
930
  feed.close();
931
+ fileReads.close();
927
932
  for (const t of timers) clearInterval(t);
928
933
  manager.stop();
929
934
  releaseLease();
@@ -0,0 +1,64 @@
1
+ import { Worker } from 'node:worker_threads';
2
+
3
+ /** A worker that died is started again after this long, so a crash can't turn into a busy loop. */
4
+ const RESTART_MS = 1000;
5
+
6
+ /**
7
+ * The facade's side of fileReadsWorker.js. `read(op, args)` resolves to the route's [body, status], or null for a
8
+ * `dm:` that is the hub's own thread; with `{ text: true }` the body comes as its JSON text. The worker also runs the transcript feed when given `transcriptsDir`, calling
9
+ * `onTranscripts` once per pass that saw a transcript move. A worker that dies fails the reads it had in flight and is
10
+ * started again; `close()` stops it.
11
+ */
12
+ export function createFileReads({ roots, transcriptsDir = null, onTranscripts = () => {} }) {
13
+ const waiting = new Map();
14
+ let seq = 0;
15
+ let worker = null;
16
+ let closed = false;
17
+ let restart = null;
18
+
19
+ const start = () => {
20
+ const w = new Worker(new URL('./fileReadsWorker.js', import.meta.url), { workerData: { roots, transcriptsDir } });
21
+ w.unref();
22
+ w.on('message', (m) => {
23
+ if (m.type === 'agents') { onTranscripts(); return; }
24
+ const pending = waiting.get(m.id);
25
+ if (!pending) return;
26
+ waiting.delete(m.id);
27
+ if (m.error) pending.reject(new Error(m.error)); else pending.resolve(m.answer);
28
+ });
29
+ const failed = (error) => {
30
+ if (worker !== w) return;
31
+ worker = null;
32
+ for (const pending of waiting.values()) pending.reject(error);
33
+ waiting.clear();
34
+ if (transcriptsDir && !closed && !restart) {
35
+ restart = setTimeout(() => { restart = null; if (!closed && !worker) worker = start(); }, RESTART_MS);
36
+ restart.unref();
37
+ }
38
+ };
39
+ w.on('error', (e) => { console.error(`file reads worker failed: ${e && e.stack ? e.stack : e}`); failed(e); });
40
+ w.on('exit', (code) => failed(new Error(`the file reads worker exited (${code})`)));
41
+ return w;
42
+ };
43
+ if (transcriptsDir) worker = start();
44
+
45
+ return {
46
+ read(op, args, { text = false } = {}) {
47
+ if (closed) return Promise.reject(new Error('the file reads worker is closed'));
48
+ worker ??= start();
49
+ const id = ++seq;
50
+ const answer = new Promise((resolve, reject) => waiting.set(id, { resolve, reject }));
51
+ worker.postMessage({ id, op, args, text });
52
+ return answer;
53
+ },
54
+ close() {
55
+ closed = true;
56
+ clearTimeout(restart);
57
+ const w = worker;
58
+ worker = null;
59
+ for (const pending of waiting.values()) pending.reject(new Error('the file reads worker is closed'));
60
+ waiting.clear();
61
+ if (w) w.terminate();
62
+ },
63
+ };
64
+ }
@@ -0,0 +1,61 @@
1
+ // The facade's file reads, off the runner's event loop: channel and brief files and Cursor transcripts are read and
2
+ // parsed here with the same hub functions the routes called inline, so each answer is the same, byte for byte.
3
+ import { parentPort, workerData } from 'node:worker_threads';
4
+ import { askDetail, channelParticipant, listChannels, participantThread, projectMessages, taskMessages } from '../hub/lib/chat.mjs';
5
+ import { channelFileFor, taskBriefFor } from './channelFiles.js';
6
+ import { watchTranscripts } from './transcriptFeed.js';
7
+
8
+ const { roots, transcriptsDir } = workerData;
9
+ const NOT_FOUND = (error) => [{ error }, 404];
10
+
11
+ /** Each op answers [body, status]; a dm: that isn't a channel participant is the hub's own (null: the caller reads it). */
12
+ const OPS = {
13
+ brief: ({ key }) => {
14
+ const brief = taskBriefFor(key, roots);
15
+ return brief ? [brief, 200] : NOT_FOUND('no such channel');
16
+ },
17
+ ask: ({ key, ref, at, aliases }) => {
18
+ const file = channelFileFor(key, roots);
19
+ if (!file) return NOT_FOUND('no such channel');
20
+ const detail = askDetail(file, { ref, at, aliases });
21
+ return detail ? [detail, 200] : NOT_FOUND('no open ask in this channel');
22
+ },
23
+ channels: ({ sinceMs }) => [listChannels(roots, sinceMs), 200],
24
+ messages: ({ ch, historyMs }) => {
25
+ if (ch.startsWith('task:')) {
26
+ const file = channelFileFor(ch.slice(5), roots);
27
+ return file ? [{ messages: taskMessages(file) }, 200] : NOT_FOUND('no such channel');
28
+ }
29
+ if (ch.startsWith('project:')) {
30
+ const messages = projectMessages(roots, ch.slice(8), Date.now() - historyMs);
31
+ return messages ? [{ messages }, 200] : NOT_FOUND('no such channel');
32
+ }
33
+ if (ch.startsWith('dm:')) {
34
+ const part = channelParticipant(ch.slice(3));
35
+ if (!part) return null;
36
+ const file = channelFileFor(part.key, roots);
37
+ return file ? [{ messages: participantThread(file, part.role) }, 200] : NOT_FOUND('no such person');
38
+ }
39
+ return NOT_FOUND('no such channel');
40
+ },
41
+ };
42
+
43
+ parentPort.on('message', ({ id, op, args, text = false }) => {
44
+ try {
45
+ const answer = OPS[op](args);
46
+ // As JSON text the answer crosses to the main thread as one string, not an object graph it must clone and stringify.
47
+ parentPort.postMessage({ id, answer: text && answer ? [JSON.stringify(answer[0]), answer[1]] : answer });
48
+ } catch (e) {
49
+ parentPort.postMessage({ id, error: String((e && e.stack) || e) });
50
+ }
51
+ });
52
+
53
+ if (transcriptsDir) {
54
+ // One message per pass of the feed, however many transcript files it saw move.
55
+ let queued = false;
56
+ watchTranscripts(transcriptsDir, () => {
57
+ if (queued) return;
58
+ queued = true;
59
+ queueMicrotask(() => { queued = false; parentPort.postMessage({ type: 'agents' }); });
60
+ });
61
+ }
@@ -0,0 +1,88 @@
1
+ // Cursor transcript writes, noticed within about a second: the hub's lib/transcript-feed.mjs with the same passes and
2
+ // the same notices, except that a folder is listed again only when its own mtime moved (an entry was added, removed
3
+ // or renamed). Its fast pass listed every session folder of every active project every second, ~770 readdir calls a
4
+ // second on a busy Mac; here that is a stat per folder, and a readdir only for the folders that changed.
5
+ import fs from 'node:fs';
6
+ import path from 'node:path';
7
+
8
+ const FAST_MS = 1000;
9
+ const FULL_MS = 30e3;
10
+ const ACTIVE_MS = 6 * 3600e3;
11
+
12
+ const safe = (fn, fallback) => { try { return fn(); } catch { return fallback; } };
13
+ const sig = (file) => safe(() => { const s = fs.statSync(file); return { m: s.mtimeMs, z: s.size }; }, null);
14
+
15
+ export function watchTranscripts(projectsDir, onChange, { fastMs = FAST_MS, fullMs = FULL_MS, activeMs = ACTIVE_MS } = {}) {
16
+ const seen = new Map();
17
+ const activeSessions = new Set();
18
+ const activeRoots = new Set();
19
+ /** folder → { m, entries }: its listing as of its mtime `m`. */
20
+ const listings = new Map();
21
+
22
+ const list = (dir) => {
23
+ const m = safe(() => fs.statSync(dir).mtimeMs, null);
24
+ if (m === null) { listings.delete(dir); return []; }
25
+ const hit = listings.get(dir);
26
+ if (hit && hit.m === m) return hit.entries;
27
+ const entries = safe(() => fs.readdirSync(dir, { withFileTypes: true }).map((d) => ({ name: d.name, dir: d.isDirectory() })), []);
28
+ listings.set(dir, { m, entries });
29
+ return entries;
30
+ };
31
+ const dirs = (dir) => list(dir).filter((e) => e.dir).map((e) => e.name);
32
+ const jsonl = (dir) => list(dir).filter((e) => e.name.endsWith('.jsonl')).map((e) => path.join(dir, e.name));
33
+ const sessionFiles = (sessionDir) => [...jsonl(sessionDir), ...jsonl(path.join(sessionDir, 'subagents'))];
34
+
35
+ const check = (file, now, notify) => {
36
+ const s = sig(file);
37
+ if (!s) return false;
38
+ const was = seen.get(file);
39
+ seen.set(file, s);
40
+ if (notify && (!was || was.m !== s.m || was.z !== s.z)) onChange(file);
41
+ return now - s.m < activeMs;
42
+ };
43
+
44
+ const full = (notify) => {
45
+ const now = Date.now();
46
+ activeSessions.clear();
47
+ activeRoots.clear();
48
+ const present = new Set();
49
+ for (const project of dirs(projectsDir)) {
50
+ const root = path.join(projectsDir, project, 'agent-transcripts');
51
+ const sessions = dirs(root);
52
+ if (!sessions.length) continue;
53
+ present.add(root);
54
+ const rootRecent = now - (sig(root)?.m || 0) < activeMs;
55
+ let any = false;
56
+ for (const id of sessions) {
57
+ const dir = path.join(root, id);
58
+ present.add(dir).add(path.join(dir, 'subagents'));
59
+ let live = false;
60
+ for (const f of sessionFiles(dir)) live = check(f, now, notify) || live;
61
+ if (live) { activeSessions.add(dir); any = true; }
62
+ }
63
+ if (any || rootRecent) activeRoots.add(root);
64
+ }
65
+ present.add(projectsDir);
66
+ for (const dir of listings.keys()) if (!present.has(dir)) listings.delete(dir);
67
+ for (const file of seen.keys()) if (!present.has(path.dirname(file))) seen.delete(file);
68
+ };
69
+
70
+ const fast = () => {
71
+ const now = Date.now();
72
+ for (const root of activeRoots) {
73
+ for (const id of dirs(root)) {
74
+ const dir = path.join(root, id);
75
+ if (!activeSessions.has(dir) && sessionFiles(dir).some((f) => !seen.has(f))) activeSessions.add(dir);
76
+ }
77
+ }
78
+ for (const dir of activeSessions) for (const f of sessionFiles(dir)) check(f, now, true);
79
+ };
80
+
81
+ full(false);
82
+ const timers = [setInterval(() => safe(fast), fastMs), setInterval(() => safe(() => full(true)), fullMs)];
83
+ for (const t of timers) t.unref();
84
+ return {
85
+ watching: () => ({ sessions: activeSessions.size, projects: activeRoots.size }),
86
+ close: () => { for (const t of timers) clearInterval(t); },
87
+ };
88
+ }