instar 1.3.409 → 1.3.411
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +13 -3
- package/dist/commands/server.js.map +1 -1
- package/dist/lifeline/MessageQueue.d.ts +22 -2
- package/dist/lifeline/MessageQueue.d.ts.map +1 -1
- package/dist/lifeline/MessageQueue.js +41 -1
- package/dist/lifeline/MessageQueue.js.map +1 -1
- package/dist/lifeline/TelegramLifeline.js +2 -2
- package/dist/lifeline/TelegramLifeline.js.map +1 -1
- package/dist/messaging/MessageProcessingLedger.d.ts +22 -0
- package/dist/messaging/MessageProcessingLedger.d.ts.map +1 -1
- package/dist/messaging/MessageProcessingLedger.js +58 -6
- package/dist/messaging/MessageProcessingLedger.js.map +1 -1
- package/dist/messaging/ingressDedup.d.ts +3 -1
- package/dist/messaging/ingressDedup.d.ts.map +1 -1
- package/dist/messaging/ingressDedup.js +1 -1
- package/dist/messaging/ingressDedup.js.map +1 -1
- package/dist/messaging/stuckMessageRecovery.d.ts +13 -2
- package/dist/messaging/stuckMessageRecovery.d.ts.map +1 -1
- package/dist/messaging/stuckMessageRecovery.js +20 -4
- package/dist/messaging/stuckMessageRecovery.js.map +1 -1
- package/dist/monitoring/SessionWatchdog.d.ts +1 -1
- package/dist/monitoring/SessionWatchdog.d.ts.map +1 -1
- package/dist/monitoring/SessionWatchdog.js +55 -25
- package/dist/monitoring/SessionWatchdog.js.map +1 -1
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +5 -0
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +47 -47
- package/upgrades/1.3.411.md +93 -0
- package/upgrades/side-effects/stuck-recovery-replay-dedupe.md +88 -0
- package/upgrades/side-effects/watchdog-nonblocking-scans.md +81 -0
|
@@ -41,10 +41,30 @@ CREATE TABLE IF NOT EXISTS message_ledger (
|
|
|
41
41
|
reply_idempotency_key TEXT,
|
|
42
42
|
reply_epoch INTEGER,
|
|
43
43
|
input_snapshot TEXT,
|
|
44
|
-
attempts INTEGER NOT NULL DEFAULT 0
|
|
44
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
45
|
+
sender_envelope TEXT
|
|
45
46
|
);
|
|
46
47
|
CREATE INDEX IF NOT EXISTS idx_message_ledger_state ON message_ledger(state);
|
|
48
|
+
CREATE INDEX IF NOT EXISTS idx_message_ledger_topic_committed ON message_ledger(topic, reply_committed_at);
|
|
47
49
|
`;
|
|
50
|
+
/**
|
|
51
|
+
* Apply the schema, then idempotently add columns introduced after a DB was
|
|
52
|
+
* first created. ALTER TABLE ADD COLUMN throws "duplicate column name" on a DB
|
|
53
|
+
* that already has it — caught and ignored. This keeps existing agents' ledgers
|
|
54
|
+
* upgrading in place with no PostUpdateMigrator step (SQLite schema, per the
|
|
55
|
+
* file-header contract).
|
|
56
|
+
*/
|
|
57
|
+
function ensureSchema(db) {
|
|
58
|
+
db.exec(SCHEMA);
|
|
59
|
+
for (const col of ['sender_envelope TEXT']) {
|
|
60
|
+
try {
|
|
61
|
+
db.exec(`ALTER TABLE message_ledger ADD COLUMN ${col}`);
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
/* column already exists — idempotent */
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
48
68
|
export function resolveMessageLedgerPath(stateDir, agentId) {
|
|
49
69
|
const safe = agentId.replace(/[^A-Za-z0-9._-]/g, '_') || 'default';
|
|
50
70
|
return path.join(stateDir, 'state', `message-ledger.${safe}.sqlite`);
|
|
@@ -81,14 +101,14 @@ export class MessageProcessingLedger {
|
|
|
81
101
|
db.pragma('journal_mode = WAL');
|
|
82
102
|
db.pragma('synchronous = NORMAL');
|
|
83
103
|
db.pragma('busy_timeout = 5000');
|
|
84
|
-
db
|
|
104
|
+
ensureSchema(db);
|
|
85
105
|
return new MessageProcessingLedger(db, dbPath);
|
|
86
106
|
}
|
|
87
107
|
/** Open an in-memory ledger (tests). */
|
|
88
108
|
static openMemory() {
|
|
89
109
|
const db = new Database(':memory:');
|
|
90
110
|
db.pragma('busy_timeout = 5000');
|
|
91
|
-
db
|
|
111
|
+
ensureSchema(db);
|
|
92
112
|
return new MessageProcessingLedger(db, ':memory:');
|
|
93
113
|
}
|
|
94
114
|
/**
|
|
@@ -98,10 +118,11 @@ export class MessageProcessingLedger {
|
|
|
98
118
|
*/
|
|
99
119
|
record(dedupeKey, opts) {
|
|
100
120
|
const now = new Date().toISOString();
|
|
121
|
+
const senderJson = opts.sender ? JSON.stringify(opts.sender) : null;
|
|
101
122
|
const info = this.db
|
|
102
|
-
.prepare(`INSERT OR IGNORE INTO message_ledger (dedupe_key, platform, topic, state, received_at, input_snapshot)
|
|
103
|
-
VALUES (?, ?, ?, 'received', ?, ?)`)
|
|
104
|
-
.run(dedupeKey, opts.platform, opts.topic ?? null, now, opts.input ?? null);
|
|
123
|
+
.prepare(`INSERT OR IGNORE INTO message_ledger (dedupe_key, platform, topic, state, received_at, input_snapshot, sender_envelope)
|
|
124
|
+
VALUES (?, ?, ?, 'received', ?, ?, ?)`)
|
|
125
|
+
.run(dedupeKey, opts.platform, opts.topic ?? null, now, opts.input ?? null, senderJson);
|
|
105
126
|
const row = this.get(dedupeKey);
|
|
106
127
|
return { firstSeen: info.changes === 1, state: row.state };
|
|
107
128
|
}
|
|
@@ -183,6 +204,25 @@ export class MessageProcessingLedger {
|
|
|
183
204
|
return !Number.isNaN(startedMs) && nowMs - startedMs > maxProcessingMs;
|
|
184
205
|
});
|
|
185
206
|
}
|
|
207
|
+
/**
|
|
208
|
+
* Reply-evidence check (the no-DUPLICATE-re-run half of stuck recovery): has
|
|
209
|
+
* ANY inbound on this topic been reply_committed at/after `sinceISO`? If so the
|
|
210
|
+
* agent already answered this topic since the stuck entry arrived, so re-running
|
|
211
|
+
* it would re-deliver an already-handled message (the 2026-06-07 every-~10-min
|
|
212
|
+
* "from Unknown" replay loop). Durable across restarts (reads the ledger, not
|
|
213
|
+
* in-memory state). Cheap: indexed on (topic, reply_committed_at).
|
|
214
|
+
*/
|
|
215
|
+
hasReplyCommittedForTopicSince(topic, sinceISO) {
|
|
216
|
+
const row = this.db
|
|
217
|
+
.prepare(`SELECT 1 FROM message_ledger
|
|
218
|
+
WHERE topic = ?
|
|
219
|
+
AND state IN ('reply_committed','cursor_advanced')
|
|
220
|
+
AND reply_committed_at IS NOT NULL
|
|
221
|
+
AND reply_committed_at >= ?
|
|
222
|
+
LIMIT 1`)
|
|
223
|
+
.get(topic, sinceISO);
|
|
224
|
+
return !!row;
|
|
225
|
+
}
|
|
186
226
|
get(dedupeKey) {
|
|
187
227
|
const row = this.db.prepare(`SELECT * FROM message_ledger WHERE dedupe_key = ?`).get(dedupeKey);
|
|
188
228
|
return row ? rowToEntry(row) : null;
|
|
@@ -208,6 +248,18 @@ function rowToEntry(row) {
|
|
|
208
248
|
replyEpoch: row.reply_epoch ?? null,
|
|
209
249
|
inputSnapshot: row.input_snapshot ?? null,
|
|
210
250
|
attempts: row.attempts ?? 0,
|
|
251
|
+
senderEnvelope: parseSender(row.sender_envelope),
|
|
211
252
|
};
|
|
212
253
|
}
|
|
254
|
+
function parseSender(raw) {
|
|
255
|
+
if (typeof raw !== 'string' || !raw)
|
|
256
|
+
return null;
|
|
257
|
+
try {
|
|
258
|
+
const v = JSON.parse(raw);
|
|
259
|
+
return v && typeof v === 'object' ? v : null;
|
|
260
|
+
}
|
|
261
|
+
catch {
|
|
262
|
+
return null;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
213
265
|
//# sourceMappingURL=MessageProcessingLedger.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MessageProcessingLedger.js","sourceRoot":"","sources":["../../src/messaging/MessageProcessingLedger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAEtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"MessageProcessingLedger.js","sourceRoot":"","sources":["../../src/messaging/MessageProcessingLedger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAEtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAgC7B,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;;;CAkBd,CAAC;AAEF;;;;;;GAMG;AACH,SAAS,YAAY,CAAC,EAAwB;IAC5C,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAChB,KAAK,MAAM,GAAG,IAAI,CAAC,sBAAsB,CAAC,EAAE,CAAC;QAC3C,IAAI,CAAC;YACH,EAAE,CAAC,IAAI,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;QAC1D,CAAC;QAAC,MAAM,CAAC;YACP,wCAAwC;QAC1C,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,QAAgB,EAAE,OAAe;IACxE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC;IACnE,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,kBAAkB,IAAI,SAAS,CAAC,CAAC;AACvE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CAAC,SAAiB,EAAE,UAAkB;IAC9E,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,SAAS,KAAK,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACtG,CAAC;AAED,MAAM,OAAO,uBAAuB;IACjB,EAAE,CAAuB;IACjC,IAAI,CAAS;IAEtB,YAAoB,EAAwB,EAAE,MAAc;QAC1D,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC;QACnB,sEAAsE;QACtE,8DAA8D;QAC9D,oBAAoB,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;YAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAe,EAAE,QAAgB;QAC3C,MAAM,MAAM,GAAG,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC3D,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC;YAAC,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;QAChE,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAChC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACjC,YAAY,CAAC,EAAE,CAAC,CAAC;QACjB,OAAO,IAAI,uBAAuB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACjD,CAAC;IAED,wCAAwC;IACxC,MAAM,CAAC,UAAU;QACf,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC;QACpC,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACjC,YAAY,CAAC,EAAE,CAAC,CAAC;QACjB,OAAO,IAAI,uBAAuB,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAiG;QAIzH,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACpE,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;+CACuC,CACxC;aACA,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,UAAU,CAAC,CAAC;QAC1F,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;QACjC,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;IAC7D,CAAC;IAED,iFAAiF;IACjF,SAAS,CAAC,SAAiB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,iBAAiB,IAAI,GAAG,CAAC,KAAK,KAAK,iBAAiB,CAAC,CAAC;IACvF,CAAC;IAED;;;;OAIG;IACH,eAAe,CAAC,SAAiB,EAAE,KAAa;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,CAAC,GAAG;YAAE,OAAO,KAAK,CAAC;QACvB,IAAI,GAAG,CAAC,KAAK,KAAK,iBAAiB,IAAI,GAAG,CAAC,KAAK,KAAK,iBAAiB;YAAE,OAAO,KAAK,CAAC;QACrF,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;8BAEsB,CACvB;aACA,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;OAIG;IACH,WAAW,CAAC,SAAiB,EAAE,mBAA2B,EAAE,KAAa;QACvE,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;qFAE6E,CAC9E;aACA,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IAC1E,CAAC;IAED,yEAAyE;IACzE,aAAa,CAAC,SAAiB;QAC7B,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;4DACoD,CACrD;aACA,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;OAKG;IACH,sBAAsB,CAAC,SAAiB,EAAE,IAA6F;QACrI,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;;;;oFAM4E,CAC7E;aACA,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACvG,CAAC;IAED;;;;OAIG;IACH,YAAY,CAAC,eAAuB,EAAE,QAAgB,IAAI,CAAC,GAAG,EAAE;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,yDAAyD,CAAC;aAClE,GAAG,EAAW,CAAC;QAClB,OAAO,IAAI;aACR,GAAG,CAAC,UAAU,CAAC;aACf,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACZ,IAAI,CAAC,CAAC,CAAC,mBAAmB;gBAAE,OAAO,KAAK,CAAC;YACzC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;YACpD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,GAAG,SAAS,GAAG,eAAe,CAAC;QACzE,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,8BAA8B,CAAC,KAAa,EAAE,QAAgB;QAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CACN;;;;;iBAKS,CACV;aACA,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACxB,OAAO,CAAC,CAAC,GAAG,CAAC;IACf,CAAC;IAED,GAAG,CAAC,SAAiB;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mDAAmD,CAAC,CAAC,GAAG,CAAC,SAAS,CAAQ,CAAC;QACvG,OAAO,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACtC,CAAC;IAED,KAAK;QACH,IAAI,CAAC;YAAC,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;IACtD,CAAC;CACF;AAED,SAAS,UAAU,CAAC,GAAQ;IAC1B,OAAO;QACL,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI;QACxB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,UAAU,EAAE,GAAG,CAAC,WAAW;QAC3B,mBAAmB,EAAE,GAAG,CAAC,qBAAqB,IAAI,IAAI;QACtD,gBAAgB,EAAE,GAAG,CAAC,kBAAkB,IAAI,IAAI;QAChD,gBAAgB,EAAE,GAAG,CAAC,kBAAkB,IAAI,IAAI;QAChD,mBAAmB,EAAE,GAAG,CAAC,qBAAqB,IAAI,IAAI;QACtD,UAAU,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI;QACnC,aAAa,EAAE,GAAG,CAAC,cAAc,IAAI,IAAI;QACzC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,CAAC;QAC3B,cAAc,EAAE,WAAW,CAAC,GAAG,CAAC,eAAe,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACjD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1B,OAAO,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,CAAoB,CAAC,CAAC,CAAC,IAAI,CAAC;IACnE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
* safe on its own — worst case it drops nothing it shouldn't (the gate only ever
|
|
22
22
|
* drops an event the ledger shows was already handled or is actively in flight).
|
|
23
23
|
*/
|
|
24
|
-
import { MessageProcessingLedger } from './MessageProcessingLedger.js';
|
|
24
|
+
import { MessageProcessingLedger, type SenderEnvelope } from './MessageProcessingLedger.js';
|
|
25
25
|
/** Stable provider-level identity for an inbound event (spec §2 contract item 4). */
|
|
26
26
|
export declare function dedupeKeyFor(platform: string, topicId: number | string, eventId: number | string): string;
|
|
27
27
|
export type IngressAction = 'process' | 'drop';
|
|
@@ -35,6 +35,8 @@ export interface DecideIngressOpts {
|
|
|
35
35
|
topic?: string | null;
|
|
36
36
|
/** The raw inbound text/context, stored so a future replay can re-run it. */
|
|
37
37
|
input?: string;
|
|
38
|
+
/** Inbound sender, stored so a stuck re-run replays as the real user (not "Unknown"). */
|
|
39
|
+
sender?: SenderEnvelope | null;
|
|
38
40
|
/** The lease fencing epoch this machine holds (0 if no lease). */
|
|
39
41
|
epoch: number;
|
|
40
42
|
/** A 'processing' entry older than this was abandoned by a fenced holder. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingressDedup.d.ts","sourceRoot":"","sources":["../../src/messaging/ingressDedup.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EACL,uBAAuB,
|
|
1
|
+
{"version":3,"file":"ingressDedup.d.ts","sourceRoot":"","sources":["../../src/messaging/ingressDedup.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EACL,uBAAuB,EAEvB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAEtC,qFAAqF;AACrF,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAEzG;AAED,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,MAAM,CAAC;AAE/C,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,aAAa,CAAC;IACtB,+CAA+C;IAC/C,MAAM,EAAE,YAAY,GAAG,iBAAiB,GAAG,iBAAiB,GAAG,WAAW,CAAC;CAC5E;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,MAAM,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IAC/B,kEAAkE;IAClE,KAAK,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,eAAe,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,uBAAuB,EAC/B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,iBAAiB,GACtB,eAAe,CA8BjB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,uBAAuB,EAC/B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,UAAU,SAAI,GACb,IAAI,CAGN"}
|
|
@@ -34,7 +34,7 @@ export function dedupeKeyFor(platform, topicId, eventId) {
|
|
|
34
34
|
export function decideIngress(ledger, dedupeKey, opts) {
|
|
35
35
|
const now = opts.now ?? Date.now;
|
|
36
36
|
// Idempotent insert — firstSeen tells us if this is a brand-new event.
|
|
37
|
-
ledger.record(dedupeKey, { platform: opts.platform, topic: opts.topic ?? null, input: opts.input });
|
|
37
|
+
ledger.record(dedupeKey, { platform: opts.platform, topic: opts.topic ?? null, input: opts.input, sender: opts.sender ?? null });
|
|
38
38
|
const entry = ledger.get(dedupeKey);
|
|
39
39
|
// Defensive: record() just inserted-or-found it, so entry is non-null.
|
|
40
40
|
if (!entry) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingressDedup.js","sourceRoot":"","sources":["../../src/messaging/ingressDedup.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAEL,0BAA0B,
|
|
1
|
+
{"version":3,"file":"ingressDedup.js","sourceRoot":"","sources":["../../src/messaging/ingressDedup.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAEL,0BAA0B,GAE3B,MAAM,8BAA8B,CAAC;AAEtC,qFAAqF;AACrF,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,OAAwB,EAAE,OAAwB;IAC/F,OAAO,GAAG,QAAQ,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;AAC7C,CAAC;AAwBD;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,MAA+B,EAC/B,SAAiB,EACjB,IAAuB;IAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACjC,uEAAuE;IACvE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC;IAEjI,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACpC,uEAAuE;IACvE,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IACrD,CAAC;IAED,IAAI,KAAK,CAAC,KAAK,KAAK,iBAAiB,IAAI,KAAK,CAAC,KAAK,KAAK,iBAAiB,EAAE,CAAC;QAC3E,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;IACvD,CAAC;IAED,IAAI,KAAK,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;QACjC,oEAAoE;QACpE,MAAM,SAAS,GAAG,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC1F,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,GAAG,EAAE,GAAG,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC;QACnF,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9C,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;QAC1D,CAAC;QACD,iEAAiE;QACjE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACjD,CAAC;IAED,4EAA4E;IAC5E,MAAM,CAAC,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9C,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAA+B,EAC/B,SAAiB,EACjB,KAAa,EACb,UAAU,GAAG,CAAC;IAEd,MAAM,CAAC,WAAW,CAAC,SAAS,EAAE,0BAA0B,CAAC,SAAS,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,CAAC;IACxF,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC"}
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* `attempts < maxReplayAttempts` (a message the agent legitimately never
|
|
18
18
|
* answered is not re-run forever). Lease-gated: a standby never re-injects.
|
|
19
19
|
*/
|
|
20
|
-
import type { MessageProcessingLedger } from './MessageProcessingLedger.js';
|
|
20
|
+
import type { MessageProcessingLedger, SenderEnvelope } from './MessageProcessingLedger.js';
|
|
21
21
|
export interface StuckRecoveryDeps {
|
|
22
22
|
ledger: MessageProcessingLedger;
|
|
23
23
|
/** Only the awake machine recovers; a standby must not inject. */
|
|
@@ -28,14 +28,25 @@ export interface StuckRecoveryDeps {
|
|
|
28
28
|
maxProcessingMs: number;
|
|
29
29
|
/** Give up re-running an entry after this many attempts (avoid storms). Default 3. */
|
|
30
30
|
maxReplayAttempts?: number;
|
|
31
|
+
/**
|
|
32
|
+
* Reply-evidence guard (no-DUPLICATE-re-run half): true if the agent already
|
|
33
|
+
* replied to `topic` at/after `sinceISO`. When true, a stuck entry is treated
|
|
34
|
+
* as already-handled — committed, NOT re-injected. Defaults to the ledger's own
|
|
35
|
+
* `hasReplyCommittedForTopicSince`; injectable for tests. This is what stops the
|
|
36
|
+
* 2026-06-07 "re-run an already-answered message every ~10 min" loop, even when
|
|
37
|
+
* the original reply failed to commit its own entry (server flap / dup orphan).
|
|
38
|
+
*/
|
|
39
|
+
hasRepliedSince?: (topic: string, sinceISO: string) => boolean;
|
|
31
40
|
/** Re-route the stored input as the holder (set current-inbound key, then inject). */
|
|
32
|
-
reinject: (topicId: string, dedupeKey: string, text: string) => void;
|
|
41
|
+
reinject: (topicId: string, dedupeKey: string, text: string, sender: SenderEnvelope | null) => void;
|
|
33
42
|
now?: () => number;
|
|
34
43
|
logger?: (msg: string) => void;
|
|
35
44
|
}
|
|
36
45
|
export interface StuckRecoveryResult {
|
|
37
46
|
recovered: number;
|
|
38
47
|
skipped: number;
|
|
48
|
+
/** Entries recognized as already-answered (reply evidence) and committed, not re-run. */
|
|
49
|
+
alreadyHandled: number;
|
|
39
50
|
}
|
|
40
51
|
/**
|
|
41
52
|
* Re-run inbound events stranded in `processing`. Idempotent-safe to call
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stuckMessageRecovery.d.ts","sourceRoot":"","sources":["../../src/messaging/stuckMessageRecovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"stuckMessageRecovery.d.ts","sourceRoot":"","sources":["../../src/messaging/stuckMessageRecovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAG5F,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,uBAAuB,CAAC;IAChC,kEAAkE;IAClE,UAAU,EAAE,MAAM,OAAO,CAAC;IAC1B,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;IACd,qFAAqF;IACrF,eAAe,EAAE,MAAM,CAAC;IACxB,sFAAsF;IACtF,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;IAC/D,sFAAsF;IACtF,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,GAAG,IAAI,KAAK,IAAI,CAAC;IACpG,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,yFAAyF;IACzF,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,iBAAiB,GAAG,mBAAmB,CA2CjF"}
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
* `attempts < maxReplayAttempts` (a message the agent legitimately never
|
|
18
18
|
* answered is not re-run forever). Lease-gated: a standby never re-injects.
|
|
19
19
|
*/
|
|
20
|
+
import { computeReplyIdempotencyKey } from './MessageProcessingLedger.js';
|
|
20
21
|
/**
|
|
21
22
|
* Re-run inbound events stranded in `processing`. Idempotent-safe to call
|
|
22
23
|
* repeatedly (boot + on a cadence): a re-run re-claims under the current epoch,
|
|
@@ -25,15 +26,29 @@
|
|
|
25
26
|
export function recoverStuckMessages(deps) {
|
|
26
27
|
const maxAttempts = deps.maxReplayAttempts ?? 3;
|
|
27
28
|
if (!deps.holdsLease())
|
|
28
|
-
return { recovered: 0, skipped: 0 };
|
|
29
|
+
return { recovered: 0, skipped: 0, alreadyHandled: 0 };
|
|
30
|
+
const repliedSince = deps.hasRepliedSince ?? ((topic, sinceISO) => deps.ledger.hasReplyCommittedForTopicSince(topic, sinceISO));
|
|
29
31
|
const stuck = deps.ledger.reclaimStuck(deps.maxProcessingMs, deps.now ? deps.now() : undefined);
|
|
30
32
|
let recovered = 0;
|
|
31
33
|
let skipped = 0;
|
|
34
|
+
let alreadyHandled = 0;
|
|
32
35
|
for (const entry of stuck) {
|
|
33
36
|
if (!entry.inputSnapshot || !entry.topic) {
|
|
34
37
|
skipped++;
|
|
35
38
|
continue;
|
|
36
39
|
}
|
|
40
|
+
// Reply-evidence guard: if the agent already answered this topic since the
|
|
41
|
+
// stuck entry arrived, the entry is a duplicate/superseded inbound that was
|
|
42
|
+
// effectively handled (its own reply just failed to commit — server flap, or
|
|
43
|
+
// a same-topic dup orphaned by the current-inbound overwrite). Commit it so
|
|
44
|
+
// it leaves 'processing' for good; do NOT re-inject (that is the replay loop).
|
|
45
|
+
if (repliedSince(entry.topic, entry.receivedAt)) {
|
|
46
|
+
deps.ledger.commitReply(entry.dedupeKey, computeReplyIdempotencyKey(entry.dedupeKey, 0), deps.epoch);
|
|
47
|
+
deps.ledger.advanceCursor(entry.dedupeKey);
|
|
48
|
+
deps.logger?.(`stuck-recovery: ${entry.dedupeKey} already answered on its topic since receipt — committing, not re-running`);
|
|
49
|
+
alreadyHandled++;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
37
52
|
if (entry.attempts >= maxAttempts) {
|
|
38
53
|
// Re-run budget exhausted — leave it (a message that legitimately never
|
|
39
54
|
// got a reply, or a persistently failing turn) rather than loop forever.
|
|
@@ -41,12 +56,13 @@ export function recoverStuckMessages(deps) {
|
|
|
41
56
|
skipped++;
|
|
42
57
|
continue;
|
|
43
58
|
}
|
|
44
|
-
// Re-claim under the current epoch (bumps attempts) and re-route the turn
|
|
59
|
+
// Re-claim under the current epoch (bumps attempts) and re-route the turn,
|
|
60
|
+
// preserving the real sender so the replay is not "from Unknown".
|
|
45
61
|
deps.ledger.beginProcessing(entry.dedupeKey, deps.epoch);
|
|
46
|
-
deps.reinject(entry.topic, entry.dedupeKey, entry.inputSnapshot);
|
|
62
|
+
deps.reinject(entry.topic, entry.dedupeKey, entry.inputSnapshot, entry.senderEnvelope);
|
|
47
63
|
deps.logger?.(`stuck-recovery: re-ran ${entry.dedupeKey} (attempt ${entry.attempts + 1})`);
|
|
48
64
|
recovered++;
|
|
49
65
|
}
|
|
50
|
-
return { recovered, skipped };
|
|
66
|
+
return { recovered, skipped, alreadyHandled };
|
|
51
67
|
}
|
|
52
68
|
//# sourceMappingURL=stuckMessageRecovery.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stuckMessageRecovery.js","sourceRoot":"","sources":["../../src/messaging/stuckMessageRecovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;
|
|
1
|
+
{"version":3,"file":"stuckMessageRecovery.js","sourceRoot":"","sources":["../../src/messaging/stuckMessageRecovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAkC1E;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAuB;IAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAC;IAChD,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;QAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IAE/E,MAAM,YAAY,GAChB,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,8BAA8B,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IAE7G,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAChG,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACzC,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,2EAA2E;QAC3E,4EAA4E;QAC5E,6EAA6E;QAC7E,4EAA4E;QAC5E,+EAA+E;QAC/E,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,EAAE,0BAA0B,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YACrG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,CAAC,MAAM,EAAE,CAAC,mBAAmB,KAAK,CAAC,SAAS,2EAA2E,CAAC,CAAC;YAC7H,cAAc,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,QAAQ,IAAI,WAAW,EAAE,CAAC;YAClC,wEAAwE;YACxE,yEAAyE;YACzE,IAAI,CAAC,MAAM,EAAE,CAAC,gCAAgC,KAAK,CAAC,SAAS,UAAU,KAAK,CAAC,QAAQ,WAAW,CAAC,CAAC;YAClG,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,2EAA2E;QAC3E,kEAAkE;QAClE,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;QACvF,IAAI,CAAC,MAAM,EAAE,CAAC,0BAA0B,KAAK,CAAC,SAAS,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3F,SAAS,EAAE,CAAC;IACd,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC;AAChD,CAAC"}
|
|
@@ -142,7 +142,7 @@ export declare class SessionWatchdog extends EventEmitter {
|
|
|
142
142
|
* 5. server.ts checks message history before activating triage (data-level guard)
|
|
143
143
|
* 6. TriageOrchestrator Pattern 2b re-validates before reinjecting (redundant check)
|
|
144
144
|
*/
|
|
145
|
-
checkCompactionIdle(tmuxSession: string): void
|
|
145
|
+
checkCompactionIdle(tmuxSession: string): Promise<void>;
|
|
146
146
|
private handleEscalation;
|
|
147
147
|
/**
|
|
148
148
|
* LLM gate: Before entering escalation, ask whether the command is
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SessionWatchdog.d.ts","sourceRoot":"","sources":["../../src/monitoring/SessionWatchdog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;
|
|
1
|
+
{"version":3,"file":"SessionWatchdog.d.ts","sourceRoot":"","sources":["../../src/monitoring/SessionWatchdog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAyC3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAQ3E,oBAAY,eAAe;IACzB,UAAU,IAAI;IACd,KAAK,IAAI;IACT,OAAO,IAAI;IACX,OAAO,IAAI;IACX,WAAW,IAAI;CAChB;AAQD,UAAU,eAAe;IACvB,KAAK,EAAE,eAAe,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,eAAe,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,OAAO,CAAC,EAAE,WAAW,GAAG,MAAM,GAAG,SAAS,CAAC;IAC3C,gEAAgE;IAChE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,8CAA8C;AAC9C,MAAM,WAAW,aAAa;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AA4ED,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IACzC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;IAC5D,iBAAiB,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACzC,cAAc,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;CACvC;AAED,qBAAa,eAAgB,SAAQ,YAAY;IAC/C,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,QAAQ,CAA+C;IAC/D,OAAO,CAAC,eAAe,CAAsC;IAC7D,OAAO,CAAC,mBAAmB,CAA2B;IACtD,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,OAAO,CAAS;IAExB,+EAA+E;IAC/E,YAAY,EAAE,oBAAoB,GAAG,IAAI,CAAQ;IAEjD,+EAA+E;IAC/E,OAAO,CAAC,mBAAmB,CAAqB;IAEhD,yEAAyE;IACzE,OAAO,CAAC,gBAAgB,CAAK;IAE7B,sEAAsE;IACtE,OAAO,CAAC,oBAAoB,CAAwC;IAEpE,4EAA4E;IAC5E,OAAO,CAAC,uBAAuB,CAA6B;IAC5D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,2BAA2B,CAAW;IAE9D,yEAAyE;IACzE,OAAO,CAAC,oBAAoB,CAA6B;IACzD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAAU;IAE1D;;;;;OAKG;IACH,OAAO,CAAC,cAAc,CAA0C;IAChE,wGAAwG;IACxG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;gBAE/B,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,KAAK,EAAE,YAAY;IAerF,KAAK,IAAI,IAAI;IAOb,IAAI,IAAI,IAAI;IAOZ,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAOlC,SAAS,IAAI,OAAO;IAIpB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IAKxC,SAAS,IAAI;QACX,OAAO,EAAE,OAAO,CAAC;QACjB,QAAQ,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,eAAe,GAAG,IAAI,CAAA;SAAE,CAAC,CAAC;QACtE,mBAAmB,EAAE,iBAAiB,EAAE,CAAC;KAC1C;YAgBa,IAAI;YAqBJ,YAAY;IAiH1B;;;;;;;;;;;;;;;;;;;OAmBG;IACH,gBAAgB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAyC3C;;;;;;;;;;;;OAYG;IACG,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YA4C/C,gBAAgB;IAwF9B;;;;;;;;OAQG;YACW,cAAc;IAkE5B;;;;;;;;OAQG;YACW,YAAY;YAIZ,eAAe;YAgDf,iBAAiB;IA8B/B;;;;;;;;;;OAUG;YACW,wBAAwB;IAwDtC;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAO9B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAkBvB,OAAO,CAAC,UAAU;IAclB,OAAO,CAAC,YAAY;IAgBpB,OAAO,CAAC,UAAU;IAWlB,OAAO,CAAC,cAAc;IAWtB,OAAO,CAAC,kBAAkB;IA+B1B;;;OAGG;IACH,OAAO,CAAC,YAAY;IAiBpB;;;OAGG;IACH,OAAO,CAAC,YAAY;IAapB;;OAEG;IACH,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,iBAAiB,EAAE;IAgB9C;;;OAGG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,aAAa;IA8BzC;;OAEG;IACH,SAAS,IAAI,IAAI;CAwBlB"}
|
|
@@ -12,15 +12,42 @@
|
|
|
12
12
|
* Level 3: SIGKILL the stuck child PID
|
|
13
13
|
* Level 4: Kill tmux session
|
|
14
14
|
*/
|
|
15
|
-
import {
|
|
15
|
+
import { execFile } from 'node:child_process';
|
|
16
|
+
import { promisify } from 'node:util';
|
|
16
17
|
import { EventEmitter } from 'node:events';
|
|
17
18
|
import fs from 'node:fs';
|
|
18
19
|
import path from 'node:path';
|
|
19
20
|
import { maybeRotateJsonl } from '../utils/jsonl-rotation.js';
|
|
20
21
|
import { evaluateThrottleSettle, RATE_LIMIT_SETTLED_CAPTURE_LINES, RATE_LIMIT_DEFAULT_SETTLE_MS, } from './rateLimitDetection.js';
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
const execFileAsync = promisify(execFile);
|
|
23
|
+
/**
|
|
24
|
+
* ASYNC shell exec — the watchdog poll runs every 30s over EVERY running
|
|
25
|
+
* session, several `ps`/`pgrep` probes each. With the old `spawnSync` those
|
|
26
|
+
* probes blocked the single Node event loop for the full duration of each
|
|
27
|
+
* subprocess; under load (dozens of sessions, a busy box) the cumulative stall
|
|
28
|
+
* made the server miss its own /health window → false "server temporarily down"
|
|
29
|
+
* + a restart loop (2026-06-07 incident). execFile is non-blocking: the event
|
|
30
|
+
* loop (and /health) stays responsive while `ps` runs. Returns captured stdout
|
|
31
|
+
* (or '' on a non-zero exit / timeout), matching the old spawnSync semantics
|
|
32
|
+
* where a pgrep/egrep no-match simply yielded ''.
|
|
33
|
+
*/
|
|
34
|
+
async function shellExecAsync(cmd, timeout = 5000) {
|
|
35
|
+
try {
|
|
36
|
+
const { stdout } = await execFileAsync('/bin/sh', ['-c', cmd], {
|
|
37
|
+
encoding: 'utf-8',
|
|
38
|
+
timeout,
|
|
39
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
40
|
+
});
|
|
41
|
+
return stdout ?? '';
|
|
42
|
+
}
|
|
43
|
+
catch (err) {
|
|
44
|
+
// @silent-fallback-ok — a failed process probe is not a degradation: a
|
|
45
|
+
// pgrep/egrep no-match, a dead PID, or a timeout legitimately means "no
|
|
46
|
+
// match", identical to the prior synchronous behavior where spawnSync simply
|
|
47
|
+
// yielded an empty stdout string. Return any captured stdout, else ''.
|
|
48
|
+
const e = err;
|
|
49
|
+
return e && typeof e.stdout === 'string' ? e.stdout : '';
|
|
50
|
+
}
|
|
24
51
|
}
|
|
25
52
|
import { createRequire } from 'node:module';
|
|
26
53
|
// ESM module: a bare CJS `require` is undefined and throws at runtime. Bind a
|
|
@@ -203,6 +230,9 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
203
230
|
catch (err) {
|
|
204
231
|
console.error(`[Watchdog] Error checking "${session.tmuxSession}":`, err);
|
|
205
232
|
}
|
|
233
|
+
// Yield to the event loop between sessions so a poll over many sessions
|
|
234
|
+
// can never monopolize the loop and starve /health (which shares it).
|
|
235
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
206
236
|
}
|
|
207
237
|
}
|
|
208
238
|
finally {
|
|
@@ -219,12 +249,12 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
219
249
|
// the stuck-command path but still run compaction-idle detection —
|
|
220
250
|
// that path is output-based and doesn't strictly need a PID (its own
|
|
221
251
|
// process guard is null-safe).
|
|
222
|
-
const claudePid = this.getClaudePid(tmuxSession);
|
|
252
|
+
const claudePid = await this.getClaudePid(tmuxSession);
|
|
223
253
|
if (!claudePid) {
|
|
224
|
-
this.checkCompactionIdle(tmuxSession);
|
|
254
|
+
await this.checkCompactionIdle(tmuxSession);
|
|
225
255
|
return;
|
|
226
256
|
}
|
|
227
|
-
const children = this.getChildProcesses(claudePid);
|
|
257
|
+
const children = await this.getChildProcesses(claudePid);
|
|
228
258
|
const stuckChild = children.find(c => {
|
|
229
259
|
if (this.isExcluded(c.command))
|
|
230
260
|
return false;
|
|
@@ -245,7 +275,7 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
245
275
|
// contain active sibling producers, the "stuck" process is just
|
|
246
276
|
// waiting for upstream output from a legitimate long-running
|
|
247
277
|
// pipeline. Skip escalation.
|
|
248
|
-
if (this.hasActivePipelineSibling(stuckChild.pid, stuckChild.command)) {
|
|
278
|
+
if (await this.hasActivePipelineSibling(stuckChild.pid, stuckChild.command)) {
|
|
249
279
|
console.log(`[Watchdog] "${tmuxSession}": ${stuckChild.command.slice(0, 60)} ` +
|
|
250
280
|
`is consuming an active pipeline — skipping escalation`);
|
|
251
281
|
this.temporaryExclusions.add(stuckChild.pid);
|
|
@@ -299,7 +329,7 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
299
329
|
// sitting at a bare prompt with no one at the terminal to nudge them.
|
|
300
330
|
// This is the polling-based fallback for the PreCompact event path which
|
|
301
331
|
// is unreliable (Claude Code doesn't always fire the event).
|
|
302
|
-
this.checkCompactionIdle(tmuxSession);
|
|
332
|
+
await this.checkCompactionIdle(tmuxSession);
|
|
303
333
|
// Server-side throttle detection — catch sessions stopped at a prompt after
|
|
304
334
|
// Anthropic's "temporarily limiting requests" throttle exhausted Claude's
|
|
305
335
|
// own retries. Signal-only: emits 'rate-limited' for the RateLimitSentinel
|
|
@@ -369,7 +399,7 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
369
399
|
* 5. server.ts checks message history before activating triage (data-level guard)
|
|
370
400
|
* 6. TriageOrchestrator Pattern 2b re-validates before reinjecting (redundant check)
|
|
371
401
|
*/
|
|
372
|
-
checkCompactionIdle(tmuxSession) {
|
|
402
|
+
async checkCompactionIdle(tmuxSession) {
|
|
373
403
|
// Cooldown — don't re-emit for the same session within 5 minutes
|
|
374
404
|
const lastEmitted = this.compactionIdleCooldowns.get(tmuxSession);
|
|
375
405
|
if (lastEmitted && (Date.now() - lastEmitted) < SessionWatchdog.COMPACTION_IDLE_COOLDOWN_MS) {
|
|
@@ -377,9 +407,9 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
377
407
|
}
|
|
378
408
|
// Guard 1: Structural process check — if Claude has active child processes,
|
|
379
409
|
// it's executing tools/commands, not stalled. Skip entirely.
|
|
380
|
-
const claudePid = this.getClaudePid(tmuxSession);
|
|
410
|
+
const claudePid = await this.getClaudePid(tmuxSession);
|
|
381
411
|
if (claudePid) {
|
|
382
|
-
const children = this.getChildProcesses(claudePid);
|
|
412
|
+
const children = await this.getChildProcesses(claudePid);
|
|
383
413
|
const activeChildren = children.filter(c => !this.isExcluded(c.command));
|
|
384
414
|
if (activeChildren.length > 0)
|
|
385
415
|
return;
|
|
@@ -545,13 +575,13 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
545
575
|
* @deprecated method name retained for v0.x compat — internal callers
|
|
546
576
|
* use the framework-generic shape.
|
|
547
577
|
*/
|
|
548
|
-
getClaudePid(tmuxSession) {
|
|
578
|
+
async getClaudePid(tmuxSession) {
|
|
549
579
|
return this.getFrameworkPid(tmuxSession);
|
|
550
580
|
}
|
|
551
|
-
getFrameworkPid(tmuxSession) {
|
|
581
|
+
async getFrameworkPid(tmuxSession) {
|
|
552
582
|
try {
|
|
553
583
|
// Get pane PID
|
|
554
|
-
const panePidStr =
|
|
584
|
+
const panePidStr = (await shellExecAsync(`${this.config.sessions.tmuxPath} list-panes -t "=${tmuxSession}" -F "#{pane_pid}" 2>/dev/null`)).trim().split('\n')[0];
|
|
555
585
|
if (!panePidStr)
|
|
556
586
|
return null;
|
|
557
587
|
const panePid = parseInt(panePidStr, 10);
|
|
@@ -562,7 +592,7 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
562
592
|
// own command first — if it matches a known framework binary,
|
|
563
593
|
// return it. Without this, pgrep -P finds no match and the
|
|
564
594
|
// watchdog silently no-ops.
|
|
565
|
-
const paneCmd =
|
|
595
|
+
const paneCmd = (await shellExecAsync(`ps -p ${panePid} -o comm= 2>/dev/null`)).trim();
|
|
566
596
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
567
597
|
const { listProcessSignals } = require('./frameworkProcessSignals.js');
|
|
568
598
|
const signals = listProcessSignals();
|
|
@@ -581,7 +611,7 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
581
611
|
// Fallback: pane runs a shell wrapper that has a framework CLI as
|
|
582
612
|
// a child. egrep alternation over every framework's bracket needle.
|
|
583
613
|
const needle = signals.map((s) => s.psGrepNeedle).join('|');
|
|
584
|
-
const childPidStr =
|
|
614
|
+
const childPidStr = (await shellExecAsync(`pgrep -P ${panePid} 2>/dev/null | xargs -I@ ps -p @ -o pid=,command= 2>/dev/null | egrep -i '${needle}' | head -1 | awk '{print $1}'`)).trim();
|
|
585
615
|
if (!childPidStr)
|
|
586
616
|
return null;
|
|
587
617
|
const pid = parseInt(childPidStr, 10);
|
|
@@ -592,15 +622,15 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
592
622
|
return null;
|
|
593
623
|
}
|
|
594
624
|
}
|
|
595
|
-
getChildProcesses(pid) {
|
|
625
|
+
async getChildProcesses(pid) {
|
|
596
626
|
try {
|
|
597
|
-
const childPidsStr =
|
|
627
|
+
const childPidsStr = (await shellExecAsync(`pgrep -P ${pid} 2>/dev/null`)).trim();
|
|
598
628
|
if (!childPidsStr)
|
|
599
629
|
return [];
|
|
600
630
|
const childPids = childPidsStr.split('\n').filter(Boolean).join(',');
|
|
601
631
|
if (!childPids)
|
|
602
632
|
return [];
|
|
603
|
-
const output =
|
|
633
|
+
const output = (await shellExecAsync(`ps -o pid=,etime=,command= -p ${childPids} 2>/dev/null`)).trim();
|
|
604
634
|
if (!output)
|
|
605
635
|
return [];
|
|
606
636
|
const results = [];
|
|
@@ -635,7 +665,7 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
635
665
|
* Returns true if this PID looks like a waiting consumer in an active
|
|
636
666
|
* pipeline — in which case escalation should be skipped.
|
|
637
667
|
*/
|
|
638
|
-
hasActivePipelineSibling(pid, command) {
|
|
668
|
+
async hasActivePipelineSibling(pid, command) {
|
|
639
669
|
// Step 1: is the command a stdin-consuming candidate at all?
|
|
640
670
|
const consumer = STDIN_CONSUMER_PATTERNS.find(p => p.cmd.test(command));
|
|
641
671
|
if (!consumer)
|
|
@@ -646,11 +676,11 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
646
676
|
return false;
|
|
647
677
|
// Step 3: find the process group and peers.
|
|
648
678
|
try {
|
|
649
|
-
const pgidStr =
|
|
679
|
+
const pgidStr = (await shellExecAsync(`ps -o pgid= -p ${pid} 2>/dev/null`)).trim();
|
|
650
680
|
const pgid = parseInt(pgidStr, 10);
|
|
651
681
|
// Check process group peers (pipelines share pgid on macOS/Linux).
|
|
652
682
|
if (!isNaN(pgid) && pgid > 0) {
|
|
653
|
-
const peersOutput =
|
|
683
|
+
const peersOutput = (await shellExecAsync(`ps -o pid=,command= -g ${pgid} 2>/dev/null`)).trim();
|
|
654
684
|
for (const line of peersOutput.split('\n')) {
|
|
655
685
|
const match = line.trim().match(/^(\d+)\s+(.+)$/);
|
|
656
686
|
if (!match)
|
|
@@ -669,11 +699,11 @@ export class SessionWatchdog extends EventEmitter {
|
|
|
669
699
|
// Fallback: also check direct descendants of the consumer PID. In
|
|
670
700
|
// some shell exec patterns the producer becomes a CHILD of the
|
|
671
701
|
// exec'd consumer rather than a pgid sibling.
|
|
672
|
-
const childPidsStr =
|
|
702
|
+
const childPidsStr = (await shellExecAsync(`pgrep -P ${pid} 2>/dev/null`)).trim();
|
|
673
703
|
if (childPidsStr) {
|
|
674
704
|
const childPids = childPidsStr.split('\n').filter(Boolean).join(',');
|
|
675
705
|
if (childPids) {
|
|
676
|
-
const childOutput =
|
|
706
|
+
const childOutput = (await shellExecAsync(`ps -o pid=,command= -p ${childPids} 2>/dev/null`)).trim();
|
|
677
707
|
for (const line of childOutput.split('\n')) {
|
|
678
708
|
const match = line.trim().match(/^(\d+)\s+(.+)$/);
|
|
679
709
|
if (!match)
|