instar 1.3.395 → 1.3.397
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +9 -1
- package/dist/commands/server.js.map +1 -1
- package/dist/lifeline/MessageQueue.d.ts +25 -1
- package/dist/lifeline/MessageQueue.d.ts.map +1 -1
- package/dist/lifeline/MessageQueue.js +30 -0
- package/dist/lifeline/MessageQueue.js.map +1 -1
- package/dist/lifeline/TelegramLifeline.d.ts +12 -2
- package/dist/lifeline/TelegramLifeline.d.ts.map +1 -1
- package/dist/lifeline/TelegramLifeline.js +83 -70
- package/dist/lifeline/TelegramLifeline.js.map +1 -1
- package/dist/lifeline/replayPolicy.d.ts +66 -0
- package/dist/lifeline/replayPolicy.d.ts.map +1 -0
- package/dist/lifeline/replayPolicy.js +86 -0
- package/dist/lifeline/replayPolicy.js.map +1 -0
- package/dist/monitoring/SessionReaper.d.ts +4 -0
- package/dist/monitoring/SessionReaper.d.ts.map +1 -1
- package/dist/monitoring/SessionReaper.js +8 -1
- package/dist/monitoring/SessionReaper.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +3 -3
- package/upgrades/1.3.396.md +31 -0
- package/upgrades/1.3.397.md +43 -0
- package/upgrades/side-effects/lifeline-replay-budget-classification.md +97 -0
- package/upgrades/side-effects/reaper-transcript-projectdir.md +46 -0
|
@@ -16,8 +16,14 @@ export interface QueuedMessage {
|
|
|
16
16
|
photoPath?: string;
|
|
17
17
|
documentPath?: string;
|
|
18
18
|
documentName?: string;
|
|
19
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* Strikes from genuine HTTP-400 rejections (message-specific / "poison").
|
|
21
|
+
* Named `replayFailures` for on-disk back-compat with queues written before
|
|
22
|
+
* the transient/poison split (2026-06-06); semantically the poison counter.
|
|
23
|
+
*/
|
|
20
24
|
replayFailures?: number;
|
|
25
|
+
/** Strikes from transient capacity/availability failures (timeout/5xx/down). */
|
|
26
|
+
transientReplayFailures?: number;
|
|
21
27
|
}
|
|
22
28
|
export declare class MessageQueue {
|
|
23
29
|
private queuePath;
|
|
@@ -35,6 +41,24 @@ export declare class MessageQueue {
|
|
|
35
41
|
* Peek at the queue without draining.
|
|
36
42
|
*/
|
|
37
43
|
peek(): QueuedMessage[];
|
|
44
|
+
/**
|
|
45
|
+
* Remove a single message by id and persist. Used by durable replay: a
|
|
46
|
+
* message is removed from the persisted queue ONLY after it has been
|
|
47
|
+
* delivered or deliberately dropped — so a process exit mid-replay can never
|
|
48
|
+
* lose an undelivered message (the 2026-06-06 topic-21487 untracked-loss bug,
|
|
49
|
+
* where drain() emptied the disk queue before delivery confirmed).
|
|
50
|
+
* Returns true if a message was removed.
|
|
51
|
+
*/
|
|
52
|
+
remove(id: string): boolean;
|
|
53
|
+
/**
|
|
54
|
+
* Patch the replay-strike counters of a queued message in place and persist.
|
|
55
|
+
* Leaves the message ON DISK (durable) — used when a forward fails and the
|
|
56
|
+
* message must be retried on the next replay tick. No-op if the id is gone.
|
|
57
|
+
*/
|
|
58
|
+
updateReplayCounters(id: string, counters: {
|
|
59
|
+
replayFailures: number;
|
|
60
|
+
transientReplayFailures: number;
|
|
61
|
+
}): void;
|
|
38
62
|
get length(): number;
|
|
39
63
|
private load;
|
|
40
64
|
private save;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MessageQueue.d.ts","sourceRoot":"","sources":["../../src/lifeline/MessageQueue.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB
|
|
1
|
+
{"version":3,"file":"MessageQueue.d.ts","sourceRoot":"","sources":["../../src/lifeline/MessageQueue.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gFAAgF;IAChF,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAuB;gBAExB,QAAQ,EAAE,MAAM;IAK5B;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,aAAa,GAAG,IAAI;IAKjC;;OAEG;IACH,KAAK,IAAI,aAAa,EAAE;IAOxB;;OAEG;IACH,IAAI,IAAI,aAAa,EAAE;IAIvB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAU3B;;;;OAIG;IACH,oBAAoB,CAClB,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,uBAAuB,EAAE,MAAM,CAAA;KAAE,GACpE,IAAI;IAQP,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,OAAO,CAAC,IAAI;IAWZ,OAAO,CAAC,IAAI;CASb"}
|
|
@@ -35,6 +35,36 @@ export class MessageQueue {
|
|
|
35
35
|
peek() {
|
|
36
36
|
return [...this.queue];
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* Remove a single message by id and persist. Used by durable replay: a
|
|
40
|
+
* message is removed from the persisted queue ONLY after it has been
|
|
41
|
+
* delivered or deliberately dropped — so a process exit mid-replay can never
|
|
42
|
+
* lose an undelivered message (the 2026-06-06 topic-21487 untracked-loss bug,
|
|
43
|
+
* where drain() emptied the disk queue before delivery confirmed).
|
|
44
|
+
* Returns true if a message was removed.
|
|
45
|
+
*/
|
|
46
|
+
remove(id) {
|
|
47
|
+
const before = this.queue.length;
|
|
48
|
+
this.queue = this.queue.filter(m => m.id !== id);
|
|
49
|
+
if (this.queue.length !== before) {
|
|
50
|
+
this.save();
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Patch the replay-strike counters of a queued message in place and persist.
|
|
57
|
+
* Leaves the message ON DISK (durable) — used when a forward fails and the
|
|
58
|
+
* message must be retried on the next replay tick. No-op if the id is gone.
|
|
59
|
+
*/
|
|
60
|
+
updateReplayCounters(id, counters) {
|
|
61
|
+
const msg = this.queue.find(m => m.id === id);
|
|
62
|
+
if (!msg)
|
|
63
|
+
return;
|
|
64
|
+
msg.replayFailures = counters.replayFailures;
|
|
65
|
+
msg.transientReplayFailures = counters.transientReplayFailures;
|
|
66
|
+
this.save();
|
|
67
|
+
}
|
|
38
68
|
get length() {
|
|
39
69
|
return this.queue.length;
|
|
40
70
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MessageQueue.js","sourceRoot":"","sources":["../../src/lifeline/MessageQueue.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"MessageQueue.js","sourceRoot":"","sources":["../../src/lifeline/MessageQueue.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAwB7B,MAAM,OAAO,YAAY;IACf,SAAS,CAAS;IAClB,KAAK,GAAoB,EAAE,CAAC;IAEpC,YAAY,QAAgB;QAC1B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,qBAAqB,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,GAAkB;QACxB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAI;QACF,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,EAAU;QACf,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QACjC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YACjC,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAClB,EAAU,EACV,QAAqE;QAErE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,GAAG,CAAC,cAAc,GAAG,QAAQ,CAAC,cAAc,CAAC;QAC7C,GAAG,CAAC,uBAAuB,GAAG,QAAQ,CAAC,uBAAuB,CAAC;QAC/D,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAEO,IAAI;QACV,IAAI,CAAC;YACH,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;gBAClE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IAEO,IAAI;QACV,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;YACvD,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/D,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,kCAAkC,GAAG,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;CACF"}
|
|
@@ -172,7 +172,19 @@ export declare class TelegramLifeline {
|
|
|
172
172
|
* (awake) this is a single `existsSync` no-op per forward.
|
|
173
173
|
*/
|
|
174
174
|
private requestWakeIfSlept;
|
|
175
|
+
/**
|
|
176
|
+
* Boolean façade kept for the inbound-handler callers that only care whether
|
|
177
|
+
* the forward landed. Replay uses {@link forwardToServerClassified} so it can
|
|
178
|
+
* tell a message-specific rejection (poison) from a transient outage.
|
|
179
|
+
*/
|
|
175
180
|
private forwardToServer;
|
|
181
|
+
/**
|
|
182
|
+
* Forward a message to the server and classify the result so replay can
|
|
183
|
+
* budget correctly: only a genuine HTTP-400 rejection ('poison') burns the
|
|
184
|
+
* drop budget; timeout / 5xx / 503-boot / network refusal are 'transient'
|
|
185
|
+
* (never drop a real message), and 426 is 'skew' (coordinated restart).
|
|
186
|
+
*/
|
|
187
|
+
private forwardToServerClassified;
|
|
176
188
|
/**
|
|
177
189
|
* Resolve the destination topic for update-class alerts (version-skew,
|
|
178
190
|
* delivery-health). Reads `agent-updates-topic` from server state. If
|
|
@@ -232,8 +244,6 @@ export declare class TelegramLifeline {
|
|
|
232
244
|
private driftPromoter;
|
|
233
245
|
private watchdog;
|
|
234
246
|
private handleLifelineCommand;
|
|
235
|
-
/** Max times a message can fail replay before being dropped. */
|
|
236
|
-
private static readonly MAX_REPLAY_FAILURES;
|
|
237
247
|
private replayQueue;
|
|
238
248
|
/** Whether we've already notified for the current outage. Reset on recovery. */
|
|
239
249
|
private hasNotifiedServerDown;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TelegramLifeline.d.ts","sourceRoot":"","sources":["../../src/lifeline/TelegramLifeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;
|
|
1
|
+
{"version":3,"file":"TelegramLifeline.d.ts","sourceRoot":"","sources":["../../src/lifeline/TelegramLifeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AA6NH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,aAAa,CAAgC;IACrD,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,UAAU,CAAmB;IACrC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,WAAW,CAA8C;IACjE,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAA6B;IAClD,OAAO,CAAC,cAAc,CAA+C;IACrE,OAAO,CAAC,qBAAqB,CAA+C;IAC5E,OAAO,CAAC,uBAAuB,CAA+C;IAC9E,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,aAAa,CAAQ;IAG7B,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,kBAAkB,CAAuB;IACjD,OAAO,CAAC,oBAAoB,CAA8C;gBAE9D,UAAU,CAAC,EAAE,MAAM;IAwF/B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAqO5B;;;OAGG;YACW,iBAAiB;IAgB/B;;;;;;OAMG;IACH,OAAO,CAAC,8BAA8B;IA6FtC;;;;OAIG;IACH,OAAO,CAAC,uBAAuB;IAc/B;;;;;;;;OAQG;IACH,OAAO,CAAC,oBAAoB;IAU5B;;;;OAIG;IACH,OAAO,CAAC,8BAA8B;IAkBtC;;;;OAIG;IACH,OAAO,CAAC,gCAAgC;IAoBxC;;;;OAIG;IACH,OAAO,CAAC,iCAAiC;IAYzC,mEAAmE;IACnE,OAAO,CAAC,yBAAyB;IAOjC,qDAAqD;IACrD,OAAO,CAAC,sBAAsB;IA6B9B,0EAA0E;YAC5D,oBAAoB;IAOlC;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAwCvB;;;;;;;OAOG;YACW,oBAAoB;YAkDpB,IAAI;YA+EJ,aAAa;IAgG3B;;OAEG;YACW,kBAAkB;IAoDhC;;OAEG;YACW,aAAa;IAwB3B;;;OAGG;YACW,gBAAgB;IA2B9B;;OAEG;YACW,qBAAqB;IAmDnC;;;OAGG;YACW,oBAAoB;IAmDlC;;;;;;;;OAQG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAK;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAQ;IAEvD;;;;OAIG;IACH,OAAO,CAAC,kBAAkB,CAAS;IAEnC,+DAA+D;IAC/D,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsB;IAEtD;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;IAM1B;;;;OAIG;YACW,eAAe;IAQ7B;;;;;OAKG;YACW,yBAAyB;IAoIvC;;;;;;;;;;OAUG;IACH,OAAO,CAAC,wBAAwB;IAuBhC;;;;;;;;;OASG;IACH,OAAO,CAAC,iBAAiB;IAuDzB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,0BAA0B;IA8BlC,uCAAuC;IACvC,OAAO,CAAC,0BAA0B,CAAK;IACvC,OAAO,CAAC,oBAAoB,CAAK;IACjC,OAAO,CAAC,oBAAoB,CAAuB;IAEnD;;;;;;;;;OASG;IACH,OAAO,CAAC,iBAAiB,CAAS;IAClC;;+CAE2C;IAC3C,OAAO,CAAC,sBAAsB,CAAK;IAEnC,OAAO,CAAC,YAAY,CAAoC;IACxD,OAAO,CAAC,aAAa,CAAsC;IAC3D,OAAO,CAAC,QAAQ,CAAuC;YAIzC,qBAAqB;YAsGrB,WAAW;IAoHzB,gFAAgF;IAChF,OAAO,CAAC,qBAAqB,CAAS;IACtC,4DAA4D;IAC5D,OAAO,CAAC,yBAAyB,CAAK;IACtC,0FAA0F;IAC1F,OAAO,CAAC,sBAAsB,CAAK;IACnC,mGAAmG;IACnG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAe;IAE9D,4EAA4E;IAC5E,OAAO,CAAC,cAAc,CAA6B;IACnD,0FAA0F;IAC1F,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAc;IAC7D,6EAA6E;IAC7E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAAO;IAE3D;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAc1B,OAAO,CAAC,kBAAkB;IAkB1B;;;OAGG;IACH,OAAO,CAAC,kBAAkB;YAgBZ,gBAAgB;YAmChB,mBAAmB;IA4BjC;;;;OAIG;YACW,qBAAqB;IAenC;;OAEG;YACW,mBAAmB;IA+CjC;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAiC1B;;OAEG;YACW,sBAAsB;IAsFpC;;;;;;;;;;;;OAYG;YACW,kBAAkB;IAoGhC;;OAEG;IACH,OAAO,CAAC,cAAc;IA4BtB;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAWjC;;OAEG;YACW,oBAAoB;IASlC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAcxB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAkBzB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAa5B;;;;;;;;;OASG;IACH;;;;;;OAMG;IACH,OAAO,CAAC,oBAAoB;YA0Cd,aAAa;IAmG3B;;OAEG;YACW,mBAAmB;IAuDjC;;OAEG;IACH,OAAO,CAAC,sBAAsB;YAyBhB,WAAW;YAqBX,UAAU;IASxB;;;;;;OAMG;IACH,OAAO,CAAC,iBAAiB;YAMX,OAAO;IAmErB,OAAO,CAAC,UAAU;IAiBlB,OAAO,CAAC,UAAU;CAWnB"}
|
|
@@ -47,6 +47,7 @@ import { SessionRecoveryConsumer } from '../core/SessionRecoveryConsumer.js';
|
|
|
47
47
|
import { retryWithBackoff } from './retryWithBackoff.js';
|
|
48
48
|
import { notifyMessageDropped } from './droppedMessages.js';
|
|
49
49
|
import { ForwardTransientError, ForwardBadRequestError, ForwardServerBootError, ForwardVersionSkewError, isTerminalForwardError, } from './forwardErrors.js';
|
|
50
|
+
import { decideReplay } from './replayPolicy.js';
|
|
50
51
|
import { writeStartupMarker } from './startupMarker.js';
|
|
51
52
|
import { shouldOwnTelegramPoll } from './telegramPollOwnership.js';
|
|
52
53
|
import { writeLease as writePollOwnerLease } from './TelegramPollOwnerLease.js';
|
|
@@ -1273,7 +1274,21 @@ export class TelegramLifeline {
|
|
|
1273
1274
|
console.log('[Lifeline] Server is asleep — wrote wake-request; message will replay once it boots');
|
|
1274
1275
|
}
|
|
1275
1276
|
}
|
|
1277
|
+
/**
|
|
1278
|
+
* Boolean façade kept for the inbound-handler callers that only care whether
|
|
1279
|
+
* the forward landed. Replay uses {@link forwardToServerClassified} so it can
|
|
1280
|
+
* tell a message-specific rejection (poison) from a transient outage.
|
|
1281
|
+
*/
|
|
1276
1282
|
async forwardToServer(topicId, text, rawMsg) {
|
|
1283
|
+
return (await this.forwardToServerClassified(topicId, text, rawMsg)) === 'ok';
|
|
1284
|
+
}
|
|
1285
|
+
/**
|
|
1286
|
+
* Forward a message to the server and classify the result so replay can
|
|
1287
|
+
* budget correctly: only a genuine HTTP-400 rejection ('poison') burns the
|
|
1288
|
+
* drop budget; timeout / 5xx / 503-boot / network refusal are 'transient'
|
|
1289
|
+
* (never drop a real message), and 426 is 'skew' (coordinated restart).
|
|
1290
|
+
*/
|
|
1291
|
+
async forwardToServerClassified(topicId, text, rawMsg) {
|
|
1277
1292
|
// a2a spoof-defense fields (MENTOR-LIVE-READINESS-SPEC §Recipient side). The
|
|
1278
1293
|
// server's /internal/telegram-forward handler passes these to
|
|
1279
1294
|
// `TelegramAdapter.dispatchAgentMessageHook` so the a2a hook can distinguish
|
|
@@ -1371,16 +1386,23 @@ export class TelegramLifeline {
|
|
|
1371
1386
|
// gets its own user-visible notification.
|
|
1372
1387
|
this.versionSkewActive = false;
|
|
1373
1388
|
this.versionSkewAlertSentAt = 0;
|
|
1374
|
-
return
|
|
1389
|
+
return 'ok';
|
|
1375
1390
|
}
|
|
1376
1391
|
catch (err) {
|
|
1377
1392
|
// Version-skew handler: emit signal + request restart via orchestrator.
|
|
1378
1393
|
if (err instanceof ForwardVersionSkewError) {
|
|
1379
1394
|
this.handleVersionSkew(err, topicId);
|
|
1380
|
-
return
|
|
1395
|
+
return 'skew';
|
|
1381
1396
|
}
|
|
1382
1397
|
this.consecutiveForwardFailures++;
|
|
1383
|
-
|
|
1398
|
+
// A genuine HTTP-400 is the ONLY message-specific ("poison") failure —
|
|
1399
|
+
// the server looked at this message and refused it. Everything else
|
|
1400
|
+
// (transient 5xx, 503 boot, fetch timeout/AbortError, network refusal)
|
|
1401
|
+
// is a capacity/availability failure that says nothing about the message
|
|
1402
|
+
// and must NOT burn the replay drop budget.
|
|
1403
|
+
if (err instanceof ForwardBadRequestError)
|
|
1404
|
+
return 'poison';
|
|
1405
|
+
return 'transient';
|
|
1384
1406
|
}
|
|
1385
1407
|
}
|
|
1386
1408
|
/**
|
|
@@ -1628,35 +1650,52 @@ export class TelegramLifeline {
|
|
|
1628
1650
|
await this.sendToTopic(topicId, 'Unknown lifeline command. Try /lifeline help');
|
|
1629
1651
|
}
|
|
1630
1652
|
// ── Queue Replay ──────────────────────────────────────────
|
|
1631
|
-
/** Max times a message can fail replay before being dropped. */
|
|
1632
|
-
static MAX_REPLAY_FAILURES = 3;
|
|
1633
1653
|
async replayQueue() {
|
|
1634
|
-
|
|
1654
|
+
// Durable consume: work from a SNAPSHOT and remove a message from the
|
|
1655
|
+
// PERSISTED queue only after it is delivered or deliberately dropped. The
|
|
1656
|
+
// old drain() emptied the on-disk queue up front, so a process exit
|
|
1657
|
+
// mid-replay (update / version-skew / launchd restart — all common during
|
|
1658
|
+
// the very episodes that trigger queuing) lost the in-memory messages with
|
|
1659
|
+
// no trace. That is the 2026-06-06 topic-21487 untracked-loss: a real user
|
|
1660
|
+
// question vanished without even a dropped-messages.json record.
|
|
1661
|
+
const messages = this.queue.peek();
|
|
1635
1662
|
if (messages.length === 0)
|
|
1636
1663
|
return;
|
|
1637
1664
|
console.log(`[Lifeline] Replaying ${messages.length} queued messages`);
|
|
1638
1665
|
let replayed = 0;
|
|
1639
1666
|
let failed = 0;
|
|
1640
1667
|
let dropped = 0;
|
|
1668
|
+
const deliveredByTopic = new Map();
|
|
1641
1669
|
for (const msg of messages) {
|
|
1642
|
-
//
|
|
1643
|
-
|
|
1644
|
-
//
|
|
1645
|
-
//
|
|
1646
|
-
//
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1670
|
+
// CRITICAL: never drop while a version-skew episode is in flight. A hard
|
|
1671
|
+
// incompatibility (HTTP 426) makes EVERY forward fail for a reason
|
|
1672
|
+
// unrelated to the message; the coordinated restart resolves it. Stop
|
|
1673
|
+
// replaying and leave every remaining message safely on disk (the
|
|
1674
|
+
// 2026-05-20 b2lead-insights silent-drop failure mode).
|
|
1675
|
+
if (this.versionSkewActive)
|
|
1676
|
+
break;
|
|
1677
|
+
const outcome = await this.forwardToServerClassified(msg.topicId, msg.text, {
|
|
1678
|
+
message_id: parseInt(msg.id.replace('tg-', ''), 10) || 0,
|
|
1679
|
+
from: {
|
|
1680
|
+
id: msg.fromUserId,
|
|
1681
|
+
first_name: msg.fromFirstName,
|
|
1682
|
+
username: msg.fromUsername,
|
|
1683
|
+
},
|
|
1684
|
+
chat: { id: parseInt(this.config.chatId, 10) },
|
|
1685
|
+
message_thread_id: msg.topicId,
|
|
1686
|
+
text: msg.text,
|
|
1687
|
+
date: Math.floor(new Date(msg.timestamp).getTime() / 1000),
|
|
1688
|
+
});
|
|
1689
|
+
const decision = decideReplay(outcome, {
|
|
1690
|
+
poisonFailures: msg.replayFailures ?? 0,
|
|
1691
|
+
transientFailures: msg.transientReplayFailures ?? 0,
|
|
1692
|
+
});
|
|
1693
|
+
if (decision.action === 'delivered') {
|
|
1694
|
+
this.queue.remove(msg.id);
|
|
1695
|
+
replayed++;
|
|
1696
|
+
deliveredByTopic.set(msg.topicId, (deliveredByTopic.get(msg.topicId) ?? 0) + 1);
|
|
1658
1697
|
}
|
|
1659
|
-
if (
|
|
1698
|
+
else if (decision.action === 'drop') {
|
|
1660
1699
|
dropped++;
|
|
1661
1700
|
// Before the drop becomes silent: persist the record, report a
|
|
1662
1701
|
// degradation, and tell the original sender their message was lost.
|
|
@@ -1667,8 +1706,8 @@ export class TelegramLifeline {
|
|
|
1667
1706
|
messageId: msg.id,
|
|
1668
1707
|
senderName: msg.fromFirstName ?? msg.fromUsername ?? String(msg.fromUserId),
|
|
1669
1708
|
text: msg.text,
|
|
1670
|
-
retryCount:
|
|
1671
|
-
reason:
|
|
1709
|
+
retryCount: decision.poisonFailures + decision.transientFailures,
|
|
1710
|
+
reason: decision.dropReason ?? 'Message could not be delivered',
|
|
1672
1711
|
sendToTopic: (topicId, body) => this.sendToTopic(topicId, body),
|
|
1673
1712
|
});
|
|
1674
1713
|
}
|
|
@@ -1678,50 +1717,27 @@ export class TelegramLifeline {
|
|
|
1678
1717
|
// the queue doesn't stall.
|
|
1679
1718
|
console.error(`[Lifeline] notifyMessageDropped threw for ${msg.id}:`, err instanceof Error ? err.message : err);
|
|
1680
1719
|
}
|
|
1681
|
-
console.warn(`[Lifeline] Dropping message ${msg.id}
|
|
1682
|
-
|
|
1683
|
-
}
|
|
1684
|
-
const forwarded = await this.forwardToServer(msg.topicId, msg.text, {
|
|
1685
|
-
message_id: parseInt(msg.id.replace('tg-', ''), 10) || 0,
|
|
1686
|
-
from: {
|
|
1687
|
-
id: msg.fromUserId,
|
|
1688
|
-
first_name: msg.fromFirstName,
|
|
1689
|
-
username: msg.fromUsername,
|
|
1690
|
-
},
|
|
1691
|
-
chat: { id: parseInt(this.config.chatId, 10) },
|
|
1692
|
-
message_thread_id: msg.topicId,
|
|
1693
|
-
text: msg.text,
|
|
1694
|
-
date: Math.floor(new Date(msg.timestamp).getTime() / 1000),
|
|
1695
|
-
});
|
|
1696
|
-
if (forwarded) {
|
|
1697
|
-
replayed++;
|
|
1720
|
+
console.warn(`[Lifeline] Dropping message ${msg.id}: ${decision.dropReason} — ${msg.text.slice(0, 80)}`);
|
|
1721
|
+
this.queue.remove(msg.id);
|
|
1698
1722
|
}
|
|
1699
1723
|
else {
|
|
1700
|
-
//
|
|
1701
|
-
//
|
|
1702
|
-
//
|
|
1703
|
-
//
|
|
1704
|
-
//
|
|
1705
|
-
//
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
this.queue.enqueue(msg);
|
|
1724
|
+
// requeue — persist the updated strike counters IN PLACE; the message
|
|
1725
|
+
// stays on disk for the next replay tick. A transient failure NEVER
|
|
1726
|
+
// burns the poison budget, so a slow / overloaded / restarting server
|
|
1727
|
+
// can no longer drop a real message (the 2026-06-06 topic-21487
|
|
1728
|
+
// false-drop bug, where a CPU-starved-but-up server burned all 3
|
|
1729
|
+
// attempts in ~90s and dropped the user's question).
|
|
1730
|
+
this.queue.updateReplayCounters(msg.id, {
|
|
1731
|
+
replayFailures: decision.poisonFailures,
|
|
1732
|
+
transientReplayFailures: decision.transientFailures,
|
|
1733
|
+
});
|
|
1711
1734
|
failed++;
|
|
1712
|
-
// If the server
|
|
1713
|
-
//
|
|
1714
|
-
|
|
1735
|
+
// If the server is unavailable, stop replaying — the remaining messages
|
|
1736
|
+
// are already safely persisted (we only remove on delivery/drop) and
|
|
1737
|
+
// will be retried on the next recovery.
|
|
1738
|
+
if (outcome === 'transient' && !this.supervisor.healthy) {
|
|
1715
1739
|
const remaining = messages.length - replayed - failed - dropped;
|
|
1716
|
-
|
|
1717
|
-
console.log(`[Lifeline] Server went down during replay — re-queuing ${remaining} remaining messages`);
|
|
1718
|
-
// Re-queue remaining unprocessed messages (preserve their failure counts)
|
|
1719
|
-
const currentIndex = messages.indexOf(msg);
|
|
1720
|
-
for (let i = currentIndex + 1; i < messages.length; i++) {
|
|
1721
|
-
this.queue.enqueue(messages[i]);
|
|
1722
|
-
}
|
|
1723
|
-
failed += remaining;
|
|
1724
|
-
}
|
|
1740
|
+
console.log(`[Lifeline] Server unavailable during replay — ${remaining} message(s) remain queued`);
|
|
1725
1741
|
break;
|
|
1726
1742
|
}
|
|
1727
1743
|
}
|
|
@@ -1731,13 +1747,10 @@ export class TelegramLifeline {
|
|
|
1731
1747
|
if (replayed > 0 || failed > 0 || dropped > 0) {
|
|
1732
1748
|
console.log(`[Lifeline] Replay complete: ${replayed} delivered, ${failed} re-queued, ${dropped} dropped`);
|
|
1733
1749
|
}
|
|
1734
|
-
// Notify the user that their queued messages were delivered
|
|
1750
|
+
// Notify the user that their queued messages were delivered.
|
|
1735
1751
|
if (replayed > 0) {
|
|
1736
|
-
|
|
1737
|
-
const replayedTopics = new Set(messages.filter((_, i) => i < replayed + failed + dropped).map(m => m.topicId));
|
|
1738
|
-
for (const topicId of replayedTopics) {
|
|
1752
|
+
for (const [topicId, count] of deliveredByTopic) {
|
|
1739
1753
|
try {
|
|
1740
|
-
const count = messages.filter(m => m.topicId === topicId).length;
|
|
1741
1754
|
await this.sendToTopic(topicId, count === 1
|
|
1742
1755
|
? '✓ Server recovered — your queued message has been delivered.'
|
|
1743
1756
|
: `✓ Server recovered — ${count} queued messages delivered.`);
|