instar 1.3.331 → 1.3.332
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +12 -0
- package/dist/commands/server.js.map +1 -1
- package/dist/core/LiveTailSource.d.ts +19 -0
- package/dist/core/LiveTailSource.d.ts.map +1 -1
- package/dist/core/LiveTailSource.js +21 -0
- package/dist/core/LiveTailSource.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +2 -2
- package/upgrades/1.3.332.md +42 -0
- package/upgrades/side-effects/livetail-stale-signal.md +43 -0
|
@@ -74,6 +74,21 @@ export interface LiveTailSourceDeps {
|
|
|
74
74
|
failureBackoffBaseMs?: number;
|
|
75
75
|
/** Backoff ceiling. Default 5 min. */
|
|
76
76
|
failureBackoffMaxMs?: number;
|
|
77
|
+
/**
|
|
78
|
+
* Eternal Sentinel condition 4 ("No Unbounded Loops" / P19): the backoff
|
|
79
|
+
* keeps retrying a failing topic forever (correct — the standby copy should
|
|
80
|
+
* converge whenever the peer recovers), but a topic whose flushes have been
|
|
81
|
+
* failing past staleSignalAfterMs must SAY SO once per episode instead of
|
|
82
|
+
* going quietly stale. Wired to DegradationReporter in server.ts; omitted →
|
|
83
|
+
* silent (tests / channels without a reporter).
|
|
84
|
+
*/
|
|
85
|
+
reportStaleStandby?: (info: {
|
|
86
|
+
topic: string;
|
|
87
|
+
failingForMs: number;
|
|
88
|
+
consecutiveFailures: number;
|
|
89
|
+
}) => void;
|
|
90
|
+
/** Sustained-failure threshold for the one-per-episode stale signal. Default 30 min. */
|
|
91
|
+
staleSignalAfterMs?: number;
|
|
77
92
|
now?: () => number;
|
|
78
93
|
logger?: (msg: string) => void;
|
|
79
94
|
}
|
|
@@ -104,6 +119,10 @@ export declare class LiveTailSource {
|
|
|
104
119
|
private failures;
|
|
105
120
|
/** Per-topic earliest next attempt (ms epoch) while backing off. */
|
|
106
121
|
private nextAttemptAt;
|
|
122
|
+
/** Per-topic failure-episode start (ms epoch; absent = healthy). */
|
|
123
|
+
private failingSince;
|
|
124
|
+
/** Episode-keyed one-shot latch for the stale-standby signal (value = failingSince it fired for). */
|
|
125
|
+
private staleSignaledFor;
|
|
107
126
|
constructor(deps: LiveTailSourceDeps);
|
|
108
127
|
private log;
|
|
109
128
|
private now;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LiveTailSource.d.ts","sourceRoot":"","sources":["../../src/core/LiveTailSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAEH,MAAM,WAAW,kBAAkB;IACjC,+EAA+E;IAC/E,eAAe,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IAC3C,oEAAoE;IACpE,YAAY,EAAE,MAAM,MAAM,EAAE,CAAC;IAC7B,oFAAoF;IACpF,SAAS,EAAE;QACT,SAAS,EAAE,CAAC,KAAK,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;KACzF,CAAC;IACF;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IAC5C;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8EAA8E;IAC9E,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,sCAAsC;IACtC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,6FAA6F;IAC7F,GAAG,EAAE,MAAM,CAAC;IACZ,sEAAsE;IACtE,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;
|
|
1
|
+
{"version":3,"file":"LiveTailSource.d.ts","sourceRoot":"","sources":["../../src/core/LiveTailSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAEH,MAAM,WAAW,kBAAkB;IACjC,+EAA+E;IAC/E,eAAe,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IAC3C,oEAAoE;IACpE,YAAY,EAAE,MAAM,MAAM,EAAE,CAAC;IAC7B,oFAAoF;IACpF,SAAS,EAAE;QACT,SAAS,EAAE,CAAC,KAAK,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;KACzF,CAAC;IACF;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IAC5C;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8EAA8E;IAC9E,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,sCAAsC;IACtC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,mBAAmB,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAC1G,wFAAwF;IACxF,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,6FAA6F;IAC7F,GAAG,EAAE,MAAM,CAAC;IACZ,sEAAsE;IACtE,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAOD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAqB;IACvC,uEAAuE;IACvE,OAAO,CAAC,QAAQ,CAA6B;IAC7C,wEAAwE;IACxE,OAAO,CAAC,GAAG,CAA6B;IACxC,iFAAiF;IACjF,OAAO,CAAC,eAAe,CAA6B;IACpD,qEAAqE;IACrE,OAAO,CAAC,QAAQ,CAA6B;IAC7C,oEAAoE;IACpE,OAAO,CAAC,aAAa,CAA6B;IAClD,oEAAoE;IACpE,OAAO,CAAC,YAAY,CAA6B;IACjD,qGAAqG;IACrG,OAAO,CAAC,gBAAgB,CAA6B;gBAEzC,IAAI,EAAE,kBAAkB;IAIpC,OAAO,CAAC,GAAG;IAIX,OAAO,CAAC,GAAG;IAIX,0EAA0E;IAC1E,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAIjC;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC;IA+FxE,oFAAoF;IAC9E,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAQzD,2FAA2F;IACrF,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;CAIlD"}
|
|
@@ -45,6 +45,7 @@
|
|
|
45
45
|
const DEFAULT_MAX_FLUSH_BYTES = 256 * 1024;
|
|
46
46
|
const DEFAULT_BACKOFF_BASE_MS = 5_000;
|
|
47
47
|
const DEFAULT_BACKOFF_MAX_MS = 300_000;
|
|
48
|
+
const DEFAULT_STALE_SIGNAL_AFTER_MS = 30 * 60_000;
|
|
48
49
|
export class LiveTailSource {
|
|
49
50
|
d;
|
|
50
51
|
/** Per-topic content already streamed (the prefix the standby has). */
|
|
@@ -57,6 +58,10 @@ export class LiveTailSource {
|
|
|
57
58
|
failures = new Map();
|
|
58
59
|
/** Per-topic earliest next attempt (ms epoch) while backing off. */
|
|
59
60
|
nextAttemptAt = new Map();
|
|
61
|
+
/** Per-topic failure-episode start (ms epoch; absent = healthy). */
|
|
62
|
+
failingSince = new Map();
|
|
63
|
+
/** Episode-keyed one-shot latch for the stale-standby signal (value = failingSince it fired for). */
|
|
64
|
+
staleSignaledFor = new Map();
|
|
60
65
|
constructor(deps) {
|
|
61
66
|
this.d = deps;
|
|
62
67
|
}
|
|
@@ -140,10 +145,26 @@ export class LiveTailSource {
|
|
|
140
145
|
const backoff = Math.min(base * 2 ** (failures - 1), cap);
|
|
141
146
|
this.nextAttemptAt.set(topic, this.now() + backoff);
|
|
142
147
|
this.log(`flush of topic ${topic} seq ${nextSeq} not acknowledged — retry in ${Math.round(backoff / 1000)}s (failure #${failures})`);
|
|
148
|
+
// Eternal Sentinel condition 4: retrying forever is correct, but a topic
|
|
149
|
+
// whose standby copy has been stale past the threshold says so ONCE per
|
|
150
|
+
// episode (episode-keyed latch, same defensive shape as the supervisor's
|
|
151
|
+
// SlowRetrySentinelEscalation — a fresh episode re-arms automatically).
|
|
152
|
+
const episodeStart = this.failingSince.get(topic) ?? now;
|
|
153
|
+
if (!this.failingSince.has(topic))
|
|
154
|
+
this.failingSince.set(topic, episodeStart);
|
|
155
|
+
const failingForMs = this.now() - episodeStart;
|
|
156
|
+
if (failingForMs >= (this.d.staleSignalAfterMs ?? DEFAULT_STALE_SIGNAL_AFTER_MS)
|
|
157
|
+
&& this.staleSignaledFor.get(topic) !== episodeStart) {
|
|
158
|
+
this.staleSignaledFor.set(topic, episodeStart);
|
|
159
|
+
this.log(`topic ${topic} standby copy STALE — flushes failing for ${Math.round(failingForMs / 60_000)}min (signaling once; retries continue)`);
|
|
160
|
+
this.d.reportStaleStandby?.({ topic, failingForMs, consecutiveFailures: failures });
|
|
161
|
+
}
|
|
143
162
|
return { topic, seq: this.currentSeq(topic), flushed: false };
|
|
144
163
|
}
|
|
145
164
|
this.failures.delete(topic);
|
|
146
165
|
this.nextAttemptAt.delete(topic);
|
|
166
|
+
this.failingSince.delete(topic);
|
|
167
|
+
this.staleSignaledFor.delete(topic);
|
|
147
168
|
this.seq.set(topic, nextSeq);
|
|
148
169
|
this.streamed.set(topic, full);
|
|
149
170
|
if (version !== undefined)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LiveTailSource.js","sourceRoot":"","sources":["../../src/core/LiveTailSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;
|
|
1
|
+
{"version":3,"file":"LiveTailSource.js","sourceRoot":"","sources":["../../src/core/LiveTailSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AA8DH,MAAM,uBAAuB,GAAG,GAAG,GAAG,IAAI,CAAC;AAC3C,MAAM,uBAAuB,GAAG,KAAK,CAAC;AACtC,MAAM,sBAAsB,GAAG,OAAO,CAAC;AACvC,MAAM,6BAA6B,GAAG,EAAE,GAAG,MAAM,CAAC;AAElD,MAAM,OAAO,cAAc;IACR,CAAC,CAAqB;IACvC,uEAAuE;IAC/D,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,wEAAwE;IAChE,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,iFAAiF;IACzE,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,qEAAqE;IAC7D,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,oEAAoE;IAC5D,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAClD,oEAAoE;IAC5D,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjD,qGAAqG;IAC7F,gBAAgB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAErD,YAAY,IAAwB;QAClC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;IAChB,CAAC;IAEO,GAAG,CAAC,CAAS;QACnB,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC;IAEO,GAAG;QACT,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,0EAA0E;IAC1E,UAAU,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAAa,EAAE,IAAgB;QAC9C,MAAM,OAAO,GAAiB,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACrF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;YACjB,sEAAsE;YACtE,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9C,IAAI,OAAO,KAAK,SAAS,IAAI,GAAG,GAAG,OAAO;gBAAE,OAAO,OAAO,CAAC;YAE3D,sEAAsE;YACtE,2EAA2E;YAC3E,wEAAwE;YACxE,wBAAwB;YACxB,IAAI,IAAI,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;gBACxC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC;oBAAE,OAAO,OAAO,CAAC;YAC5D,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,KAAK,CAAC,CAAC;QAEhD,MAAM,kBAAkB,GAAG,GAAiB,EAAE;YAC5C,wEAAwE;YACxE,8CAA8C;YAC9C,IAAI,OAAO,KAAK,SAAS;gBAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACpE,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC;QAEF,IAAI,KAAa,CAAC;QAClB,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,OAAO,kBAAkB,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,qEAAqE;YACrE,IAAI,CAAC,GAAG,CAAC,SAAS,KAAK,8DAA8D,CAAC,CAAC;YACvF,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,kBAAkB,EAAE,CAAC;QAC9B,CAAC;QAED,6EAA6E;QAC7E,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,aAAa,IAAI,uBAAuB,CAAC;QACjE,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,GAAG,CAAC,SAAS,KAAK,kBAAkB,KAAK,CAAC,MAAM,MAAM,QAAQ,+BAA+B,CAAC,CAAC;YACpG,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACrF,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,0EAA0E;YAC1E,4EAA4E;YAC5E,0EAA0E;YAC1E,4CAA4C;YAC5C,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACrD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,oBAAoB,IAAI,uBAAuB,CAAC;YACpE,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,IAAI,sBAAsB,CAAC;YACjE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YACpD,IAAI,CAAC,GAAG,CACN,kBAAkB,KAAK,QAAQ,OAAO,gCAAgC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,QAAQ,GAAG,CAC3H,CAAC;YAEF,yEAAyE;YACzE,wEAAwE;YACxE,yEAAyE;YACzE,wEAAwE;YACxE,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC;YACzD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;YAC9E,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;YAC/C,IAAI,YAAY,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,IAAI,6BAA6B,CAAC;mBACzE,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,YAAY,EAAE,CAAC;gBACzD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,SAAS,KAAK,6CAA6C,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,wCAAwC,CAAC,CAAC;gBAC/I,IAAI,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,QAAQ,EAAE,CAAC,CAAC;YACtF,CAAC;YACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAChE,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC/B,IAAI,OAAO,KAAK,SAAS;YAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACpE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAChD,CAAC;IAED,oFAAoF;IACpF,KAAK,CAAC,QAAQ,CAAC,IAAgB;QAC7B,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC;YAC1C,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,2FAA2F;IAC3F,KAAK,CAAC,QAAQ,CAAC,IAAgB;QAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-06-06T03:
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-06-06T03:31:31.289Z",
|
|
5
|
+
"instarVersion": "1.3.332",
|
|
6
6
|
"entryCount": 199,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
Audit fix #3 under "No Unbounded Loops" (P19, Eternal Sentinel condition 4):
|
|
9
|
+
the live-tail flusher's capped backoff (#867) correctly retries a failing topic
|
|
10
|
+
forever, but a topic whose standby copy went stale never said so. Now a
|
|
11
|
+
per-topic episode latch (the `SlowRetrySentinelEscalation` shape) fires once
|
|
12
|
+
per outage when flushes have failed ≥30min: one log line + one
|
|
13
|
+
`DegradationReporter` record (`LiveTail.standbyFreshness`, topic NAME included,
|
|
14
|
+
housekeeping channel — the reporter's per-feature 1h cooldown bounds even an
|
|
15
|
+
all-topics-stale storm to a single alert). Success clears the episode. The
|
|
16
|
+
`reportStaleStandby` dep is optional — omitted, behavior is byte-identical.
|
|
17
|
+
|
|
18
|
+
## What to Tell Your User
|
|
19
|
+
|
|
20
|
+
Only relevant if I run on more than one machine: when the standby machine's
|
|
21
|
+
copy of a conversation falls behind for over half an hour (because syncs to it
|
|
22
|
+
keep failing), that fact is now recorded in my health/degradation log instead
|
|
23
|
+
of being invisible. Nothing pings you — but if a machine takeover ever resumes
|
|
24
|
+
a conversation from an older point, there's now a checkable record explaining
|
|
25
|
+
exactly which conversation was behind and since when.
|
|
26
|
+
|
|
27
|
+
## Summary of New Capabilities
|
|
28
|
+
|
|
29
|
+
- Stale-standby visibility: one degradation record per conversation per outage
|
|
30
|
+
when cross-machine sync has been failing ≥30min (`LiveTail.standbyFreshness`
|
|
31
|
+
in the degradation log / Process Health surfaces); retries continue
|
|
32
|
+
unchanged. No configuration needed.
|
|
33
|
+
|
|
34
|
+
## Evidence
|
|
35
|
+
|
|
36
|
+
Gap noted by #867's own second-pass reviewer ("backoff but no breaker") and
|
|
37
|
+
resolved per the ratified Eternal Sentinel clause: persistence kept, silence
|
|
38
|
+
removed. Focused adversarial second-pass: CONCUR (episode timing, firing bound
|
|
39
|
+
= threshold + one backoff window, recordNoNewContent edge traced safe,
|
|
40
|
+
DegradationReporter flood analysis). Tests: 21 green in LiveTailSource.test.ts
|
|
41
|
+
incl. the P19 sustained-failure bound (~100 failing windows → exactly 1
|
|
42
|
+
signal) and the server.ts wiring pin; tsc clean.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Side-Effects Review — Live-Tail Stale-Standby Signal
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `livetail-stale-signal`
|
|
4
|
+
**Date:** `2026-06-06`
|
|
5
|
+
**Author:** `Echo (instar-dev agent)`
|
|
6
|
+
**Second-pass reviewer:** `focused adversarial reviewer subagent — CONCUR (all four novel-integration probes clean; the episode-latch pattern itself carries two same-night CONCUR reviews)`
|
|
7
|
+
|
|
8
|
+
## Summary of the change
|
|
9
|
+
|
|
10
|
+
`LiveTailSource` gains the Eternal Sentinel's condition-4 observability for its capped-backoff retry loop: per-topic `failingSince` episode stamp + episode-keyed one-shot latch; the first attempt at/after `staleSignalAfterMs` (default 30min) logs once and calls the optional `reportStaleStandby` dep once per episode; success clears both. `server.ts` wires the dep to `DegradationReporter` (`LiveTail.standbyFreshness`, topic name resolved). Files: `LiveTailSource.ts`, one wiring block in `server.ts`, tests.
|
|
11
|
+
|
|
12
|
+
## Decision-point inventory
|
|
13
|
+
|
|
14
|
+
- `LiveTailSource` failure branch — **modify (additive)** — adds episode accounting + one-shot signal; backoff, retry, delta, and seq semantics untouched.
|
|
15
|
+
- `server.ts` LiveTailSource construction — **modify** — wires the reporter dep.
|
|
16
|
+
|
|
17
|
+
## 1. Over-block / 2. Under-block
|
|
18
|
+
|
|
19
|
+
Nothing blocked — pure signal. Detection bound: threshold + one backoff window (≤5min at cap; reviewer-confirmed attempts always resume when the window opens). Known pre-existing edge (traced by reviewer, probe 3): content reverted to exactly-streamed state while failures>0 keeps rebuilding content per tick (version gate stays bypassed) — a #867-era micro-inefficiency in a bizarre edge, unchanged here; `failingSince` persisting there is correct (success remains the only exit). Remaining audit targets <!-- tracked: CMT-1109 -->.
|
|
20
|
+
|
|
21
|
+
## 3. Level-of-abstraction fit / 4. Signal vs authority
|
|
22
|
+
|
|
23
|
+
The latch lives in the loop it observes (third use of the established suppressor shape tonight); delivery rides the standard degradation channel — deliberately NOT the attention queue or Telegram (a stale standby copy is operator-relevant housekeeping, not a user decision). **Signal-only** per `docs/signal-vs-authority.md`: the dep can only record; removing it restores byte-identical behavior (test-pinned).
|
|
24
|
+
|
|
25
|
+
## 5. Interactions
|
|
26
|
+
|
|
27
|
+
- **Concurrency:** flushAll is sequential; all map mutations are post-await synchronous code — no double-stamp path (reviewer probe 1).
|
|
28
|
+
- **Flood:** all topics share ONE reporter feature key with a 1h alert cooldown — N simultaneously-stale topics ⇒ at most one user-facing alert; per-topic records remain for inspection (reviewer probe 4). Bounded Notification Surface: no topic creation, no per-element pings.
|
|
29
|
+
- **Handoff force path:** shares failure accounting — a forced attempt's failure correctly extends the episode.
|
|
30
|
+
|
|
31
|
+
## 6. External surfaces / 7. Rollback
|
|
32
|
+
|
|
33
|
+
Logs + degradation records only; no API/schema/config/persistent state (defaults in code; dep optional). Rollback = revert; only the silence returns.
|
|
34
|
+
|
|
35
|
+
## Conclusion
|
|
36
|
+
|
|
37
|
+
Third and final observability fix in tonight's P19 series: the reaper loop got its back-off (#863), the supervisor healer got its voice (#871), the lease wire got its brakes (#874), and now the live-tail's sanctioned forever-retry can no longer go quietly stale. Persistence everywhere, silence nowhere.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## Phase 5 — Second-pass review (cross-machine continuity → performed)
|
|
42
|
+
|
|
43
|
+
The episode-keyed one-shot latch pattern carries two same-night line-level CONCUR reviews (supervisor escalation; mesh brakes). A focused adversarial pass probed only this PR's novel integration points: (1) episode-stamp timing across the awaited broadcast + same-topic concurrency — no double-stamp (sequential flushAll, single-threaded post-await mutations); (2) firing bound under backoff gating — threshold + ≤1 backoff window, and no stuck-in-backoff-without-attempts path exists; (3) the `recordNoNewContent`-while-failing edge — traced: cannot fire spuriously, cannot wedge, success is the only episode exit; (4) DegradationReporter flood analysis — per-feature 1h cooldown caps user-facing volume at one alert regardless of topic count. Ran the 21-test suite + tsc — green/clean. **Verdict: CONCUR.**
|