instar 1.3.331 → 1.3.333
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +12 -0
- package/dist/commands/server.js.map +1 -1
- package/dist/core/LiveTailSource.d.ts +19 -0
- package/dist/core/LiveTailSource.d.ts.map +1 -1
- package/dist/core/LiveTailSource.js +21 -0
- package/dist/core/LiveTailSource.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +2 -2
- package/upgrades/1.3.333.md +55 -0
- package/upgrades/side-effects/livetail-stale-signal.md +43 -0
- package/upgrades/side-effects/test-env-hermetic-reply-scripts.md +67 -0
- package/upgrades/test-env-hermetic-reply-scripts.eli16.md +9 -0
|
@@ -74,6 +74,21 @@ export interface LiveTailSourceDeps {
|
|
|
74
74
|
failureBackoffBaseMs?: number;
|
|
75
75
|
/** Backoff ceiling. Default 5 min. */
|
|
76
76
|
failureBackoffMaxMs?: number;
|
|
77
|
+
/**
|
|
78
|
+
* Eternal Sentinel condition 4 ("No Unbounded Loops" / P19): the backoff
|
|
79
|
+
* keeps retrying a failing topic forever (correct — the standby copy should
|
|
80
|
+
* converge whenever the peer recovers), but a topic whose flushes have been
|
|
81
|
+
* failing past staleSignalAfterMs must SAY SO once per episode instead of
|
|
82
|
+
* going quietly stale. Wired to DegradationReporter in server.ts; omitted →
|
|
83
|
+
* silent (tests / channels without a reporter).
|
|
84
|
+
*/
|
|
85
|
+
reportStaleStandby?: (info: {
|
|
86
|
+
topic: string;
|
|
87
|
+
failingForMs: number;
|
|
88
|
+
consecutiveFailures: number;
|
|
89
|
+
}) => void;
|
|
90
|
+
/** Sustained-failure threshold for the one-per-episode stale signal. Default 30 min. */
|
|
91
|
+
staleSignalAfterMs?: number;
|
|
77
92
|
now?: () => number;
|
|
78
93
|
logger?: (msg: string) => void;
|
|
79
94
|
}
|
|
@@ -104,6 +119,10 @@ export declare class LiveTailSource {
|
|
|
104
119
|
private failures;
|
|
105
120
|
/** Per-topic earliest next attempt (ms epoch) while backing off. */
|
|
106
121
|
private nextAttemptAt;
|
|
122
|
+
/** Per-topic failure-episode start (ms epoch; absent = healthy). */
|
|
123
|
+
private failingSince;
|
|
124
|
+
/** Episode-keyed one-shot latch for the stale-standby signal (value = failingSince it fired for). */
|
|
125
|
+
private staleSignaledFor;
|
|
107
126
|
constructor(deps: LiveTailSourceDeps);
|
|
108
127
|
private log;
|
|
109
128
|
private now;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LiveTailSource.d.ts","sourceRoot":"","sources":["../../src/core/LiveTailSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAEH,MAAM,WAAW,kBAAkB;IACjC,+EAA+E;IAC/E,eAAe,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IAC3C,oEAAoE;IACpE,YAAY,EAAE,MAAM,MAAM,EAAE,CAAC;IAC7B,oFAAoF;IACpF,SAAS,EAAE;QACT,SAAS,EAAE,CAAC,KAAK,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;KACzF,CAAC;IACF;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IAC5C;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8EAA8E;IAC9E,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,sCAAsC;IACtC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,6FAA6F;IAC7F,GAAG,EAAE,MAAM,CAAC;IACZ,sEAAsE;IACtE,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;
|
|
1
|
+
{"version":3,"file":"LiveTailSource.d.ts","sourceRoot":"","sources":["../../src/core/LiveTailSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAEH,MAAM,WAAW,kBAAkB;IACjC,+EAA+E;IAC/E,eAAe,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IAC3C,oEAAoE;IACpE,YAAY,EAAE,MAAM,MAAM,EAAE,CAAC;IAC7B,oFAAoF;IACpF,SAAS,EAAE;QACT,SAAS,EAAE,CAAC,KAAK,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;KACzF,CAAC;IACF;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IAC5C;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8EAA8E;IAC9E,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,sCAAsC;IACtC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,mBAAmB,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAC1G,wFAAwF;IACxF,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,6FAA6F;IAC7F,GAAG,EAAE,MAAM,CAAC;IACZ,sEAAsE;IACtE,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAOD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAqB;IACvC,uEAAuE;IACvE,OAAO,CAAC,QAAQ,CAA6B;IAC7C,wEAAwE;IACxE,OAAO,CAAC,GAAG,CAA6B;IACxC,iFAAiF;IACjF,OAAO,CAAC,eAAe,CAA6B;IACpD,qEAAqE;IACrE,OAAO,CAAC,QAAQ,CAA6B;IAC7C,oEAAoE;IACpE,OAAO,CAAC,aAAa,CAA6B;IAClD,oEAAoE;IACpE,OAAO,CAAC,YAAY,CAA6B;IACjD,qGAAqG;IACrG,OAAO,CAAC,gBAAgB,CAA6B;gBAEzC,IAAI,EAAE,kBAAkB;IAIpC,OAAO,CAAC,GAAG;IAIX,OAAO,CAAC,GAAG;IAIX,0EAA0E;IAC1E,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAIjC;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC;IA+FxE,oFAAoF;IAC9E,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAQzD,2FAA2F;IACrF,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;CAIlD"}
|
|
@@ -45,6 +45,7 @@
|
|
|
45
45
|
const DEFAULT_MAX_FLUSH_BYTES = 256 * 1024;
|
|
46
46
|
const DEFAULT_BACKOFF_BASE_MS = 5_000;
|
|
47
47
|
const DEFAULT_BACKOFF_MAX_MS = 300_000;
|
|
48
|
+
const DEFAULT_STALE_SIGNAL_AFTER_MS = 30 * 60_000;
|
|
48
49
|
export class LiveTailSource {
|
|
49
50
|
d;
|
|
50
51
|
/** Per-topic content already streamed (the prefix the standby has). */
|
|
@@ -57,6 +58,10 @@ export class LiveTailSource {
|
|
|
57
58
|
failures = new Map();
|
|
58
59
|
/** Per-topic earliest next attempt (ms epoch) while backing off. */
|
|
59
60
|
nextAttemptAt = new Map();
|
|
61
|
+
/** Per-topic failure-episode start (ms epoch; absent = healthy). */
|
|
62
|
+
failingSince = new Map();
|
|
63
|
+
/** Episode-keyed one-shot latch for the stale-standby signal (value = failingSince it fired for). */
|
|
64
|
+
staleSignaledFor = new Map();
|
|
60
65
|
constructor(deps) {
|
|
61
66
|
this.d = deps;
|
|
62
67
|
}
|
|
@@ -140,10 +145,26 @@ export class LiveTailSource {
|
|
|
140
145
|
const backoff = Math.min(base * 2 ** (failures - 1), cap);
|
|
141
146
|
this.nextAttemptAt.set(topic, this.now() + backoff);
|
|
142
147
|
this.log(`flush of topic ${topic} seq ${nextSeq} not acknowledged — retry in ${Math.round(backoff / 1000)}s (failure #${failures})`);
|
|
148
|
+
// Eternal Sentinel condition 4: retrying forever is correct, but a topic
|
|
149
|
+
// whose standby copy has been stale past the threshold says so ONCE per
|
|
150
|
+
// episode (episode-keyed latch, same defensive shape as the supervisor's
|
|
151
|
+
// SlowRetrySentinelEscalation — a fresh episode re-arms automatically).
|
|
152
|
+
const episodeStart = this.failingSince.get(topic) ?? now;
|
|
153
|
+
if (!this.failingSince.has(topic))
|
|
154
|
+
this.failingSince.set(topic, episodeStart);
|
|
155
|
+
const failingForMs = this.now() - episodeStart;
|
|
156
|
+
if (failingForMs >= (this.d.staleSignalAfterMs ?? DEFAULT_STALE_SIGNAL_AFTER_MS)
|
|
157
|
+
&& this.staleSignaledFor.get(topic) !== episodeStart) {
|
|
158
|
+
this.staleSignaledFor.set(topic, episodeStart);
|
|
159
|
+
this.log(`topic ${topic} standby copy STALE — flushes failing for ${Math.round(failingForMs / 60_000)}min (signaling once; retries continue)`);
|
|
160
|
+
this.d.reportStaleStandby?.({ topic, failingForMs, consecutiveFailures: failures });
|
|
161
|
+
}
|
|
143
162
|
return { topic, seq: this.currentSeq(topic), flushed: false };
|
|
144
163
|
}
|
|
145
164
|
this.failures.delete(topic);
|
|
146
165
|
this.nextAttemptAt.delete(topic);
|
|
166
|
+
this.failingSince.delete(topic);
|
|
167
|
+
this.staleSignaledFor.delete(topic);
|
|
147
168
|
this.seq.set(topic, nextSeq);
|
|
148
169
|
this.streamed.set(topic, full);
|
|
149
170
|
if (version !== undefined)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LiveTailSource.js","sourceRoot":"","sources":["../../src/core/LiveTailSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;
|
|
1
|
+
{"version":3,"file":"LiveTailSource.js","sourceRoot":"","sources":["../../src/core/LiveTailSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AA8DH,MAAM,uBAAuB,GAAG,GAAG,GAAG,IAAI,CAAC;AAC3C,MAAM,uBAAuB,GAAG,KAAK,CAAC;AACtC,MAAM,sBAAsB,GAAG,OAAO,CAAC;AACvC,MAAM,6BAA6B,GAAG,EAAE,GAAG,MAAM,CAAC;AAElD,MAAM,OAAO,cAAc;IACR,CAAC,CAAqB;IACvC,uEAAuE;IAC/D,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,wEAAwE;IAChE,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,iFAAiF;IACzE,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,qEAAqE;IAC7D,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,oEAAoE;IAC5D,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAClD,oEAAoE;IAC5D,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjD,qGAAqG;IAC7F,gBAAgB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAErD,YAAY,IAAwB;QAClC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;IAChB,CAAC;IAEO,GAAG,CAAC,CAAS;QACnB,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC;IAEO,GAAG;QACT,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,0EAA0E;IAC1E,UAAU,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAAa,EAAE,IAAgB;QAC9C,MAAM,OAAO,GAAiB,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACrF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;YACjB,sEAAsE;YACtE,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9C,IAAI,OAAO,KAAK,SAAS,IAAI,GAAG,GAAG,OAAO;gBAAE,OAAO,OAAO,CAAC;YAE3D,sEAAsE;YACtE,2EAA2E;YAC3E,wEAAwE;YACxE,wBAAwB;YACxB,IAAI,IAAI,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;gBACxC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC;oBAAE,OAAO,OAAO,CAAC;YAC5D,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,KAAK,CAAC,CAAC;QAEhD,MAAM,kBAAkB,GAAG,GAAiB,EAAE;YAC5C,wEAAwE;YACxE,8CAA8C;YAC9C,IAAI,OAAO,KAAK,SAAS;gBAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACpE,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC;QAEF,IAAI,KAAa,CAAC;QAClB,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,OAAO,kBAAkB,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,qEAAqE;YACrE,IAAI,CAAC,GAAG,CAAC,SAAS,KAAK,8DAA8D,CAAC,CAAC;YACvF,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,kBAAkB,EAAE,CAAC;QAC9B,CAAC;QAED,6EAA6E;QAC7E,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,aAAa,IAAI,uBAAuB,CAAC;QACjE,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,GAAG,CAAC,SAAS,KAAK,kBAAkB,KAAK,CAAC,MAAM,MAAM,QAAQ,+BAA+B,CAAC,CAAC;YACpG,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACrF,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,0EAA0E;YAC1E,4EAA4E;YAC5E,0EAA0E;YAC1E,4CAA4C;YAC5C,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACrD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,oBAAoB,IAAI,uBAAuB,CAAC;YACpE,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,IAAI,sBAAsB,CAAC;YACjE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YACpD,IAAI,CAAC,GAAG,CACN,kBAAkB,KAAK,QAAQ,OAAO,gCAAgC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,QAAQ,GAAG,CAC3H,CAAC;YAEF,yEAAyE;YACzE,wEAAwE;YACxE,yEAAyE;YACzE,wEAAwE;YACxE,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC;YACzD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;YAC9E,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;YAC/C,IAAI,YAAY,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,IAAI,6BAA6B,CAAC;mBACzE,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,YAAY,EAAE,CAAC;gBACzD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,SAAS,KAAK,6CAA6C,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,wCAAwC,CAAC,CAAC;gBAC/I,IAAI,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,QAAQ,EAAE,CAAC,CAAC;YACtF,CAAC;YACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAChE,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC/B,IAAI,OAAO,KAAK,SAAS;YAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACpE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAChD,CAAC;IAED,oFAAoF;IACpF,KAAK,CAAC,QAAQ,CAAC,IAAgB;QAC7B,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC;YAC1C,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,2FAA2F;IAC3F,KAAK,CAAC,QAAQ,CAAC,IAAgB;QAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-06-06T03:
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-06-06T03:35:43.361Z",
|
|
5
|
+
"instarVersion": "1.3.333",
|
|
6
6
|
"entryCount": 199,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
Audit fix #3 under "No Unbounded Loops" (P19, Eternal Sentinel condition 4):
|
|
9
|
+
the live-tail flusher's capped backoff (#867) correctly retries a failing topic
|
|
10
|
+
forever, but a topic whose standby copy went stale never said so. Now a
|
|
11
|
+
per-topic episode latch (the `SlowRetrySentinelEscalation` shape) fires once
|
|
12
|
+
per outage when flushes have failed ≥30min: one log line + one
|
|
13
|
+
`DegradationReporter` record (`LiveTail.standbyFreshness`, topic NAME included,
|
|
14
|
+
housekeeping channel — the reporter's per-feature 1h cooldown bounds even an
|
|
15
|
+
all-topics-stale storm to a single alert). Success clears the episode. The
|
|
16
|
+
`reportStaleStandby` dep is optional — omitted, behavior is byte-identical.
|
|
17
|
+
|
|
18
|
+
Made the three `telegram-reply.sh` test harnesses hermetic against a live
|
|
19
|
+
agent session's environment: each script-spawn now blanks
|
|
20
|
+
`INSTAR_AUTH_TOKEN` so the script exercises the config-file auth fallback
|
|
21
|
+
the tests intend, instead of inheriting the runner's real session token
|
|
22
|
+
(which the test servers' auth middleware correctly rejects, failing the
|
|
23
|
+
tests only when run inside an agent session). Same hermeticity class as
|
|
24
|
+
the #862 unit-suite fix, applied to the reply-script family.
|
|
25
|
+
|
|
26
|
+
## What to Tell Your User
|
|
27
|
+
|
|
28
|
+
Only relevant if I run on more than one machine: when the standby machine's
|
|
29
|
+
copy of a conversation falls behind for over half an hour (because syncs to it
|
|
30
|
+
keep failing), that fact is now recorded in my health/degradation log instead
|
|
31
|
+
of being invisible. Nothing pings you — but if a machine takeover ever resumes
|
|
32
|
+
a conversation from an older point, there's now a checkable record explaining
|
|
33
|
+
exactly which conversation was behind and since when.
|
|
34
|
+
|
|
35
|
+
## Summary of New Capabilities
|
|
36
|
+
|
|
37
|
+
- Stale-standby visibility: one degradation record per conversation per outage
|
|
38
|
+
when cross-machine sync has been failing ≥30min (`LiveTail.standbyFreshness`
|
|
39
|
+
in the degradation log / Process Health surfaces); retries continue
|
|
40
|
+
unchanged. No configuration needed.
|
|
41
|
+
|
|
42
|
+
## Evidence
|
|
43
|
+
|
|
44
|
+
Gap noted by #867's own second-pass reviewer ("backoff but no breaker") and
|
|
45
|
+
resolved per the ratified Eternal Sentinel clause: persistence kept, silence
|
|
46
|
+
removed. Focused adversarial second-pass: CONCUR (episode timing, firing bound
|
|
47
|
+
= threshold + one backoff window, recordNoNewContent edge traced safe,
|
|
48
|
+
DegradationReporter flood analysis). Tests: 21 green in LiveTailSource.test.ts
|
|
49
|
+
incl. the P19 sustained-failure bound (~100 failing windows → exactly 1
|
|
50
|
+
signal) and the server.ts wiring pin; tsc clean.
|
|
51
|
+
|
|
52
|
+
- Bisected: `tests/integration/telegram-reply-end-to-end.test.ts` fails
|
|
53
|
+
with `INSTAR_AUTH_TOKEN` exported, passes without; single-var culprit.
|
|
54
|
+
- Post-fix: all 4 reply-script test files (32 tests) green BOTH under a
|
|
55
|
+
live agent env and a stripped env.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Side-Effects Review — Live-Tail Stale-Standby Signal
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `livetail-stale-signal`
|
|
4
|
+
**Date:** `2026-06-06`
|
|
5
|
+
**Author:** `Echo (instar-dev agent)`
|
|
6
|
+
**Second-pass reviewer:** `focused adversarial reviewer subagent — CONCUR (all four novel-integration probes clean; the episode-latch pattern itself carries two same-night CONCUR reviews)`
|
|
7
|
+
|
|
8
|
+
## Summary of the change
|
|
9
|
+
|
|
10
|
+
`LiveTailSource` gains the Eternal Sentinel's condition-4 observability for its capped-backoff retry loop: per-topic `failingSince` episode stamp + episode-keyed one-shot latch; the first attempt at/after `staleSignalAfterMs` (default 30min) logs once and calls the optional `reportStaleStandby` dep once per episode; success clears both. `server.ts` wires the dep to `DegradationReporter` (`LiveTail.standbyFreshness`, topic name resolved). Files: `LiveTailSource.ts`, one wiring block in `server.ts`, tests.
|
|
11
|
+
|
|
12
|
+
## Decision-point inventory
|
|
13
|
+
|
|
14
|
+
- `LiveTailSource` failure branch — **modify (additive)** — adds episode accounting + one-shot signal; backoff, retry, delta, and seq semantics untouched.
|
|
15
|
+
- `server.ts` LiveTailSource construction — **modify** — wires the reporter dep.
|
|
16
|
+
|
|
17
|
+
## 1. Over-block / 2. Under-block
|
|
18
|
+
|
|
19
|
+
Nothing blocked — pure signal. Detection bound: threshold + one backoff window (≤5min at cap; reviewer-confirmed attempts always resume when the window opens). Known pre-existing edge (traced by reviewer, probe 3): content reverted to exactly-streamed state while failures>0 keeps rebuilding content per tick (version gate stays bypassed) — a #867-era micro-inefficiency in a bizarre edge, unchanged here; `failingSince` persisting there is correct (success remains the only exit). Remaining audit targets <!-- tracked: CMT-1109 -->.
|
|
20
|
+
|
|
21
|
+
## 3. Level-of-abstraction fit / 4. Signal vs authority
|
|
22
|
+
|
|
23
|
+
The latch lives in the loop it observes (third use of the established suppressor shape tonight); delivery rides the standard degradation channel — deliberately NOT the attention queue or Telegram (a stale standby copy is operator-relevant housekeeping, not a user decision). **Signal-only** per `docs/signal-vs-authority.md`: the dep can only record; removing it restores byte-identical behavior (test-pinned).
|
|
24
|
+
|
|
25
|
+
## 5. Interactions
|
|
26
|
+
|
|
27
|
+
- **Concurrency:** flushAll is sequential; all map mutations are post-await synchronous code — no double-stamp path (reviewer probe 1).
|
|
28
|
+
- **Flood:** all topics share ONE reporter feature key with a 1h alert cooldown — N simultaneously-stale topics ⇒ at most one user-facing alert; per-topic records remain for inspection (reviewer probe 4). Bounded Notification Surface: no topic creation, no per-element pings.
|
|
29
|
+
- **Handoff force path:** shares failure accounting — a forced attempt's failure correctly extends the episode.
|
|
30
|
+
|
|
31
|
+
## 6. External surfaces / 7. Rollback
|
|
32
|
+
|
|
33
|
+
Logs + degradation records only; no API/schema/config/persistent state (defaults in code; dep optional). Rollback = revert; only the silence returns.
|
|
34
|
+
|
|
35
|
+
## Conclusion
|
|
36
|
+
|
|
37
|
+
Third and final observability fix in tonight's P19 series: the reaper loop got its back-off (#863), the supervisor healer got its voice (#871), the lease wire got its brakes (#874), and now the live-tail's sanctioned forever-retry can no longer go quietly stale. Persistence everywhere, silence nowhere.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## Phase 5 — Second-pass review (cross-machine continuity → performed)
|
|
42
|
+
|
|
43
|
+
The episode-keyed one-shot latch pattern carries two same-night line-level CONCUR reviews (supervisor escalation; mesh brakes). A focused adversarial pass probed only this PR's novel integration points: (1) episode-stamp timing across the awaited broadcast + same-topic concurrency — no double-stamp (sequential flushAll, single-threaded post-await mutations); (2) firing bound under backoff gating — threshold + ≤1 backoff window, and no stuck-in-backoff-without-attempts path exists; (3) the `recordNoNewContent`-while-failing edge — traced: cannot fire spuriously, cannot wedge, success is the only episode exit; (4) DegradationReporter flood analysis — per-feature 1h cooldown caps user-facing volume at one alert regardless of topic count. Ran the 21-test suite + tsc — green/clean. **Verdict: CONCUR.**
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Side-Effects Review — Hermetic env for telegram-reply script tests
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `test-env-hermetic-reply-scripts`
|
|
4
|
+
**Date:** `2026-06-05`
|
|
5
|
+
**Author:** `echo`
|
|
6
|
+
**Second-pass reviewer:** `not required (test-only change, no runtime surface)`
|
|
7
|
+
|
|
8
|
+
## Summary of the change
|
|
9
|
+
|
|
10
|
+
Three tests that spawn `telegram-reply.sh` (`tests/integration/telegram-reply-end-to-end.test.ts`, `tests/unit/telegram-reply-recoverable-classification.test.ts`, `tests/unit/reply-scripts.test.ts`) inherited the full process env via `{ ...process.env }`. Inside a live agent session, `INSTAR_AUTH_TOKEN` is exported (SessionManager injects it), the script's documented env-first auth resolution picks it up, and the test server's auth middleware rejects the mismatched Bearer before the route handler records the hit — failing the test only when run inside an agent session. Each spawn site now sets `INSTAR_AUTH_TOKEN: ''` so the script exercises the config-file fallback the tests intend. No runtime files changed.
|
|
11
|
+
|
|
12
|
+
## Decision-point inventory
|
|
13
|
+
|
|
14
|
+
No decision-point surface — test-only. The script's env-first auth precedence is intentionally UNCHANGED (it is documented, load-bearing behavior); only the tests' spawn environments changed.
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## 1. Over-block
|
|
19
|
+
|
|
20
|
+
No block/allow surface — not applicable.
|
|
21
|
+
|
|
22
|
+
## 2. Under-block
|
|
23
|
+
|
|
24
|
+
No block/allow surface — not applicable.
|
|
25
|
+
|
|
26
|
+
## 3. Level-of-abstraction fit
|
|
27
|
+
|
|
28
|
+
Right layer: the fix is at the test spawn sites, not the script. Weakening the script's env-first precedence to make tests pass would invert the actual contract. The sibling `telegram-reply-port-resolution.test.ts` already constructs a minimal env (`{ PATH }`) — the targeted blank keeps each test's existing env needs intact while removing the one leaking variable.
|
|
29
|
+
|
|
30
|
+
## 4. Signal vs authority compliance
|
|
31
|
+
|
|
32
|
+
**Required reference:** [docs/signal-vs-authority.md](../../docs/signal-vs-authority.md)
|
|
33
|
+
|
|
34
|
+
- [x] No — this change has no block/allow surface.
|
|
35
|
+
|
|
36
|
+
## 5. Interactions
|
|
37
|
+
|
|
38
|
+
- Shadowing/double-fire/races/feedback loops: none — test-only env construction.
|
|
39
|
+
- Verified the three tests + the already-hermetic sibling all pass together under a live agent env AND under a stripped env (both run locally).
|
|
40
|
+
|
|
41
|
+
## 6. External surfaces
|
|
42
|
+
|
|
43
|
+
None. No runtime change; CI behavior identical (CI never had the variable set — that's why CI was green while local was red).
|
|
44
|
+
|
|
45
|
+
## 7. Rollback cost
|
|
46
|
+
|
|
47
|
+
Trivial revert of three test lines. No persistent state.
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## Conclusion
|
|
52
|
+
|
|
53
|
+
Closes a hermeticity gap in the reply-script test family (the same class PR #862 closed for the unit suite's config leakage): tests must not change verdict based on the runner's live agent environment. Found via Zero-Failure triage of a local full-suite run. Clear to ship.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## Second-pass review (if required)
|
|
58
|
+
|
|
59
|
+
**Reviewer:** not required
|
|
60
|
+
**Independent read of the artifact:** not required — test-only.
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## Evidence pointers
|
|
65
|
+
|
|
66
|
+
- Repro: `npx vitest run tests/integration/telegram-reply-end-to-end.test.ts` fails with `INSTAR_AUTH_TOKEN` exported, passes with it unset (bisected: that single var).
|
|
67
|
+
- Post-fix: all 4 reply-script test files green under the live agent env (32 tests).
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Tests stop failing just because a real agent is running them
|
|
2
|
+
|
|
3
|
+
Three of our tests check the little script that delivers my chat replies. Each test builds a fake miniature server, points the script at it, and watches what the script does. The problem: when those tests run inside a LIVE agent work session (instead of a clean CI machine), the session environment carries the agent's REAL access token in an environment variable — and the script is designed to prefer that variable when it exists. So the script walks up to the fake test server holding the real token, the fake server's door check rejects it (wrong token), and the test fails with a confusing "the script never called the server" error — even though nothing is actually broken.
|
|
4
|
+
|
|
5
|
+
This bit us tonight: the full test suite went red on my machine for exactly this reason, while the same tests pass everywhere else. The "fix" each time would have been to shrug and re-run on CI — which trains everyone to distrust local test runs, the opposite of what a test suite is for.
|
|
6
|
+
|
|
7
|
+
The change is three one-line edits: each test now explicitly blanks that environment variable when launching the script, so the script falls back to reading the token from the test's own config file — the path the tests were always meant to exercise. A fourth sibling test already did this correctly (it builds a minimal clean environment from scratch), which is how we knew the pattern. Comments at each site explain why, so the next person who writes a test like this copies the safe pattern.
|
|
8
|
+
|
|
9
|
+
Nothing about the real script changes. Production behavior is untouched — this only makes the tests honest about what environment they run in. Tests now pass identically on a clean CI box and inside a live agent session.
|