instar 1.3.657 → 1.3.659
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +56 -0
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +14 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/SessionManager.d.ts +12 -0
- package/dist/core/SessionManager.d.ts.map +1 -1
- package/dist/core/SessionManager.js +45 -2
- package/dist/core/SessionManager.js.map +1 -1
- package/dist/core/SessionOwnershipRegistry.d.ts +14 -0
- package/dist/core/SessionOwnershipRegistry.d.ts.map +1 -1
- package/dist/core/SessionOwnershipRegistry.js +9 -0
- package/dist/core/SessionOwnershipRegistry.js.map +1 -1
- package/dist/core/devGatedFeatures.d.ts.map +1 -1
- package/dist/core/devGatedFeatures.js +12 -0
- package/dist/core/devGatedFeatures.js.map +1 -1
- package/dist/core/types.d.ts +32 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/monitoring/StrandedTopicSentinel.d.ts +102 -0
- package/dist/monitoring/StrandedTopicSentinel.d.ts.map +1 -0
- package/dist/monitoring/StrandedTopicSentinel.js +216 -0
- package/dist/monitoring/StrandedTopicSentinel.js.map +1 -0
- package/dist/monitoring/guardManifest.d.ts.map +1 -1
- package/dist/monitoring/guardManifest.js +17 -0
- package/dist/monitoring/guardManifest.js.map +1 -1
- package/dist/monitoring/rateLimitDetection.d.ts +29 -0
- package/dist/monitoring/rateLimitDetection.d.ts.map +1 -1
- package/dist/monitoring/rateLimitDetection.js +29 -0
- package/dist/monitoring/rateLimitDetection.js.map +1 -1
- package/dist/monitoring/strandedTopicDecision.d.ts +96 -0
- package/dist/monitoring/strandedTopicDecision.d.ts.map +1 -0
- package/dist/monitoring/strandedTopicDecision.js +160 -0
- package/dist/monitoring/strandedTopicDecision.js.map +1 -0
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +3 -3
- package/upgrades/1.3.658.md +24 -0
- package/upgrades/1.3.659.md +29 -0
- package/upgrades/side-effects/idle-throttle-settle-gate.md +37 -0
- package/upgrades/side-effects/stranded-inbound-self-heal.md +80 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* strandedTopicDecision — the PURE, unit-testable decision core of the
|
|
3
|
+
* StrandedTopicSentinel (spec docs/specs/stranded-inbound-self-heal.md).
|
|
4
|
+
*
|
|
5
|
+
* A topic is STRANDED when its durable ownership record names a machine that is
|
|
6
|
+
* online-by-heartbeat but, persistently across rich beats, cannot serve the
|
|
7
|
+
* topic's channel — quota-walled (`quotaState.blocked === true`, channel-
|
|
8
|
+
* independent) OR adapter-disconnected (`machineServesChannel(...) === 'no'`).
|
|
9
|
+
* Inbound for that topic routes to the owner that cannot serve it, so it is
|
|
10
|
+
* silently dead while outbound still flows from a healthy lease-holder.
|
|
11
|
+
*
|
|
12
|
+
* This module computes the strand verdict ONLY. It mutates nothing: no
|
|
13
|
+
* ownership CAS, no pin write, no session kill. Fail-closed on EVERY
|
|
14
|
+
* uncertainty (missing field, stale beat, underivable scope, pool view
|
|
15
|
+
* unavailable) — an uncertainty can NEVER manufacture a strand, it routes to
|
|
16
|
+
* SKIP. The quota arm carries detection on its own when the adapter arm skips.
|
|
17
|
+
*
|
|
18
|
+
* Signal-vs-authority: pure read + a verdict object. No authority to misuse.
|
|
19
|
+
*/
|
|
20
|
+
import { machineServesChannel, } from '../core/machineServesChannel.js';
|
|
21
|
+
/** Is this owner's rich beat genuine (present + fresh)? */
|
|
22
|
+
function richBeatAgeMs(cap, now) {
|
|
23
|
+
if (!cap.routerReceivedAt)
|
|
24
|
+
return undefined;
|
|
25
|
+
const t = Date.parse(cap.routerReceivedAt);
|
|
26
|
+
if (!Number.isFinite(t))
|
|
27
|
+
return undefined;
|
|
28
|
+
return now - t;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The pure strand evaluation for one tick. Synchronous, LLM-free, no I/O.
|
|
32
|
+
*
|
|
33
|
+
* Predicate per `active`, non-self-owned record:
|
|
34
|
+
* owner online
|
|
35
|
+
* AND ( quota arm: quotaState.blocked === true [channel-independent]
|
|
36
|
+
* OR adapter arm: machineServesChannel(servesChannels, scope) === 'no'
|
|
37
|
+
* ['unknown'/undefined-scope ⇒ SKIP that arm] )
|
|
38
|
+
* AND the condition has held ≥2 consecutive rich beats spanning ≥ dwellMs.
|
|
39
|
+
*
|
|
40
|
+
* Early no-op: < 2 machines OR !holdsLease ⇒ empty (and the map is dropped).
|
|
41
|
+
*/
|
|
42
|
+
export function evaluateStrandedTopics(input) {
|
|
43
|
+
const { records, capacities, selfMachineId, holdsLease, prevStrandedSince, now, cfg } = input;
|
|
44
|
+
// Early no-op gates (spec step 1). A single-machine agent can't strand on a
|
|
45
|
+
// peer; a non-lease-holder is not the sole actor. Drop the dwell map so a
|
|
46
|
+
// demotion/scale-down doesn't carry stale anchors.
|
|
47
|
+
if (capacities.length < 2 || !holdsLease) {
|
|
48
|
+
return { strandedSet: [], cantAssessCount: 0, nextStrandedSince: {} };
|
|
49
|
+
}
|
|
50
|
+
const capById = new Map();
|
|
51
|
+
for (const c of capacities)
|
|
52
|
+
capById.set(c.machineId, c);
|
|
53
|
+
const strandedSet = [];
|
|
54
|
+
const nextStrandedSince = {};
|
|
55
|
+
let cantAssessCount = 0;
|
|
56
|
+
for (const rec of records) {
|
|
57
|
+
// Only active, peer-owned records are candidates.
|
|
58
|
+
if (rec.status !== 'active')
|
|
59
|
+
continue;
|
|
60
|
+
if (rec.ownerMachineId === selfMachineId)
|
|
61
|
+
continue;
|
|
62
|
+
const owner = capById.get(rec.ownerMachineId);
|
|
63
|
+
// Unknown owner / not in the pool view → can't assess → skip (fail-closed).
|
|
64
|
+
if (!owner)
|
|
65
|
+
continue;
|
|
66
|
+
// A dead owner is the existing Case C's job; this sentinel only covers the
|
|
67
|
+
// online-but-unable gap.
|
|
68
|
+
if (!owner.online)
|
|
69
|
+
continue;
|
|
70
|
+
// Require a genuine rich beat: present + fresh. A stale/undecodable beat
|
|
71
|
+
// fails closed (skip) and does NOT count toward the dwell.
|
|
72
|
+
const beatAgeMs = richBeatAgeMs(owner, now);
|
|
73
|
+
if (beatAgeMs === undefined || beatAgeMs > cfg.freshnessBoundMs)
|
|
74
|
+
continue;
|
|
75
|
+
// ── Arm (a): quota — channel-independent, the dominant incident case. ──
|
|
76
|
+
const quotaArm = owner.quotaState?.blocked === true;
|
|
77
|
+
// ── Arm (b): adapter — best-effort, channel-specific. ──
|
|
78
|
+
let adapterArm = false;
|
|
79
|
+
// A missing servesChannels on an online owner is the anti-blind-spot case:
|
|
80
|
+
// we cannot evaluate the adapter arm. If the quota arm ALSO can't fire
|
|
81
|
+
// (quotaState absent/undecided), this owner is genuinely unassessable.
|
|
82
|
+
const servesPresent = owner.servesChannels !== undefined;
|
|
83
|
+
let armBlind = false;
|
|
84
|
+
if (input.resolveScope) {
|
|
85
|
+
const scope = input.resolveScope(rec.sessionKey);
|
|
86
|
+
if (scope === undefined) {
|
|
87
|
+
// Underivable scope ⇒ SKIP this arm (not a strand).
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
const serve = machineServesChannel(owner.servesChannels, scope);
|
|
91
|
+
if (serve === 'no')
|
|
92
|
+
adapterArm = true;
|
|
93
|
+
else if (serve === 'unknown') {
|
|
94
|
+
// 'unknown' (sparse/absent servesChannels) ⇒ SKIP this arm.
|
|
95
|
+
if (!servesPresent)
|
|
96
|
+
armBlind = true;
|
|
97
|
+
}
|
|
98
|
+
// 'yes' ⇒ owner can serve on this arm → not stranded here.
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const unableToServe = quotaArm || adapterArm;
|
|
102
|
+
if (!unableToServe) {
|
|
103
|
+
// Anti-blind-spot: an online owner we could NOT assess at all — quota
|
|
104
|
+
// undecided AND the adapter arm blinded by a missing servesChannels —
|
|
105
|
+
// increments the can't-assess count (spec step 4). A topic with a usable
|
|
106
|
+
// 'yes' or a derivable scope is genuinely assessed (not blind).
|
|
107
|
+
const quotaUndecided = owner.quotaState === undefined;
|
|
108
|
+
if (quotaUndecided && armBlind)
|
|
109
|
+
cantAssessCount++;
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
// ── Persistence (dwell) — anchor on the FIRST qualifying beat. ──
|
|
113
|
+
const prevSince = prevStrandedSince[rec.sessionKey];
|
|
114
|
+
const strandedSince = typeof prevSince === 'number' ? prevSince : now;
|
|
115
|
+
nextStrandedSince[rec.sessionKey] = strandedSince;
|
|
116
|
+
// The condition must have held ≥ dwellMs (which, given the ≥1 prior beat that
|
|
117
|
+
// set the anchor, spans ≥2 consecutive rich beats). A first-beat strand has
|
|
118
|
+
// dwell 0 < dwellMs → recorded in the map but NOT yet emitted.
|
|
119
|
+
if (now - strandedSince < cfg.dwellMs)
|
|
120
|
+
continue;
|
|
121
|
+
const reason = quotaArm ? 'quota' : 'adapter';
|
|
122
|
+
strandedSet.push({
|
|
123
|
+
sessionKey: rec.sessionKey,
|
|
124
|
+
ownerMachineId: rec.ownerMachineId,
|
|
125
|
+
reason,
|
|
126
|
+
strandedSince,
|
|
127
|
+
ownerBeatAgeMs: beatAgeMs,
|
|
128
|
+
servablePeerExists: servablePeerExists(rec, capacities, selfMachineId, input.resolveScope),
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
return { strandedSet, cantAssessCount, nextStrandedSince };
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Whether SOME online machine could serve the topic's channel — narrowly "online
|
|
135
|
+
* AND not quota-blocked AND machineServesChannel(...) === 'yes'" (the same
|
|
136
|
+
* PlacementExecutor filter, so detector + placement agree). Deliberately NOT a
|
|
137
|
+
* "safe failover target": it vets no pin policy, lease/router readiness, secrets,
|
|
138
|
+
* or session limits — v1 never fails over, so this only tells the operator
|
|
139
|
+
* "somewhere could serve this" vs "nowhere can (fleet-wide wall)". When the scope
|
|
140
|
+
* can't be derived, a not-quota-blocked online peer counts (quota-arm semantics).
|
|
141
|
+
*/
|
|
142
|
+
function servablePeerExists(rec, capacities, selfMachineId, resolveScope) {
|
|
143
|
+
const scope = resolveScope?.(rec.sessionKey);
|
|
144
|
+
for (const c of capacities) {
|
|
145
|
+
if (c.machineId === rec.ownerMachineId)
|
|
146
|
+
continue;
|
|
147
|
+
if (!c.online)
|
|
148
|
+
continue;
|
|
149
|
+
if (c.quotaState?.blocked === true)
|
|
150
|
+
continue;
|
|
151
|
+
if (scope === undefined) {
|
|
152
|
+
// No channel scope → quota-arm semantics: a healthy peer suffices.
|
|
153
|
+
return true;
|
|
154
|
+
}
|
|
155
|
+
if (machineServesChannel(c.servesChannels, scope) === 'yes')
|
|
156
|
+
return true;
|
|
157
|
+
}
|
|
158
|
+
return false;
|
|
159
|
+
}
|
|
160
|
+
//# sourceMappingURL=strandedTopicDecision.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strandedTopicDecision.js","sourceRoot":"","sources":["../../src/monitoring/strandedTopicDecision.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,EACL,oBAAoB,GAErB,MAAM,iCAAiC,CAAC;AAmEzC,2DAA2D;AAC3D,SAAS,aAAa,CAAC,GAAoB,EAAE,GAAW;IACtD,IAAI,CAAC,GAAG,CAAC,gBAAgB;QAAE,OAAO,SAAS,CAAC;IAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,OAAO,GAAG,GAAG,CAAC,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAA4B;IAE5B,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE,iBAAiB,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;IAE9F,4EAA4E;IAC5E,0EAA0E;IAC1E,mDAAmD;IACnD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;QACzC,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC;IACxE,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,KAAK,MAAM,CAAC,IAAI,UAAU;QAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IAExD,MAAM,WAAW,GAAoB,EAAE,CAAC;IACxC,MAAM,iBAAiB,GAA2B,EAAE,CAAC;IACrD,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,kDAAkD;QAClD,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ;YAAE,SAAS;QACtC,IAAI,GAAG,CAAC,cAAc,KAAK,aAAa;YAAE,SAAS;QAEnD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC9C,4EAA4E;QAC5E,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,2EAA2E;QAC3E,yBAAyB;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,SAAS;QAE5B,yEAAyE;QACzE,2DAA2D;QAC3D,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5C,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,GAAG,CAAC,gBAAgB;YAAE,SAAS;QAE1E,0EAA0E;QAC1E,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,EAAE,OAAO,KAAK,IAAI,CAAC;QAEpD,0DAA0D;QAC1D,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,2EAA2E;QAC3E,uEAAuE;QACvE,uEAAuE;QACvE,MAAM,aAAa,GAAG,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC;QACzD,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACjD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,oDAAoD;YACtD,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAG,oBAAoB,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;gBAChE,IAAI,KAAK,KAAK,IAAI;oBAAE,UAAU,GAAG,IAAI,CAAC;qBACjC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBAC7B,4DAA4D;oBAC5D,IAAI,CAAC,aAAa;wBAAE,QAAQ,GAAG,IAAI,CAAC;gBACtC,CAAC;gBACD,2DAA2D;YAC7D,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAG,QAAQ,IAAI,UAAU,CAAC;QAE7C,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,sEAAsE;YACtE,sEAAsE;YACtE,yEAAyE;YACzE,gEAAgE;YAChE,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC;YACtD,IAAI,cAAc,IAAI,QAAQ;gBAAE,eAAe,EAAE,CAAC;YAClD,SAAS;QACX,CAAC;QAED,mEAAmE;QACnE,MAAM,SAAS,GAAG,iBAAiB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,aAAa,GAAG,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC;QACtE,iBAAiB,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC;QAElD,8EAA8E;QAC9E,4EAA4E;QAC5E,+DAA+D;QAC/D,IAAI,GAAG,GAAG,aAAa,GAAG,GAAG,CAAC,OAAO;YAAE,SAAS;QAEhD,MAAM,MAAM,GAAiB,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5D,WAAW,CAAC,IAAI,CAAC;YACf,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,MAAM;YACN,aAAa;YACb,cAAc,EAAE,SAAS;YACzB,kBAAkB,EAAE,kBAAkB,CAAC,GAAG,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,CAAC,YAAY,CAAC;SAC3F,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,iBAAiB,EAAE,CAAC;AAC7D,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,kBAAkB,CACzB,GAA2B,EAC3B,UAA6B,EAC7B,aAAqB,EACrB,YAA+D;IAE/D,MAAM,KAAK,GAAG,YAAY,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,cAAc;YAAE,SAAS;QACjD,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,SAAS;QACxB,IAAI,CAAC,CAAC,UAAU,EAAE,OAAO,KAAK,IAAI;YAAE,SAAS;QAC7C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,mEAAmE;YACnE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,oBAAoB,CAAC,CAAC,CAAC,cAAc,EAAE,KAAK,CAAC,KAAK,KAAK;YAAE,OAAO,IAAI,CAAC;IAC3E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-06-
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-06-25T01:38:40.089Z",
|
|
5
|
+
"instarVersion": "1.3.659",
|
|
6
6
|
"entryCount": 202,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -1538,7 +1538,7 @@
|
|
|
1538
1538
|
"type": "subsystem",
|
|
1539
1539
|
"domain": "sessions",
|
|
1540
1540
|
"sourcePath": "src/core/SessionManager.ts",
|
|
1541
|
-
"contentHash": "
|
|
1541
|
+
"contentHash": "7ea591c3a48b752c066a2fdf0179804a6319177b3664bde1bcc20d9fc0a0d508",
|
|
1542
1542
|
"since": "2025-01-01"
|
|
1543
1543
|
},
|
|
1544
1544
|
"subsystem:auto-updater": {
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
The SessionManager idle-monitor used to hand a session to rate-limit recovery the instant it saw a throttle line in the last ~30 terminal lines — a single glance. That false-fires when the throttle line is stale scrollback the session has moved past, or a transient throttle that already cleared (the milder cousin of the false-rate-limit spam fixed in #1262). Behind a DARK flag (`monitoring.idleThrottleSettleGate`, dev-agent live / dark on the fleet), the idle-monitor now gates that hand-off behind the SAME settle discipline the SessionWatchdog already uses: a throttle must be present AND the terminal pane byte-identical across polls (a working session animates its spinner every tick, so a frozen pane proves the turn genuinely ended on the throttle). It is strictly more conservative — it can only ever trigger recovery LESS often, never more, so it cannot strand a genuinely-stuck session.
|
|
9
|
+
|
|
10
|
+
## What to Tell Your User
|
|
11
|
+
|
|
12
|
+
Nothing visible day-to-day — this is internal session-watching behavior and it's off everywhere except the development machine until it's soaked. The eventual benefit is fewer unnecessary "back online" pokes from a session that wasn't really stuck.
|
|
13
|
+
|
|
14
|
+
## Summary of New Capabilities
|
|
15
|
+
|
|
16
|
+
- `monitoring.idleThrottleSettleGate` (dev-gated dark flag): settle-gate the idle-monitor's rate-limit recovery hand-off, bringing it to parity with the SessionWatchdog's settle discipline.
|
|
17
|
+
- `nextIdleThrottleAction` (pure helper): the unit-testable settle decision (`emit`/`wait`/`fall-through`) the idle path consults each tick.
|
|
18
|
+
- Deferred (CMT-1785): unifying the idle-monitor and watchdog detection paths (one shared trigger + one shared per-tick capture) + distinguishing an active-tail throttle from old scrollback.
|
|
19
|
+
|
|
20
|
+
## Evidence
|
|
21
|
+
|
|
22
|
+
- **Reproduction (the gap):** the idle-monitor's `rateLimitedAtIdle` emit fired on `detectRateLimited(recentOutput)` from a single capture, with no settle check — unlike the SessionWatchdog's `checkRateLimited`, which already required a settled (pane-unchanged-across-polls) throttle before acting.
|
|
23
|
+
- **Before → after:** before, a still-running idle session with a stale/transient throttle line in its buffer got an unnecessary recovery hand-off (≤1 stray "back online" message, after #1262 neutralized the finished-session spam). After, the idle-monitor re-samples the pane every idle tick and only hands off once the throttle has genuinely settled; a stale or already-cleared throttle line no longer triggers recovery. Flag off (the fleet) is byte-identical to the legacy behavior.
|
|
24
|
+
- **Verified:** 6 unit tests for the pure settle decision (`nextIdleThrottleAction`, every boundary), 130 green across the rate-limit/watchdog suites, no regression, clean typecheck. Multi-angle review (3 internal lenses + codex + gemini, 2 rounds) caught a critical defect in an earlier draft (the settle check running only on the first idle tick, making "settled" unreachable) — fixed to re-sample every tick and verified.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
A new pure-signal monitoring sentinel, `StrandedTopicSentinel`, ships behind a DARK dev-gate (`monitoring.strandedTopicSentinel.enabled`, omitted from ConfigDefaults ⇒ live on a development agent, dark on the fleet). It closes a real, proven gap: a Telegram/Slack topic's durable ownership record can name a machine that is **online-by-heartbeat but cannot serve** — quota-walled (`quotaState.blocked`) or adapter-disconnected (`servesChannels` omits the topic's channel) — while a healthy machine holds the lease. Inbound for that topic routes to the owner that can't answer, so it is **silently dead** (outbound still flows from the healthy machine — the "my replies send but the user's messages never arrive" split). The existing `OwnershipReconciler` only force-claims a *provably-DEAD* owner (offline ≥180s) and only iterates *pinned* topics, so a walled-but-online owner defers forever and an unpinned strand is never even considered.
|
|
9
|
+
|
|
10
|
+
The sentinel scans the in-memory ownership cache against the replicated machine-pool view each tick (default 60s), applies a fail-closed predicate with a persistence gate (the unable-to-serve condition must hold ≥2 rich beats over ≥`dwellMs`, default 30s, so a transient quota/adapter blip can't trip it), and raises ONE aggregated `agent-health` attention item per (owner-machine, stranding-window) — never one-per-topic — plus a separate LOW "can't-assess" item if a heartbeat/schema regression blinds it. It is **lease-holder-only** (so peers don't double-report), a **strict no-op on a single-machine agent**, registers in the GuardRegistry (`GET /guards`), and **MUTATES NOTHING** — no ownership CAS, no pin write, no session kill.
|
|
11
|
+
|
|
12
|
+
This is the deliberately-safe HALF of the fix. Spec-convergence review (3 rounds) established that the *instinctive* auto-failover is unsafe with today's primitives — there is no per-topic remote-liveness signal, the reachability signal is self-reported and up to `failoverThresholdMs` stale, and there's no hysteresis — so a wrong failover would yank a live conversation mid-reply, which is strictly worse than the bug. The auto-failover is therefore deferred to a tracked v2 (`CMT-1786`) whose prerequisites are each named in the spec.
|
|
13
|
+
|
|
14
|
+
## What to Tell Your User
|
|
15
|
+
|
|
16
|
+
Nothing visible day-to-day — it's off everywhere except the development machine until it's soaked. The eventual benefit: a conversation whose messages are silently going to a machine that can't answer them gets caught within about a minute, instead of staying invisible until you notice you're being ignored.
|
|
17
|
+
|
|
18
|
+
## Summary of New Capabilities
|
|
19
|
+
|
|
20
|
+
- `monitoring.strandedTopicSentinel.enabled` (dev-gated dark flag): a pure-signal sentinel that detects an online-but-unable-to-serve owner stranding a topic's inbound and raises one aggregated `agent-health` attention item; registered on `GET /guards`.
|
|
21
|
+
- `evaluateStrandedTopics(...)` (pure helper): the unit-testable, fail-closed strand decision (quota arm + best-effort adapter arm + dwell persistence + reconciliation).
|
|
22
|
+
- Deferred (CMT-1786): the auto-failover that actually re-points a stranded topic to a healthy server, plus its named prerequisites (per-topic remote liveness signal, hysteresis/cooldown, claim-time re-assertion, atomic CAS+pin, reason-stamped nonce, OwnershipReconciler unification) and a live userbot harness for real-Telegram UX regression testing.
|
|
23
|
+
|
|
24
|
+
## Evidence
|
|
25
|
+
|
|
26
|
+
- **Reproduction (the live incident, 2026-06-24):** 17 of 25 topics were owned by an offline/quota-walled Mac Mini; every one had silently-dead Telegram inbound; every code review and CI gate was green the whole time; the wedge surfaced only when the operator reported missing messages. `GET /pool/placement?topic=N` showed `owner=Mini, pendingReplacement:true` — the signal was present and unmonitored. The only recovery was a human hand-editing `.instar/ownership/local/<topicId>.json`.
|
|
27
|
+
- **Before → after:** before, this class of breakage was invisible to all automated checks until a user got bitten. After (dev-gated), the moment a topic's owner is persistently online-but-unable-to-serve, ONE attention item fires within ~`dwellMs + tickMs` (≈ a minute or two on a healthy heartbeat cadence) naming the stranded topics, the walled owner + reason, whether a servable peer exists, and the signal's staleness. Flag off (the fleet) is byte-identical to today (no sentinel).
|
|
28
|
+
- **Safety verified:** the sentinel is pure signal — it raises an attention item and writes nothing. The fail-closed predicate (missing field / stale beat / underivable scope / pool view unavailable ⇒ SKIP) means an uncertainty can never manufacture a strand; the lease-holder-sole-actor + single-machine no-op prevent duplicate items; the aggregated item rides the existing `AttentionTopicGuard` flood ceiling.
|
|
29
|
+
- **Tests:** 29 unit tests on the pure decision core (both sides of every boundary — quota arm, three-valued adapter arm, persistence/dwell, fail-closed skips, `strandedSince` reconciliation, can't-assess counting, servable-peer, single-machine/non-lease-holder no-op, sync/LLM-free invariant) + GuardRegistry registration + a `/guards`-posture integration check. `tsc --noEmit` clean. Multi-angle spec review (6 internal reviewers + GPT-5.5 cross-model, 3 rounds) drove the design from an unsafe auto-failover to this safe detector and caught a critical predicate bug (the three-valued `machineServesChannel` would have made a `!fn(...)` detector never fire).
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Side-Effects Review — Idle-monitor throttle settle-gate
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `idle-throttle-settle-gate`
|
|
4
|
+
**Date:** `2026-06-24`
|
|
5
|
+
**Author:** Echo (autonomous, 8-hour run)
|
|
6
|
+
**Spec:** `docs/specs/idle-throttle-settle-gate.md` (review-convergence + approved)
|
|
7
|
+
**Second-pass reviewer:** REQUIRED (touches the SessionManager idle/recovery path) — verdict appended below.
|
|
8
|
+
|
|
9
|
+
## Summary of the change
|
|
10
|
+
|
|
11
|
+
The safe slice of the false-rate-limit F3 residual (CMT-1785). The SessionManager idle-monitor emitted `rateLimitedAtIdle` (handing to RateLimitSentinel recovery) the instant `detectRateLimited(last-30-lines)` matched — a single glance, which false-fires on a stale/transient throttle line. Behind a DARK flag, the idle-monitor now gates that hand-off behind the SAME settle discipline the SessionWatchdog already uses (`evaluateThrottleSettle`: throttle present AND pane byte-identical across polls = the turn genuinely ended on the throttle). Strictly more conservative — it can only ever emit `rateLimitedAtIdle` LESS often, never more.
|
|
12
|
+
|
|
13
|
+
Files modified:
|
|
14
|
+
- `src/monitoring/rateLimitDetection.ts` — new pure `nextIdleThrottleAction` (a thin wrapper over the existing `evaluateThrottleSettle`; returns `'emit'`/`'wait'`/`'fall-through'`). No change to `evaluateThrottleSettle` or `detectRateLimited`.
|
|
15
|
+
- `src/core/SessionManager.ts` — (a) new dark-flagged per-tick settle check inside `if (isActuallyIdle)` AHEAD of the first-idle-tick gate, so the pane is re-sampled every tick (load-bearing — see F1 below); (b) the first-tick legacy emit fenced to `detectRateLimited(recentOutput) && !this.idleThrottleSettleGate` (flag-off only); (c) per-session `idleThrottleSettle` Map + `idleThrottleSettleGate` field from new opt; (d) unconditional cleanup of the Map on `sessionComplete` (constructor) + on session-active.
|
|
16
|
+
- `src/core/devGatedFeatures.ts` — new `idleThrottleSettleGate` DEV_GATED_FEATURES entry (`monitoring.idleThrottleSettleGate.enabled`, omitted ⇒ dev-live/dark-fleet).
|
|
17
|
+
- `src/core/types.ts` — `MonitoringConfig.idleThrottleSettleGate?: { enabled?: boolean }`.
|
|
18
|
+
- `src/commands/server.ts` — resolves the flag via `resolveDevAgentGate(...)` and threads it into the SessionManager opts.
|
|
19
|
+
|
|
20
|
+
Files added:
|
|
21
|
+
- `tests/unit/idle-throttle-settle-gate.test.ts` — 6 unit tests for `nextIdleThrottleAction` (every decision boundary: no-throttle→fall-through, first-sighting→wait, unchanged≥settleMs→emit, unchanged<settleMs→wait, pane-changed→wait+restart, transient-clears→fall-through).
|
|
22
|
+
- `docs/specs/idle-throttle-settle-gate.md` (+ `.eli16.md`, convergence report).
|
|
23
|
+
|
|
24
|
+
## Blast radius
|
|
25
|
+
|
|
26
|
+
- **Flag OFF (the FLEET, by default):** byte-identical to legacy. The per-tick block is skipped entirely (`if (this.idleThrottleSettleGate)` false); the first-tick emit fires exactly as before; the `idleThrottleSettle` Map is never written (cleanup deletes are no-ops on an empty map). Confirmed by the adversarial reviewer + the existing watcher/quota suites unchanged.
|
|
27
|
+
- **Flag ON (dev only):** the idle-path rate-limit hand-off is settle-gated. It only WITHHOLDS a spurious `rateLimitedAtIdle`; it never adds a kill/send/authority. RateLimitSentinel stays the recovery authority. The SessionWatchdog independently settle-gates the same class (the two emits are deduped by `RateLimitSentinel.report()`), so no double-recovery and no coverage gap.
|
|
28
|
+
- **Multi-machine:** machine-local-by-design — per-process state about locally-running sessions; no replicated/proxied state.
|
|
29
|
+
- **Migration parity:** dev-gated flag, `enabled` omitted from ConfigDefaults (no user default written) ⇒ no migration surface. No hooks/CLAUDE.md/skill/dashboard changes.
|
|
30
|
+
|
|
31
|
+
## Rollback
|
|
32
|
+
|
|
33
|
+
Flip `monitoring.idleThrottleSettleGate.enabled` false (or revert the commit) ⇒ legacy immediate emit. Fully additive and reversible; nothing irreversible ships.
|
|
34
|
+
|
|
35
|
+
## Second-pass reviewer verdict
|
|
36
|
+
|
|
37
|
+
Multi-angle spec-converge (adversarial + decision-completeness + lessons/integration internal lenses, + codex-cli:gpt-5.5 + gemini-2.5-pro external) over 2 rounds. Round 1 caught a CRITICAL functional defect (F1: the settle check ran only on the first idle tick → `'settled'` unreachable → flag-ON would emit recovery never, silently delegating to the watchdog) and a LOW map-cleanup leak (F2). Both fixed (per-tick re-sample; unconditional constructor cleanup) and verified RESOLVED + CONVERGED in round 2, with new adversarial checks (idle-kill starvation, flag-off regression, decision boundaries) clean. External minors (overstated safety claim; polling redundancy) folded into the spec. 130 unit tests green, no regression, clean typecheck. The one non-blocking note (per-tick wider capture cost when flag ON) is documented + deferred to the CMT-1785 unification. Verdict: APPROVED.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# Side-Effects Review — StrandedTopicSentinel (online-but-unable-to-serve inbound detector)
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `stranded-inbound-self-heal`
|
|
4
|
+
**Date:** `2026-06-24`
|
|
5
|
+
**Author:** Echo (autonomous)
|
|
6
|
+
**Second-pass reviewer:** not-required (Tier 2; dev-gated, pure-signal monitoring sentinel — no mutation, no fleet runtime path, no operator surface)
|
|
7
|
+
|
|
8
|
+
## Summary of the change
|
|
9
|
+
|
|
10
|
+
Adds a new dark-gated, pure-signal monitoring sentinel, `StrandedTopicSentinel`, that detects a Telegram/Slack topic whose durable ownership record names a machine that is **online-by-heartbeat but cannot serve** (quota-walled `quotaState.blocked`, or adapter-disconnected `servesChannels` omits the topic's channel) while a healthy machine holds the lease — so inbound for that topic is silently dead. It raises ONE aggregated `agent-health` attention item per (owner-machine, stranding-window) and a separate LOW "can't-assess" item if a heartbeat/schema regression blinds it. It **MUTATES NOTHING** (no ownership CAS, no pin write, no session kill); its sole output is an advisory attention item. The instinctive auto-failover was deferred to a tracked v2 (CMT-1786) because spec-convergence review proved it unsafe with today's primitives (no per-topic remote liveness, self-reported stale reachability, no hysteresis) — a wrong failover drops a live conversation, strictly worse than the bug.
|
|
11
|
+
|
|
12
|
+
Files added:
|
|
13
|
+
- `src/monitoring/strandedTopicDecision.ts` — the PURE, unit-testable decision core (`evaluateStrandedTopics`): fail-closed predicate (quota arm + best-effort adapter arm), dwell persistence, `strandedSince` reconciliation, `servablePeerExists`.
|
|
14
|
+
- `src/monitoring/StrandedTopicSentinel.ts` — the sentinel (tick loop, per-owner stranding-window discipline, attention emission, `guardStatus`).
|
|
15
|
+
- `tests/unit/stranded-topic-sentinel.test.ts` (29), `tests/integration/stranded-topic-guard-posture.test.ts` (4), `tests/e2e/stranded-topic-guards-lifecycle.test.ts` (2).
|
|
16
|
+
- `docs/specs/stranded-inbound-self-heal.md` (+ `.eli16.md` + convergence report), `upgrades/next/stranded-inbound-self-heal.md`.
|
|
17
|
+
|
|
18
|
+
Files modified:
|
|
19
|
+
- `src/core/types.ts` — `MonitoringConfig.strandedTopicSentinel?` config type.
|
|
20
|
+
- `src/config/ConfigDefaults.ts` — the `strandedTopicSentinel` defaults block (tickMs/dwellMs/freshnessBoundMs/clearAfterTicks); `enabled` OMITTED (dev-gate decides).
|
|
21
|
+
- `src/core/devGatedFeatures.ts` — `DEV_GATED_FEATURES` entry (pure-signal justification).
|
|
22
|
+
- `src/monitoring/guardManifest.ts` — `GUARD_MANIFEST` entry (`expectRuntime`, `expectedTickMs`).
|
|
23
|
+
- `src/core/SessionOwnershipRegistry.ts` — added `all()` (delegates to `store.all?.()`; reads empty for a store lacking it) so the registry the server holds can be scanned.
|
|
24
|
+
- `src/commands/server.ts` — wires the sentinel (late, after pool boot; `selfMachineId` via a lazy getter to avoid the #1190 boot-ordering null-capture; `raiseAttention` → `telegram.createAttentionItem`; GuardRegistry registration).
|
|
25
|
+
- `tests/unit/lint-dev-agent-dark-gate.test.ts` — hand-shifted the golden line-map keys +14 (my ConfigDefaults block added 14 lines, NO new `enabled:` literal).
|
|
26
|
+
|
|
27
|
+
## Decision-point inventory
|
|
28
|
+
|
|
29
|
+
- **Added**: the strand predicate (`evaluateStrandedTopics`) — a READ-ONLY verdict, not a gate. It decides whether to RAISE an attention item; it authorizes no mutation. Fail-closed on every uncertainty.
|
|
30
|
+
- **Added**: the lease-holder-sole-actor + single-machine early-no-op gates — decide WHETHER THIS MACHINE evaluates at all (so peers don't double-report). Observe-only.
|
|
31
|
+
- No agent-to-user or ownership mutation decision point is added. This is pure-signal monitoring.
|
|
32
|
+
|
|
33
|
+
## 1. Over-block
|
|
34
|
+
|
|
35
|
+
None — the change blocks nothing. It is observe-only. The only "false positive" risk is a spurious attention item, mitigated by: ≥2-rich-beat dwell persistence (kills transient blips), missing-field/stale-beat/underivable-scope ⇒ SKIP (fail-closed — an uncertainty can never manufacture a strand), and lease-holder-only emission (no duplicate items across machines). A spurious item is cheap and rides the existing `AttentionTopicGuard` flood ceiling.
|
|
36
|
+
|
|
37
|
+
## 2. Under-block
|
|
38
|
+
|
|
39
|
+
By design, v1 only DETECTS — it does not remediate; the operator/agent acts on the alert (manual remediation documented in the spec; auto-failover is the tracked v2). The adapter arm is best-effort: for Telegram the channel scope is shared adapter config so the adapter arm rarely fires (the quota arm carries the Telegram case — the actual incident); the adapter arm's real value is Slack per-workspace. A dead (offline) owner is intentionally NOT covered (that is the existing OwnershipReconciler Case C's job) — this sentinel covers only the online-but-unable gap.
|
|
40
|
+
|
|
41
|
+
## 3. Level-of-abstraction fit
|
|
42
|
+
|
|
43
|
+
Right layer: a `src/monitoring/` sentinel, mirroring the established pattern (ContextWedgeSentinel/ResumeQueue) — deps-injected, `lastTickAt` liveness, GuardRegistry registration, dark-gate via DEV_GATED_FEATURES. The pure decision is extracted to a separate module so it is unit-testable without tmux/HTTP.
|
|
44
|
+
|
|
45
|
+
## 4. Signal vs authority compliance
|
|
46
|
+
|
|
47
|
+
**Required reference:** [docs/signal-vs-authority.md](../../docs/signal-vs-authority.md). FULLY compliant — this is the textbook SIGNAL side. The sentinel holds NO authority: it raises an advisory attention item and writes nothing else (no CAS, no pin, no kill, no direct user message). The deliberate review-driven retreat from a mutating design to a pure detector is exactly the standard's prescription (the dangerous mutation is deferred until the primitives that make it safe exist).
|
|
48
|
+
|
|
49
|
+
## 5. Interactions
|
|
50
|
+
|
|
51
|
+
- **OwnershipReconciler (Case C):** disjoint by trigger — Case C covers offline-DEAD+pinned owners; this covers online-but-UNABLE owners (which Case C explicitly defers as `deferredNoEvidence`). No shared writer (this sentinel writes nothing).
|
|
52
|
+
- **AttentionTopicGuard:** the aggregated item rides the existing flood ceiling; NORMAL/LOW priority never bypasses it.
|
|
53
|
+
- **GuardRegistry / GET /guards:** registers so a silently-disabled instance is visible (the exact failure class this feature exists to surface).
|
|
54
|
+
- **Host spawn cap / event loop:** the tick is synchronous, LLM-free, acquires NO spawn-cap slot (asserted by test), and does NO synchronous peer probe — it reads the in-memory ownership cache + replicated heartbeat view only.
|
|
55
|
+
|
|
56
|
+
## 6. External surfaces
|
|
57
|
+
|
|
58
|
+
The only external-visible effect is an `agent-health` attention item (and a LOW can't-assess item) when a strand is detected — calm, aggregated, deduped, flood-ceiling-bounded, with signal-staleness disclosed in the text. No new HTTP route, no dashboard change, no message to a user channel.
|
|
59
|
+
|
|
60
|
+
## 6b. Operator-surface quality (Operator-Surface Quality standard)
|
|
61
|
+
|
|
62
|
+
No operator surface — not applicable. This change touches no dashboard renderer/markup, approval page, or grant/revoke/secret-drop form. Its only operator-facing output is a plain-language attention item whose text leads with the situation ("inbound for topics X is going to <machine>, which can't serve them; <machine> can"), de-emphasizes identifiers, and discloses signal staleness.
|
|
63
|
+
|
|
64
|
+
## 7. Multi-machine posture (Cross-Machine Coherence)
|
|
65
|
+
|
|
66
|
+
**machine-local BY DESIGN.** The detection is per-machine observability over the REPLICATED heartbeat machine-pool view + the local in-memory ownership cache; it writes no durable state, so there is nothing to replicate or strand on a topic transfer. Exactly ONE machine reports per strand: the lease-holder is the sole actor (`syncStatus.holdsLease`), so peers stay observe-only and the user hears one voice. A single-machine agent is a strict no-op (`machines().length < 2` early return — nothing can be stranded on a peer that doesn't exist).
|
|
67
|
+
|
|
68
|
+
## 8. Rollback cost
|
|
69
|
+
|
|
70
|
+
Trivial and instant. The feature ships dark on the fleet (`monitoring.strandedTopicSentinel.enabled` omitted ⇒ resolves dark off a dev agent); flag-off is byte-identical to today (no sentinel constructed). On a dev agent, set `monitoring.strandedTopicSentinel.enabled: false` to force-dark. No durable state is written, so there is nothing to clean up on back-out.
|
|
71
|
+
|
|
72
|
+
## Conclusion
|
|
73
|
+
|
|
74
|
+
A safe, dark-gated, pure-signal detector that makes a real, proven, previously-invisible cross-machine inbound wedge loud within a bounded window — the deliberately-safe half of the fix, with the dangerous auto-failover deferred behind named, tracked prerequisites (CMT-1786). All three test tiers green; the design passed 3 spec-convergence rounds (6 internal reviewers + GPT-5.5 cross-model).
|
|
75
|
+
|
|
76
|
+
## Evidence pointers
|
|
77
|
+
|
|
78
|
+
- Spec + convergence report: `docs/specs/stranded-inbound-self-heal.md`, `docs/specs/reports/stranded-inbound-self-heal-convergence.md`.
|
|
79
|
+
- Live incident (the reproduction): 2026-06-24, 17 of 25 topics stranded on a quota-walled Mac Mini; inbound silently dead; surfaced only when the operator reported missing messages.
|
|
80
|
+
- Tests: unit (29) `tests/unit/stranded-topic-sentinel.test.ts`; integration (4) `tests/integration/stranded-topic-guard-posture.test.ts`; e2e (2) `tests/e2e/stranded-topic-guards-lifecycle.test.ts`.
|