instar 1.3.606 → 1.3.608

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/commands/server.d.ts.map +1 -1
  2. package/dist/commands/server.js +333 -32
  3. package/dist/commands/server.js.map +1 -1
  4. package/dist/config/ConfigDefaults.d.ts.map +1 -1
  5. package/dist/config/ConfigDefaults.js +38 -0
  6. package/dist/config/ConfigDefaults.js.map +1 -1
  7. package/dist/coordination/CrossMachineMandate.d.ts +53 -0
  8. package/dist/coordination/CrossMachineMandate.d.ts.map +1 -0
  9. package/dist/coordination/CrossMachineMandate.js +76 -0
  10. package/dist/coordination/CrossMachineMandate.js.map +1 -0
  11. package/dist/core/AccountCredentialShare.d.ts +115 -0
  12. package/dist/core/AccountCredentialShare.d.ts.map +1 -0
  13. package/dist/core/AccountCredentialShare.js +113 -0
  14. package/dist/core/AccountCredentialShare.js.map +1 -0
  15. package/dist/core/AccountFollowMeGrants.d.ts +92 -0
  16. package/dist/core/AccountFollowMeGrants.d.ts.map +1 -0
  17. package/dist/core/AccountFollowMeGrants.js +106 -0
  18. package/dist/core/AccountFollowMeGrants.js.map +1 -0
  19. package/dist/core/CoherenceJournal.d.ts +1 -1
  20. package/dist/core/CoherenceJournal.d.ts.map +1 -1
  21. package/dist/core/CoherenceJournal.js +12 -2
  22. package/dist/core/CoherenceJournal.js.map +1 -1
  23. package/dist/core/PairingEpochManager.d.ts +68 -0
  24. package/dist/core/PairingEpochManager.d.ts.map +1 -0
  25. package/dist/core/PairingEpochManager.js +88 -0
  26. package/dist/core/PairingEpochManager.js.map +1 -0
  27. package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
  28. package/dist/core/PostUpdateMigrator.js +38 -0
  29. package/dist/core/PostUpdateMigrator.js.map +1 -1
  30. package/dist/core/SecretStore.d.ts +44 -0
  31. package/dist/core/SecretStore.d.ts.map +1 -1
  32. package/dist/core/SecretStore.js +126 -0
  33. package/dist/core/SecretStore.js.map +1 -1
  34. package/dist/core/SecretSync.d.ts.map +1 -1
  35. package/dist/core/SecretSync.js +6 -0
  36. package/dist/core/SecretSync.js.map +1 -1
  37. package/dist/core/SubscriptionAccountMetaReplicatedStore.d.ts +106 -0
  38. package/dist/core/SubscriptionAccountMetaReplicatedStore.d.ts.map +1 -0
  39. package/dist/core/SubscriptionAccountMetaReplicatedStore.js +263 -0
  40. package/dist/core/SubscriptionAccountMetaReplicatedStore.js.map +1 -0
  41. package/dist/core/SubscriptionPool.d.ts +23 -5
  42. package/dist/core/SubscriptionPool.d.ts.map +1 -1
  43. package/dist/core/SubscriptionPool.js +30 -6
  44. package/dist/core/SubscriptionPool.js.map +1 -1
  45. package/dist/core/devGatedFeatures.d.ts.map +1 -1
  46. package/dist/core/devGatedFeatures.js +12 -0
  47. package/dist/core/devGatedFeatures.js.map +1 -1
  48. package/dist/core/types.d.ts +37 -0
  49. package/dist/core/types.d.ts.map +1 -1
  50. package/dist/core/types.js.map +1 -1
  51. package/dist/monitoring/AutonomousLivenessReconciler.d.ts +278 -0
  52. package/dist/monitoring/AutonomousLivenessReconciler.d.ts.map +1 -0
  53. package/dist/monitoring/AutonomousLivenessReconciler.js +660 -0
  54. package/dist/monitoring/AutonomousLivenessReconciler.js.map +1 -0
  55. package/dist/monitoring/ResumeQueue.d.ts +10 -0
  56. package/dist/monitoring/ResumeQueue.d.ts.map +1 -1
  57. package/dist/monitoring/ResumeQueue.js +12 -0
  58. package/dist/monitoring/ResumeQueue.js.map +1 -1
  59. package/dist/scaffold/templates.d.ts.map +1 -1
  60. package/dist/scaffold/templates.js +7 -0
  61. package/dist/scaffold/templates.js.map +1 -1
  62. package/dist/server/AgentServer.d.ts +1 -0
  63. package/dist/server/AgentServer.d.ts.map +1 -1
  64. package/dist/server/AgentServer.js +6 -0
  65. package/dist/server/AgentServer.js.map +1 -1
  66. package/dist/server/routes.d.ts +4 -0
  67. package/dist/server/routes.d.ts.map +1 -1
  68. package/dist/server/routes.js +10 -0
  69. package/dist/server/routes.js.map +1 -1
  70. package/package.json +1 -1
  71. package/src/data/builtin-manifest.json +64 -64
  72. package/src/scaffold/templates.ts +7 -0
  73. package/upgrades/1.3.607.md +25 -0
  74. package/upgrades/1.3.608.md +29 -0
  75. package/upgrades/side-effects/autonomous-liveness-reconciler.md +39 -0
  76. package/upgrades/side-effects/ws52-account-follow-me-pr1.md +74 -0
@@ -0,0 +1,660 @@
1
+ /**
2
+ * AutonomousLivenessReconciler — a level-triggered control loop that self-heals
3
+ * an autonomous run whose state file says ACTIVE (with time remaining) but has
4
+ * NO live session executing it.
5
+ *
6
+ * THE INCIDENT (2026-06-16): an autonomous run on topic 12476 was reaped under
7
+ * `age-limit` (the periodic per-session lifetime recycle). The recycle is meant
8
+ * to be invisible — the run should revive and continue — but the reap was tagged
9
+ * `midWork:false`, so it was never offered to the resume queue, and the run
10
+ * silently died. Its state file still said active with ~15h remaining; no tmux
11
+ * session was running it; nothing watched for the contradiction. When the user
12
+ * messaged 1h+ later there was no live session to answer.
13
+ *
14
+ * THE FIX (this module): don't perfect the edge — add a control loop that
15
+ * continuously compares desired state (run active+remaining) vs actual state
16
+ * (a live session exists) and converges. It self-heals regardless of HOW the
17
+ * run got orphaned. It is the liveness twin of the progress heartbeat: a
18
+ * heartbeat covers "alive but quiet"; this covers "dead but marked active".
19
+ *
20
+ * SIGNAL-ONLY / SAFE DIRECTION: in dryRun it only LOGS "would respawn" (plus a
21
+ * shadow "would-have-capped"). Live, its worst action is a bounded, capped,
22
+ * lease-gated, quota-gated, pressure-gated respawn of a run that genuinely
23
+ * should be alive (a false respawn wastes a few tokens; a missed one is the
24
+ * silent death above). It never blocks, delays, or rewrites anything. Ships dark
25
+ * on the fleet (dev-agent gate), dryRun-first on dev.
26
+ *
27
+ * THE RUN-STATE FILE IS UNTRUSTED. It is read ONLY for the binary
28
+ * "active + remaining" decision, the topicId, and the generation key
29
+ * (started_at). cwd / resumeUuid / spawn target are NEVER sourced from it — they
30
+ * come from authoritative registries (see §Action in the spec).
31
+ *
32
+ * Spec: docs/specs/autonomous-liveness-reconciler.md
33
+ */
34
+ const DEFAULTS = {
35
+ tickIntervalSec: 120,
36
+ debounceTicks: 2,
37
+ debounceWindowSec: 180,
38
+ respawnTimeoutMs: 45_000,
39
+ respawnCapPerWindow: 3,
40
+ respawnCapWindowSec: 21_600, // 6h
41
+ spawnFailureRetryCeiling: 6,
42
+ maxPressureBlockedTicks: 10,
43
+ maxPressureBlockedSec: 1_800, // 30m
44
+ allowFreshFallback: false,
45
+ notifyUser: true,
46
+ };
47
+ export class AutonomousLivenessReconciler {
48
+ deps;
49
+ cfg;
50
+ timer = null;
51
+ /** Per-topic debounce observation state. */
52
+ observed = new Map();
53
+ /** Per-topic explicit condition record. */
54
+ conditions = new Map();
55
+ /** Redie counter: respawn-that-restarted-then-redied timestamps (ms). */
56
+ redie = new Map();
57
+ /** Spawn-failure counter: respawn-that-threw-before-any-session timestamps (ms). */
58
+ spawnFailure = new Map();
59
+ /** Topics that have hit the redie cap and been surfaced (so we don't re-raise every tick). */
60
+ cappedSurfaced = new Set();
61
+ /** Per-topic dedupe guards for the other raise-to-attention paths (P17). */
62
+ spawnFailSurfaced = new Set();
63
+ ambiguousSurfaced = new Set();
64
+ missingUuidSurfaced = new Set();
65
+ pressureSurfaced = new Set();
66
+ lastTickAt = null;
67
+ respawnTotal = 0;
68
+ /** Effective inflight TTL (cfg.inflightSpawnTtlMs ?? respawnTimeoutMs + grace). */
69
+ inflightTtlMs;
70
+ constructor(deps, config = {}) {
71
+ this.deps = deps;
72
+ const respawnTimeoutMs = config.respawnTimeoutMs ?? DEFAULTS.respawnTimeoutMs;
73
+ this.cfg = {
74
+ enabled: config.enabled,
75
+ dryRun: config.dryRun ?? true,
76
+ tickIntervalSec: config.tickIntervalSec ?? DEFAULTS.tickIntervalSec,
77
+ debounceTicks: config.debounceTicks ?? DEFAULTS.debounceTicks,
78
+ debounceWindowSec: config.debounceWindowSec ?? DEFAULTS.debounceWindowSec,
79
+ respawnTimeoutMs,
80
+ respawnCapPerWindow: config.respawnCapPerWindow ?? DEFAULTS.respawnCapPerWindow,
81
+ respawnCapWindowSec: config.respawnCapWindowSec ?? DEFAULTS.respawnCapWindowSec,
82
+ spawnFailureRetryCeiling: config.spawnFailureRetryCeiling ?? DEFAULTS.spawnFailureRetryCeiling,
83
+ maxPressureBlockedTicks: config.maxPressureBlockedTicks ?? DEFAULTS.maxPressureBlockedTicks,
84
+ maxPressureBlockedSec: config.maxPressureBlockedSec ?? DEFAULTS.maxPressureBlockedSec,
85
+ allowFreshFallback: config.allowFreshFallback ?? DEFAULTS.allowFreshFallback,
86
+ inflightSpawnTtlMs: config.inflightSpawnTtlMs ?? respawnTimeoutMs + 30_000,
87
+ notifyUser: config.notifyUser ?? DEFAULTS.notifyUser,
88
+ };
89
+ this.inflightTtlMs = this.cfg.inflightSpawnTtlMs;
90
+ // Restore durable cap state if provided (so a crash-loop across restarts stays bounded).
91
+ const restored = this.deps.loadCapState?.();
92
+ if (restored) {
93
+ this.redie = new Map(Object.entries(restored.redie ?? {}).map(([k, v]) => [Number(k), v]));
94
+ this.spawnFailure = new Map(Object.entries(restored.spawnFailure ?? {}).map(([k, v]) => [Number(k), v]));
95
+ }
96
+ }
97
+ start() {
98
+ if (this.cfg.enabled === false)
99
+ return;
100
+ if (this.timer)
101
+ return;
102
+ this.timer = setInterval(() => {
103
+ void this.tick().catch(() => {
104
+ /* @silent-fallback-ok — a tick fault must never crash the server; the
105
+ next tick retries from fresh state (the loop is level-triggered, so a
106
+ dropped tick loses nothing — the gap is reconverged next pass). */
107
+ });
108
+ }, this.cfg.tickIntervalSec * 1000);
109
+ // Don't keep the event loop alive on this timer alone.
110
+ this.timer.unref?.();
111
+ }
112
+ stop() {
113
+ if (this.timer) {
114
+ clearInterval(this.timer);
115
+ this.timer = null;
116
+ }
117
+ }
118
+ /**
119
+ * One reconcile pass. All I/O goes through deps. Idempotent — a run already
120
+ * live / queued / not-owned / paused is a no-op. At most ONE respawn per tick.
121
+ */
122
+ async tick() {
123
+ const now = this.deps.now();
124
+ this.lastTickAt = now;
125
+ let runs;
126
+ try {
127
+ runs = this.deps.listActiveRuns();
128
+ }
129
+ catch {
130
+ // @silent-fallback-ok — can't read the run-state files → reconcile nothing
131
+ // this tick (status-quo safe direction; the next tick retries). A read
132
+ // error must never be interpreted as "no active runs to protect".
133
+ return;
134
+ }
135
+ // Hoist the once-per-tick snapshots (criterion 6, pressure gate).
136
+ const liveTopics = this.safe(() => this.deps.liveTopicSnapshot(), new Set());
137
+ const pressure = this.safe(() => this.deps.pressureTier(), 'critical');
138
+ const queuePaused = this.safeBool(() => this.deps.queuePaused(), false);
139
+ const candidateTopics = new Set();
140
+ let actedThisTick = false;
141
+ for (const run of runs) {
142
+ const topicId = run.topicId;
143
+ // ── Candidate criteria (each fails toward NOT-a-candidate — the safe side) ──
144
+ // Global gate: an emergency-paused queue means "halt all automation".
145
+ if (queuePaused) {
146
+ this.observed.delete(topicId);
147
+ continue;
148
+ }
149
+ // Criterion 1: active + remaining + CURRENT generation (obsolete run rejected).
150
+ if (run.remainingSeconds <= 0) {
151
+ this.observed.delete(topicId);
152
+ continue;
153
+ }
154
+ if (!this.isCurrentGeneration(run)) {
155
+ this.setCondition(topicId, 'healthy', now); // not ours to act on
156
+ this.observed.delete(topicId);
157
+ continue;
158
+ }
159
+ // Criterion 2: not paused.
160
+ if (run.paused) {
161
+ this.observed.delete(topicId);
162
+ continue;
163
+ }
164
+ // Criterion 4: not mid-machine-move.
165
+ if (run.movedTo != null || run.moveSuspended) {
166
+ this.setCondition(topicId, 'mid-move', now);
167
+ this.observed.delete(topicId);
168
+ continue;
169
+ }
170
+ // Criterion 5: THIS machine owns the topic AND holds the lease.
171
+ const ownerElsewhere = this.safeBool(() => this.deps.topicOwnerElsewhere(topicId), true);
172
+ const leaseHeld = this.safeBool(() => this.deps.holdsLease(), false);
173
+ if (ownerElsewhere || !leaseHeld) {
174
+ this.setCondition(topicId, 'blocked-not-owner', now);
175
+ this.observed.delete(topicId);
176
+ continue;
177
+ }
178
+ // Criterion 3: not operator-stopped (bounded to THIS run's start, not epoch-0).
179
+ const sinceIso = new Date(run.startedAtMs ?? 0).toISOString();
180
+ if (this.safeBool(() => this.deps.operatorStoppedSince(topicId, sinceIso), true)) {
181
+ this.observed.delete(topicId);
182
+ continue;
183
+ }
184
+ // Criterion 6: NO live session (own once-per-tick snapshot).
185
+ if (liveTopics.has(topicId)) {
186
+ // Stably live → debounce reset. Record live-since for the stable-live rule.
187
+ const obs = this.observed.get(topicId);
188
+ if (obs) {
189
+ if (obs.liveSinceMs == null)
190
+ obs.liveSinceMs = now;
191
+ // A FULL window of stable liveness zeroes the death evidence.
192
+ if (now - obs.liveSinceMs >= this.cfg.debounceWindowSec * 1000) {
193
+ this.observed.delete(topicId);
194
+ }
195
+ }
196
+ this.setCondition(topicId, 'healthy', now);
197
+ this.clearPressureBlock(topicId);
198
+ continue;
199
+ }
200
+ // Criterion 7: not already being (re)spawned by anyone (queue arm + in-flight arm).
201
+ if (this.safeBool(() => this.deps.topicInResumeQueue(topicId), true)) {
202
+ this.setCondition(topicId, 'blocked-queue-owns', now);
203
+ this.observed.delete(topicId);
204
+ continue;
205
+ }
206
+ if (this.inflightActive(topicId, now)) {
207
+ this.setCondition(topicId, 'blocked-queue-owns', now);
208
+ continue;
209
+ }
210
+ // Criterion 8: binding UNAMBIGUOUS — else needs-attention, never auto-respawn.
211
+ if (!this.safeBool(() => this.deps.bindingUnambiguous(topicId), false)) {
212
+ if (!this.ambiguousSurfaced.has(topicId)) {
213
+ this.ambiguousSurfaced.add(topicId);
214
+ this.deps.raiseAggregated('liveness-ambiguous-binding', `topic ${topicId} looks orphaned but its session↔topic binding is ambiguous — needs your eyes (not auto-respawning).`);
215
+ }
216
+ this.deps.audit({ ts: new Date(now).toISOString(), event: 'ambiguous-binding', topicId });
217
+ this.observed.delete(topicId);
218
+ continue;
219
+ }
220
+ // ── It's a candidate this tick ──
221
+ candidateTopics.add(topicId);
222
+ const obs = this.observed.get(topicId) ?? { firstSeenMs: now, count: 0, liveSinceMs: null };
223
+ obs.count += 1;
224
+ obs.liveSinceMs = null; // not live this tick → reset the stable-live timer
225
+ this.observed.set(topicId, obs);
226
+ const debounceElapsed = now - obs.firstSeenMs >= this.cfg.debounceWindowSec * 1000;
227
+ const debounceMet = obs.count >= this.cfg.debounceTicks && debounceElapsed;
228
+ if (!debounceMet) {
229
+ this.setCondition(topicId, 'debouncing', now);
230
+ this.deps.audit({
231
+ ts: new Date(now).toISOString(),
232
+ event: 'debouncing',
233
+ topicId,
234
+ count: obs.count,
235
+ needed: this.cfg.debounceTicks,
236
+ remainingSeconds: run.remainingSeconds,
237
+ });
238
+ continue;
239
+ }
240
+ // NOTE: do NOT setCondition('orphaned-observing') here — actOn sets the
241
+ // terminal condition for this tick, and a premature overwrite would zero the
242
+ // blocked-pressure tick counter every pass (defeating the bounded skip).
243
+ // At most ONE respawn per tick (anti spawn-storm). Later candidates wait;
244
+ // their debounce state persists. dryRun would-respawn is cheap → still log.
245
+ if (actedThisTick && !this.cfg.dryRun)
246
+ continue;
247
+ const acted = await this.actOn(run, now, pressure);
248
+ if (acted)
249
+ actedThisTick = true;
250
+ }
251
+ // GC: drop debounce/condition/cap entries that are no longer candidates and
252
+ // whose windows have fully expired (neither map grows unbounded).
253
+ this.gc(now, candidateTopics);
254
+ }
255
+ /**
256
+ * Reconcile one candidate. Returns true if an actuation (respawn / would-respawn)
257
+ * happened this tick (counts toward the one-per-tick budget).
258
+ */
259
+ async actOn(run, now, pressure) {
260
+ const topicId = run.topicId;
261
+ // ── Loop brake (P19): redie give-up, UNIFIED with the queue's resurrection count ──
262
+ const queueResurrections = this.cfg.dryRun
263
+ ? 0
264
+ : this.safe(() => this.deps.queueResurrectionCount?.(topicId) ?? 0, 0);
265
+ if (!this.withinRedieCap(topicId, now, queueResurrections)) {
266
+ if (!this.cappedSurfaced.has(topicId)) {
267
+ this.cappedSurfaced.add(topicId);
268
+ this.deps.raiseAggregated('liveness-cap', `autonomous run on topic ${topicId} hit the respawn cap (${this.cfg.respawnCapPerWindow} in ${Math.round(this.cfg.respawnCapWindowSec / 3600)}h) and keeps dying — not auto-respawning again; needs your eyes.`);
269
+ }
270
+ this.setCondition(topicId, 'capped', now);
271
+ this.deps.audit({
272
+ ts: new Date(now).toISOString(),
273
+ event: this.cfg.dryRun ? 'would-have-capped' : 'capped-gaveup',
274
+ topicId,
275
+ dryRun: this.cfg.dryRun,
276
+ redieInWindow: this.freshWindow(this.redie, topicId, now).length,
277
+ queueResurrections,
278
+ });
279
+ this.observed.delete(topicId);
280
+ return false;
281
+ }
282
+ // ── Spawn-failure budget (separate; infra flakiness must not burn the redie brake) ──
283
+ if (this.freshWindow(this.spawnFailure, topicId, now).length >= this.cfg.spawnFailureRetryCeiling) {
284
+ if (!this.spawnFailSurfaced.has(topicId)) {
285
+ this.spawnFailSurfaced.add(topicId);
286
+ this.deps.raiseAggregated('liveness-respawn-failed', `respawn of autonomous run on topic ${topicId} keeps failing before any session exists (${this.cfg.spawnFailureRetryCeiling} attempts) — likely infra; needs your eyes.`);
287
+ }
288
+ this.deps.audit({ ts: new Date(now).toISOString(), event: 'spawn-failure-ceiling', topicId });
289
+ this.observed.delete(topicId);
290
+ return false;
291
+ }
292
+ // ── Anti-reaper-thrash pressure gate — BOUNDED ──
293
+ if (pressure === 'moderate' || pressure === 'critical') {
294
+ const cond = this.conditions.get(topicId);
295
+ const blockedTicks = (cond?.state === 'blocked-pressure' ? cond.pressureBlockedTicks : 0) + 1;
296
+ const blockedSince = cond?.state === 'blocked-pressure' ? (cond.pressureBlockedSinceMs ?? now) : now;
297
+ const overTickBound = blockedTicks > this.cfg.maxPressureBlockedTicks;
298
+ const overTimeBound = now - blockedSince >= this.cfg.maxPressureBlockedSec * 1000;
299
+ if (!overTickBound && !overTimeBound) {
300
+ this.setCondition(topicId, 'blocked-pressure', now, blockedTicks, blockedSince);
301
+ this.deps.audit({
302
+ ts: new Date(now).toISOString(),
303
+ event: 'blocked-pressure',
304
+ topicId,
305
+ tier: pressure,
306
+ blockedTicks,
307
+ });
308
+ return false;
309
+ }
310
+ // Past the bound: a dead run is not load the reaper can shed. Under CRITICAL
311
+ // pressure raise ONE attention item instead of respawning; under moderate, act.
312
+ if (pressure === 'critical') {
313
+ if (!this.pressureSurfaced.has(topicId)) {
314
+ this.pressureSurfaced.add(topicId);
315
+ this.deps.raiseAggregated('liveness-sustained-pressure', `topic ${topicId} is orphaned but the machine is under sustained pressure — needs your eyes.`);
316
+ }
317
+ this.deps.audit({
318
+ ts: new Date(now).toISOString(),
319
+ event: 'pressure-bound-escalated',
320
+ topicId,
321
+ tier: pressure,
322
+ });
323
+ this.clearPressureBlock(topicId);
324
+ return false;
325
+ }
326
+ // moderate + over-bound → fall through to act.
327
+ this.clearPressureBlock(topicId);
328
+ }
329
+ else {
330
+ this.clearPressureBlock(topicId);
331
+ }
332
+ // ── Quota / session-count / migration gates (same as the drainer) ──
333
+ if (!this.safeBool(() => this.deps.quotaOk(), false)) {
334
+ this.setCondition(topicId, 'blocked-quota', now);
335
+ this.deps.audit({ ts: new Date(now).toISOString(), event: 'skipped-quota', topicId });
336
+ return false; // retry next tick when quota recovers; debounce stays satisfied
337
+ }
338
+ if (!this.safeBool(() => this.deps.sessionCountOk(), false)) {
339
+ this.setCondition(topicId, 'blocked-quota', now);
340
+ this.deps.audit({ ts: new Date(now).toISOString(), event: 'skipped-session-cap', topicId });
341
+ return false;
342
+ }
343
+ if (this.safeBool(() => this.deps.migrationInFlight(), true)) {
344
+ this.deps.audit({ ts: new Date(now).toISOString(), event: 'skipped-migration', topicId });
345
+ return false;
346
+ }
347
+ // ── Resolve authoritative respawn inputs (NEVER the untrusted state file) ──
348
+ const resumeUuid = this.safe(() => this.deps.resolveResumeUuid(topicId), null);
349
+ if (resumeUuid == null && !this.cfg.allowFreshFallback) {
350
+ if (!this.missingUuidSurfaced.has(topicId)) {
351
+ this.missingUuidSurfaced.add(topicId);
352
+ this.deps.raiseAggregated('liveness-missing-resume', `can't resume topic ${topicId} — no resume UUID on the canonical resume map; not respawning (would lose prior context). Needs your eyes.`);
353
+ }
354
+ this.deps.audit({ ts: new Date(now).toISOString(), event: 'missing-resume-uuid', topicId });
355
+ this.observed.delete(topicId);
356
+ return false;
357
+ }
358
+ const cwd = this.safe(() => this.deps.resolveCwd(topicId), null);
359
+ if (cwd == null) {
360
+ // Refuse loudly — a cwd that can't be resolved to a safe path (missing
361
+ // binding, or realpath escaping the agent home) must never spawn against a guess.
362
+ this.deps.raiseAggregated('liveness-bad-cwd', `can't resume topic ${topicId} — its working directory can't be safely resolved; not respawning. Needs your eyes.`);
363
+ this.deps.audit({ ts: new Date(now).toISOString(), event: 'unsafe-cwd', topicId });
364
+ this.observed.delete(topicId);
365
+ return false;
366
+ }
367
+ // ── dryRun: log what we WOULD do and actuate nothing ──
368
+ if (this.cfg.dryRun) {
369
+ this.deps.audit({
370
+ ts: new Date(now).toISOString(),
371
+ event: 'would-respawn',
372
+ topicId,
373
+ dryRun: true,
374
+ remainingSeconds: run.remainingSeconds,
375
+ resumeUuid: resumeUuid ? 'present' : 'fresh-fallback',
376
+ });
377
+ // Shadow the cap (spec adversarial F6): record a shadow redie so the
378
+ // reaper-thrash/cap BEHAVIOR — which real dryRun can't trigger (no real
379
+ // respawn ⇒ no re-die) — still becomes observable as a would-have-capped
380
+ // event on dev BEFORE the operator flips live. Reset debounce so the next
381
+ // tick re-observes and the shadow window accrues like the live path.
382
+ this.recordRedie(topicId, now);
383
+ this.observed.delete(topicId);
384
+ return true;
385
+ }
386
+ // ── LIVE actuation: atomic claim → recheck → spawn (bounded) → settle ──
387
+ if (!this.safeBool(() => this.deps.claimInflight(topicId), false)) {
388
+ this.setCondition(topicId, 'blocked-queue-owns', now);
389
+ this.deps.audit({ ts: new Date(now).toISOString(), event: 'claim-lost', topicId });
390
+ return false;
391
+ }
392
+ try {
393
+ // Recheck at the actuation instant (not the per-tick snapshot).
394
+ const liveNow = this.safe(() => this.deps.liveTopicSnapshot(), new Set());
395
+ const stoppedNow = this.safeBool(() => this.deps.operatorStoppedSince(topicId, new Date(run.startedAtMs ?? 0).toISOString()), true);
396
+ const ownerElsewhereNow = this.safeBool(() => this.deps.topicOwnerElsewhere(topicId), true);
397
+ const leaseHeldNow = this.safeBool(() => this.deps.holdsLease(), false);
398
+ if (liveNow.has(topicId) || stoppedNow || ownerElsewhereNow || !leaseHeldNow) {
399
+ this.deps.releaseClaim(topicId);
400
+ this.deps.audit({
401
+ ts: new Date(now).toISOString(),
402
+ event: 'recheck-aborted',
403
+ topicId,
404
+ live: liveNow.has(topicId),
405
+ stopped: stoppedNow,
406
+ ownerElsewhere: ownerElsewhereNow,
407
+ leaseHeld: leaseHeldNow,
408
+ });
409
+ this.observed.delete(topicId);
410
+ return false;
411
+ }
412
+ // SPAWN (bounded by respawnTimeoutMs).
413
+ try {
414
+ await this.withTimeout(this.deps.respawn({ topicId, resumeUuid, cwd }), this.cfg.respawnTimeoutMs);
415
+ }
416
+ catch (err) {
417
+ this.deps.releaseClaim(topicId);
418
+ this.recordSpawnFailure(topicId, now);
419
+ this.deps.audit({
420
+ ts: new Date(now).toISOString(),
421
+ event: 'respawn-failed',
422
+ topicId,
423
+ error: this.scrub(err),
424
+ });
425
+ return true; // an attempt happened; counts toward the per-tick budget
426
+ }
427
+ // POST-SPAWN SETTLE: a stop arrived during the async spawn, or a duplicate
428
+ // appeared → terminally KILL the just-spawned session (clears midWork first).
429
+ const stoppedAfter = this.safeBool(() => this.deps.operatorStoppedSince(topicId, new Date(run.startedAtMs ?? 0).toISOString()), false);
430
+ if (stoppedAfter) {
431
+ try {
432
+ await this.deps.settleKill(topicId);
433
+ }
434
+ catch {
435
+ /* settle-kill is best-effort; the operator-stop is already recorded */
436
+ }
437
+ this.deps.audit({ ts: new Date(now).toISOString(), event: 'settle-killed', topicId });
438
+ this.observed.delete(topicId);
439
+ return true;
440
+ }
441
+ // Success.
442
+ this.recordRedie(topicId, now); // a respawn happened; counts toward the redie window
443
+ this.respawnTotal += 1;
444
+ this.cappedSurfaced.delete(topicId);
445
+ this.setCondition(topicId, 'respawned', now);
446
+ this.deps.audit({
447
+ ts: new Date(now).toISOString(),
448
+ event: 'respawned',
449
+ topicId,
450
+ remainingSeconds: run.remainingSeconds,
451
+ });
452
+ if (this.cfg.notifyUser) {
453
+ void this.deps
454
+ .notifyTopic(topicId, `I noticed my run here had no live session and brought it back — picking it up.`)
455
+ .catch(() => {
456
+ /* @silent-fallback-ok — the self-heal notice is best-effort; a send
457
+ failure must never endanger the reconcile (the run is already alive,
458
+ which is the goal — a missed courtesy line is not a regression). */
459
+ });
460
+ }
461
+ // Reset debounce after a successful respawn; the next tick re-evaluates liveness.
462
+ this.observed.delete(topicId);
463
+ return true;
464
+ }
465
+ finally {
466
+ this.deps.releaseClaim(topicId);
467
+ }
468
+ }
469
+ // ── Generation guard (criterion 1) ──
470
+ isCurrentGeneration(run) {
471
+ if (run.startedAtMs == null)
472
+ return false; // no parseable started_at → safe side
473
+ const current = this.safe(() => this.deps.currentGenerationMs(run.topicId), null);
474
+ if (current == null)
475
+ return true; // no competing registration → this IS current
476
+ // Obsolete iff a NEWER registration exists than this run's started_at.
477
+ return current <= run.startedAtMs;
478
+ }
479
+ // ── In-flight predicate with stale-`spawning` TTL (criterion 7) ──
480
+ inflightActive(topicId, now) {
481
+ const s = this.safe(() => this.deps.inflightSpawnStatus(topicId), { state: 'none' });
482
+ if (s.state === 'none')
483
+ return false;
484
+ if (s.state === 'spawning') {
485
+ // A spawn wedged mid-`spawning` past the TTL is STALE → not in-flight.
486
+ return now - s.sinceMs < this.inflightTtlMs;
487
+ }
488
+ return true; // claimed / live-grace → in-flight
489
+ }
490
+ // ── Cap helpers (two separated counters) ──
491
+ freshWindow(map, topicId, now) {
492
+ const windowMs = this.cfg.respawnCapWindowSec * 1000;
493
+ const all = map.get(topicId) ?? [];
494
+ const fresh = all.filter((t) => now - t < windowMs);
495
+ if (fresh.length !== all.length) {
496
+ map.set(topicId, fresh);
497
+ this.persistCap();
498
+ }
499
+ return fresh;
500
+ }
501
+ withinRedieCap(topicId, now, queueResurrections) {
502
+ const ownRedie = this.freshWindow(this.redie, topicId, now).length;
503
+ // UNIFIED bound: own redie count + the queue's resurrection count share ONE budget.
504
+ return ownRedie + queueResurrections < this.cfg.respawnCapPerWindow;
505
+ }
506
+ recordRedie(topicId, now) {
507
+ const list = this.freshWindow(this.redie, topicId, now);
508
+ list.push(now);
509
+ this.redie.set(topicId, list);
510
+ this.persistCap();
511
+ }
512
+ recordSpawnFailure(topicId, now) {
513
+ const list = this.freshWindow(this.spawnFailure, topicId, now);
514
+ list.push(now);
515
+ this.spawnFailure.set(topicId, list);
516
+ this.persistCap();
517
+ }
518
+ persistCap() {
519
+ try {
520
+ this.deps.saveCapState?.({
521
+ redie: Object.fromEntries(this.redie.entries()),
522
+ spawnFailure: Object.fromEntries(this.spawnFailure.entries()),
523
+ });
524
+ }
525
+ catch {
526
+ /* cap persistence is best-effort */
527
+ }
528
+ }
529
+ // ── Condition record ──
530
+ setCondition(topicId, state, now, pressureBlockedTicks, pressureBlockedSinceMs) {
531
+ const prev = this.conditions.get(topicId);
532
+ this.conditions.set(topicId, {
533
+ state,
534
+ lastTransitionAt: prev?.state === state ? prev.lastTransitionAt : now,
535
+ pressureBlockedTicks: state === 'blocked-pressure' ? (pressureBlockedTicks ?? 1) : 0,
536
+ pressureBlockedSinceMs: state === 'blocked-pressure' ? (pressureBlockedSinceMs ?? now) : null,
537
+ });
538
+ }
539
+ clearPressureBlock(topicId) {
540
+ const cond = this.conditions.get(topicId);
541
+ if (cond && cond.state === 'blocked-pressure') {
542
+ cond.pressureBlockedTicks = 0;
543
+ cond.pressureBlockedSinceMs = null;
544
+ }
545
+ }
546
+ // ── GC ──
547
+ gc(now, candidateTopics) {
548
+ for (const topicId of [...this.observed.keys()]) {
549
+ if (!candidateTopics.has(topicId))
550
+ this.observed.delete(topicId);
551
+ }
552
+ const windowMs = this.cfg.respawnCapWindowSec * 1000;
553
+ const evict = (map) => {
554
+ for (const [topicId, ts] of [...map.entries()]) {
555
+ const fresh = ts.filter((t) => now - t < windowMs);
556
+ if (fresh.length === 0 && !candidateTopics.has(topicId))
557
+ map.delete(topicId);
558
+ else if (fresh.length !== ts.length)
559
+ map.set(topicId, fresh);
560
+ }
561
+ };
562
+ evict(this.redie);
563
+ evict(this.spawnFailure);
564
+ // Drop surfaced-once guards + conditions for topics no longer tracked at all.
565
+ for (const set of [
566
+ this.cappedSurfaced,
567
+ this.spawnFailSurfaced,
568
+ this.ambiguousSurfaced,
569
+ this.missingUuidSurfaced,
570
+ this.pressureSurfaced,
571
+ ]) {
572
+ for (const topicId of [...set]) {
573
+ if (!candidateTopics.has(topicId) &&
574
+ this.freshWindow(this.redie, topicId, now).length === 0 &&
575
+ this.freshWindow(this.spawnFailure, topicId, now).length === 0) {
576
+ set.delete(topicId);
577
+ }
578
+ }
579
+ }
580
+ }
581
+ // ── helpers ──
582
+ async withTimeout(p, ms) {
583
+ let timer = null;
584
+ const timeout = new Promise((_, reject) => {
585
+ timer = setTimeout(() => reject(new Error(`respawn timed out after ${ms}ms`)), ms);
586
+ timer.unref?.();
587
+ });
588
+ try {
589
+ return await Promise.race([p, timeout]);
590
+ }
591
+ finally {
592
+ if (timer)
593
+ clearTimeout(timer);
594
+ }
595
+ }
596
+ scrub(err) {
597
+ const raw = err instanceof Error ? err.message : String(err);
598
+ // Length-clamp + strip absolute paths / token-shaped substrings.
599
+ return raw
600
+ .slice(0, 300)
601
+ .replace(/\/[^\s]+/g, '<path>')
602
+ .replace(/[A-Za-z0-9_-]{24,}/g, '<redacted>');
603
+ }
604
+ safeBool(fn, fallback) {
605
+ try {
606
+ return fn();
607
+ }
608
+ catch {
609
+ // @silent-fallback-ok — every criterion dep resolves a throw toward the
610
+ // SAFE side (NOT-a-candidate / blocked), the spec's stated fail direction:
611
+ // the worst outcome of an unread gate is a deferred respawn, never a
612
+ // wrongful one. The fallback value passed by each caller encodes that side.
613
+ return fallback;
614
+ }
615
+ }
616
+ safe(fn, fallback) {
617
+ try {
618
+ return fn();
619
+ }
620
+ catch {
621
+ // @silent-fallback-ok — same fail-toward-safe contract as safeBool: a
622
+ // throwing snapshot/resolver dep degrades to the caller's safe default
623
+ // (empty set / null / 0), never an unsafe actuation.
624
+ return fallback;
625
+ }
626
+ }
627
+ guardStatus() {
628
+ return {
629
+ enabled: this.cfg.enabled !== false,
630
+ dryRun: this.cfg.dryRun,
631
+ };
632
+ }
633
+ status() {
634
+ return {
635
+ enabled: this.cfg.enabled !== false,
636
+ dryRun: this.cfg.dryRun,
637
+ tickIntervalSec: this.cfg.tickIntervalSec,
638
+ lastTickAt: this.lastTickAt == null ? null : new Date(this.lastTickAt).toISOString(),
639
+ respawnTotal: this.respawnTotal,
640
+ capPerWindow: this.cfg.respawnCapPerWindow,
641
+ capWindowSec: this.cfg.respawnCapWindowSec,
642
+ observing: [...this.observed.entries()].map(([topicId, o]) => ({
643
+ topicId,
644
+ count: o.count,
645
+ firstSeenAt: new Date(o.firstSeenMs).toISOString(),
646
+ })),
647
+ conditions: [...this.conditions.entries()].map(([topicId, c]) => ({
648
+ topicId,
649
+ state: c.state,
650
+ lastTransitionAt: new Date(c.lastTransitionAt).toISOString(),
651
+ })),
652
+ redie: [...this.redie.entries()].map(([topicId, ts]) => ({ topicId, count: ts.length })),
653
+ spawnFailure: [...this.spawnFailure.entries()].map(([topicId, ts]) => ({
654
+ topicId,
655
+ count: ts.length,
656
+ })),
657
+ };
658
+ }
659
+ }
660
+ //# sourceMappingURL=AutonomousLivenessReconciler.js.map