@coool/opencode-adaptive 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/opencode/index.d.ts +2 -0
- package/dist/adapters/opencode/index.js +2 -0
- package/dist/adapters/opencode/index.js.map +1 -1
- package/dist/adapters/opencode/process-ledger.d.ts +56 -0
- package/dist/adapters/opencode/process-ledger.js +143 -0
- package/dist/adapters/opencode/process-ledger.js.map +1 -0
- package/dist/adapters/opencode/session-ops.d.ts +2 -0
- package/dist/adapters/opencode/session-ops.js +15 -9
- package/dist/adapters/opencode/session-ops.js.map +1 -1
- package/dist/adapters/opencode/timer.d.ts +5 -0
- package/dist/adapters/opencode/timer.js +12 -0
- package/dist/adapters/opencode/timer.js.map +1 -0
- package/dist/execution/parking/index.d.ts +2 -1
- package/dist/execution/parking/index.js +2 -1
- package/dist/execution/parking/index.js.map +1 -1
- package/dist/execution/parking/wake.d.ts +52 -0
- package/dist/execution/parking/wake.js +95 -0
- package/dist/execution/parking/wake.js.map +1 -0
- package/dist/gui-experimental.cjs +3 -3
- package/dist/gui.d.ts +2 -2
- package/dist/gui.js +37 -3
- package/dist/gui.js.map +1 -1
- package/dist/health/classifier/classify.js +9 -0
- package/dist/health/classifier/classify.js.map +1 -1
- package/dist/index.js +33 -20
- package/dist/index.js.map +1 -1
- package/dist/rpc/status.d.ts +9 -0
- package/dist/rpc/status.js +1 -0
- package/dist/rpc/status.js.map +1 -1
- package/dist/state/parked-work/store.d.ts +5 -0
- package/dist/state/parked-work/store.js +24 -0
- package/dist/state/parked-work/store.js.map +1 -1
- package/dist/supervisor/supervisor.d.ts +2 -0
- package/dist/supervisor/supervisor.js +475 -106
- package/dist/supervisor/supervisor.js.map +1 -1
- package/dist/supervisor/types.d.ts +5 -0
- package/dist/supervisor/types.js.map +1 -1
- package/dist/tui.d.ts +13 -0
- package/dist/tui.js +62 -16
- package/dist/tui.js.map +1 -1
- package/dist/ui/flow/waiting.d.ts +11 -0
- package/dist/ui/flow/waiting.js +40 -0
- package/dist/ui/flow/waiting.js.map +1 -0
- package/dist/ui/gui/api.d.ts +2 -0
- package/dist/ui/gui/api.js +6 -2
- package/dist/ui/gui/api.js.map +1 -1
- package/package.json +1 -1
|
@@ -43,7 +43,7 @@ import { ReceiptStore, RECEIPTS_KEY } from "../state/receipts/index.js";
|
|
|
43
43
|
import { buildStatus } from "../rpc/status.js";
|
|
44
44
|
import { RECEIPTS_VIEW_LIMIT } from "./view.js";
|
|
45
45
|
import { ParkedWorkStore, PARKED_KEY } from "../state/parked-work/index.js";
|
|
46
|
-
import { decideRetry, executeContinuation, forMode as retryForMode, planContinuation, selectResumable, } from "../execution/index.js";
|
|
46
|
+
import { decideRetry, earliestUsable, executeContinuation, forMode as retryForMode, MAX_PARKED_ATTEMPTS, planContinuation, planWake, resourceUsableAt, selectResumable, } from "../execution/index.js";
|
|
47
47
|
import { escalationDetail } from "../execution/escalation/index.js";
|
|
48
48
|
import { listCandidates, sessionOps, storageSeam } from "../adapters/opencode/index.js";
|
|
49
49
|
import { CodexCapacitySensor } from "../adapters/codex-capacity/index.js";
|
|
@@ -149,9 +149,17 @@ export class Supervisor {
|
|
|
149
149
|
* Sessions with an owned recovery in flight (between veto/resume and the
|
|
150
150
|
* delivered continuation prompt). The host settle inside it can take
|
|
151
151
|
* unbounded time; an interruption arriving during that window must abort
|
|
152
|
-
* the stale decision instead of acting after it.
|
|
152
|
+
* the stale decision instead of acting after it. The value settles when
|
|
153
|
+
* the recovery does (it never rejects).
|
|
153
154
|
*/
|
|
154
|
-
#recoveryPending = new
|
|
155
|
+
#recoveryPending = new Map();
|
|
156
|
+
/**
|
|
157
|
+
* Units opened by an owned continuation's prompt. Such a unit can fail
|
|
158
|
+
* before the recovery that started it returns from its prompt call; its
|
|
159
|
+
* terminal failure is its own, not an echo of the superseded unit.
|
|
160
|
+
* Cleared at unit close.
|
|
161
|
+
*/
|
|
162
|
+
#continuationUnits = new Set();
|
|
155
163
|
/** Snapshot-sized budget shared by a task's terminal quota continuations. */
|
|
156
164
|
#quotaRecoveriesRemaining = new Map();
|
|
157
165
|
/** Interruptions observed while a recovery was pending on the session. */
|
|
@@ -200,6 +208,28 @@ export class Supervisor {
|
|
|
200
208
|
* unit opens, its prompt is rejected, or the session dies.
|
|
201
209
|
*/
|
|
202
210
|
#continuationTiers = new Map();
|
|
211
|
+
/**
|
|
212
|
+
* Parked-work wake (ADP-024): idle only follows successful work, so when a
|
|
213
|
+
* whole tier is blocked one timer stands in for the safe boundary at the
|
|
214
|
+
* earliest moment a candidate resource recovers. `dueAt` may lie past the
|
|
215
|
+
* timer when the delay was capped — an early fire only re-arms.
|
|
216
|
+
*/
|
|
217
|
+
#wake;
|
|
218
|
+
/** Consecutive wake sweeps that resumed nothing (backs the timer off). */
|
|
219
|
+
#fruitless = 0;
|
|
220
|
+
#disposed = false;
|
|
221
|
+
/**
|
|
222
|
+
* Whether this instance receives host reports. The host's other plugin
|
|
223
|
+
* instance (ADP-056) never does: its in-memory health is only what it
|
|
224
|
+
* hydrated, so it re-reads persisted health before a wake sweep.
|
|
225
|
+
*/
|
|
226
|
+
#live = false;
|
|
227
|
+
/**
|
|
228
|
+
* Session → resume cycles spent on its current task: a resumed unit that
|
|
229
|
+
* parks again carries the count, so park → wake → fail loops stay bounded
|
|
230
|
+
* by MAX_PARKED_ATTEMPTS. Cleared on success or a fresh user prompt.
|
|
231
|
+
*/
|
|
232
|
+
#resumeCycles = new Map();
|
|
203
233
|
#ready;
|
|
204
234
|
#ops;
|
|
205
235
|
constructor(options) {
|
|
@@ -218,8 +248,17 @@ export class Supervisor {
|
|
|
218
248
|
this.#ops = sessionOps(options.session, {
|
|
219
249
|
beforeSwitch: (sessionID, model) => {
|
|
220
250
|
this.#tracker.noteOwnSwitch(sessionID, model);
|
|
251
|
+
options.ledger.noteSwitch(sessionID, model, options.clock());
|
|
252
|
+
},
|
|
253
|
+
switchFailed: (sessionID) => {
|
|
254
|
+
options.ledger.withdrawSwitch(sessionID);
|
|
221
255
|
},
|
|
222
256
|
beforePrompt: (sessionID) => {
|
|
257
|
+
// The other plugin instance may be the one that hears this prompt:
|
|
258
|
+
// the ledger lets it recognize the continuation as Adaptive's.
|
|
259
|
+
const tier = this.#continuationTiers.get(sessionID);
|
|
260
|
+
const cycles = this.#resumeCycles.get(sessionID);
|
|
261
|
+
options.ledger.noteContinuation(sessionID, { ...(tier === undefined ? {} : { tier }), ...(cycles === undefined ? {} : { cycles }) }, options.clock());
|
|
223
262
|
this.#continuationPending.add(sessionID);
|
|
224
263
|
this.#continuationsInFlight.set(sessionID, (this.#continuationsInFlight.get(sessionID) ?? 0) + 1);
|
|
225
264
|
},
|
|
@@ -236,9 +275,10 @@ export class Supervisor {
|
|
|
236
275
|
// its ownership mark would swallow the next user prompt's
|
|
237
276
|
// admission, so it is dropped here (the hook can no longer fire).
|
|
238
277
|
// The continuation's tier intent dies with it — no episode will
|
|
239
|
-
// open to carry it.
|
|
278
|
+
// open to carry it. The other instance's view of it goes too.
|
|
240
279
|
this.#continuationPending.delete(sessionID);
|
|
241
280
|
this.#continuationTiers.delete(sessionID);
|
|
281
|
+
options.ledger.withdrawContinuation(sessionID);
|
|
242
282
|
}
|
|
243
283
|
}
|
|
244
284
|
else {
|
|
@@ -273,8 +313,10 @@ export class Supervisor {
|
|
|
273
313
|
};
|
|
274
314
|
for (const receipt of this.#receipts.snapshot())
|
|
275
315
|
bump(receipt.unitID);
|
|
276
|
-
for (const record of this.#parked.list())
|
|
316
|
+
for (const record of this.#parked.list()) {
|
|
277
317
|
bump(record.id);
|
|
318
|
+
options.ledger.noteParked(record.sessionID);
|
|
319
|
+
}
|
|
278
320
|
this.#tracker.seedEpisodes(floors);
|
|
279
321
|
// Health records hydrate the registry: the host's TUI-mode plugin
|
|
280
322
|
// reconciliation can serve RPC from an instance whose hooks (event
|
|
@@ -295,6 +337,23 @@ export class Supervisor {
|
|
|
295
337
|
return this.#settings.load();
|
|
296
338
|
};
|
|
297
339
|
this.#ready = loadState();
|
|
340
|
+
// Parked work survives restarts; its wake must too.
|
|
341
|
+
void this.#armWake();
|
|
342
|
+
}
|
|
343
|
+
/** Stop the wake timer; a disposed supervisor never re-arms it. */
|
|
344
|
+
dispose() {
|
|
345
|
+
this.#disposed = true;
|
|
346
|
+
this.#options.ledger.leave();
|
|
347
|
+
this.#wake?.cancel();
|
|
348
|
+
this.#wake = undefined;
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* This instance hears the host. The ledger ignores it from a disposed
|
|
352
|
+
* instance (a late report during teardown), so the survivor still sweeps.
|
|
353
|
+
*/
|
|
354
|
+
#markLive() {
|
|
355
|
+
this.#live = true;
|
|
356
|
+
this.#options.ledger.markLive();
|
|
298
357
|
}
|
|
299
358
|
/** Settings as of the latest load/patch (awaits the constructor load). */
|
|
300
359
|
async #ensureSettings() {
|
|
@@ -312,14 +371,18 @@ export class Supervisor {
|
|
|
312
371
|
// ---------- public entry points (never throw — fail-open, ADP-031) ----------
|
|
313
372
|
/** Handle one host report (event pump or session hook). */
|
|
314
373
|
async handleReport(report) {
|
|
374
|
+
this.#markLive();
|
|
315
375
|
try {
|
|
316
376
|
const settings = await this.#ensureSettings();
|
|
317
377
|
// An OFF session is fully inert — no supervision, no receipts, no
|
|
318
378
|
// health evidence. Idle is the exception: it is the bookkeeping
|
|
319
379
|
// boundary (parked-work sweep, dead-session override pruning) and runs
|
|
320
380
|
// for every report so other sessions' overrides are maintained; the
|
|
321
|
-
// sweep itself gates per parked session below.
|
|
322
|
-
|
|
381
|
+
// sweep itself gates per parked session below. A prompt still marks
|
|
382
|
+
// the user taking the session over (its parked work is dropped).
|
|
383
|
+
if (report.kind !== "idle" &&
|
|
384
|
+
report.kind !== "prompt" &&
|
|
385
|
+
this.#modeFor(report.sessionID, settings) === "off")
|
|
323
386
|
return;
|
|
324
387
|
await this.#dispatch(report, settings);
|
|
325
388
|
}
|
|
@@ -372,6 +435,7 @@ export class Supervisor {
|
|
|
372
435
|
* untouched and OpenCode's native retry policy stands.
|
|
373
436
|
*/
|
|
374
437
|
async handleRetry(input) {
|
|
438
|
+
this.#markLive();
|
|
375
439
|
try {
|
|
376
440
|
const settings = await this.#ensureSettings();
|
|
377
441
|
if (this.#modeFor(input.sessionID, settings) === "off")
|
|
@@ -451,6 +515,9 @@ export class Supervisor {
|
|
|
451
515
|
// (ADP-028): manual-selection suppression lifts for every session.
|
|
452
516
|
this.#manualSelections.clear();
|
|
453
517
|
}
|
|
518
|
+
// Mode and tier changes move which parked work is due and when.
|
|
519
|
+
if (result.ok)
|
|
520
|
+
await this.#rearmWake();
|
|
454
521
|
return result;
|
|
455
522
|
}
|
|
456
523
|
/** The session's mode override, or `null` when it follows the global mode. */
|
|
@@ -468,6 +535,7 @@ export class Supervisor {
|
|
|
468
535
|
await this.#sessionModes.set(sessionID, mode);
|
|
469
536
|
// A mode change is the user steering Adaptive explicitly (ADP-028).
|
|
470
537
|
this.#manualSelections.delete(sessionID);
|
|
538
|
+
await this.#rearmWake();
|
|
471
539
|
}
|
|
472
540
|
/**
|
|
473
541
|
* Bounded status snapshot for the status view (ADP-033). Health is read
|
|
@@ -478,12 +546,46 @@ export class Supervisor {
|
|
|
478
546
|
*/
|
|
479
547
|
async status() {
|
|
480
548
|
const settings = await this.#ensureSettings();
|
|
549
|
+
// Parked work too: the other instance may have parked or resumed it.
|
|
550
|
+
await this.#parked.refresh();
|
|
551
|
+
const now = this.#options.clock();
|
|
552
|
+
const { records, capacity, usableAt } = await this.#persistedView(settings, now);
|
|
553
|
+
// The wake that will actually resume the session may come later than the
|
|
554
|
+
// reset (backoff after fruitless wakes). It is this instance's own when
|
|
555
|
+
// it sweeps, else the one the live instance published.
|
|
556
|
+
const { ledger } = this.#options;
|
|
557
|
+
const wakeAt = this.#live || !ledger.liveElsewhere() ? this.#wake?.dueAt : ledger.liveWake();
|
|
558
|
+
const waiting = this.#parked
|
|
559
|
+
.list()
|
|
560
|
+
.filter((record) => this.#wakeable(record, settings))
|
|
561
|
+
.map((record) => {
|
|
562
|
+
const usable = parkedResumeAt(record, settings, usableAt);
|
|
563
|
+
return {
|
|
564
|
+
sessionID: record.sessionID,
|
|
565
|
+
tier: record.tier,
|
|
566
|
+
resumeAt: usable === undefined ? null : Math.max(usable, wakeAt ?? usable),
|
|
567
|
+
};
|
|
568
|
+
});
|
|
569
|
+
return buildStatus({
|
|
570
|
+
settings,
|
|
571
|
+
health: records,
|
|
572
|
+
capacity,
|
|
573
|
+
parked: this.#parked.count(),
|
|
574
|
+
waiting,
|
|
575
|
+
receipts: this.#receipts.snapshot().length,
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
/**
|
|
579
|
+
* Health and capacity as persisted by every instance, with when each ref
|
|
580
|
+
* becomes usable. The status surface reads this (its instance's registry
|
|
581
|
+
* can be empty when the host placed the event pump on another instance),
|
|
582
|
+
* and so does the wake of an instance that hears no host reports.
|
|
583
|
+
*/
|
|
584
|
+
async #persistedView(settings, now) {
|
|
481
585
|
await this.#healthRecords.load();
|
|
482
|
-
// The persisted mirror serves the status surface: a serving instance
|
|
483
|
-
// without the event-pump hooks must still report sensed capacity.
|
|
484
586
|
await this.#capacityStore.load();
|
|
485
|
-
const now = this.#options.clock();
|
|
486
587
|
const records = this.#healthRecords.snapshot().map((record) => sweepRecord(record, now));
|
|
588
|
+
const recordsByID = new Map(records.map((record) => [resourceKeyID(record.key), record]));
|
|
487
589
|
// Capacity readings are gathered per configured ref (mirroring the health
|
|
488
590
|
// input shape) so buildStatus stays a pure function of its inputs.
|
|
489
591
|
const capacity = {};
|
|
@@ -497,13 +599,8 @@ export class Supervisor {
|
|
|
497
599
|
capacity[schedulerHealthKey(ref)] = mergeReadings(candidates, now);
|
|
498
600
|
}
|
|
499
601
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
health: records,
|
|
503
|
-
capacity,
|
|
504
|
-
parked: this.#parked.count(),
|
|
505
|
-
receipts: this.#receipts.snapshot().length,
|
|
506
|
-
});
|
|
602
|
+
const usableAt = (ref) => resourceUsableAt(resourceChain(refKey(ref)).map((key) => recordsByID.get(resourceKeyID(key))), capacity[schedulerHealthKey(ref)] ?? null, now);
|
|
603
|
+
return { records, capacity, usableAt };
|
|
507
604
|
}
|
|
508
605
|
/** Newest-first bounded receipt view (ADP-030). */
|
|
509
606
|
async receipts() {
|
|
@@ -537,10 +634,14 @@ export class Supervisor {
|
|
|
537
634
|
// carries the attribution the tracker resolved. Only the two
|
|
538
635
|
// selection sources reach this event kind.
|
|
539
636
|
const { source } = result.transition;
|
|
540
|
-
if (source === "selected-adaptive"
|
|
637
|
+
if (source === "selected-adaptive" ||
|
|
638
|
+
// The host's other plugin instance (ADP-056) may have switched it.
|
|
639
|
+
this.#options.ledger.takeSwitch(report.sessionID, report.model, this.#options.clock())) {
|
|
541
640
|
this.#manualSelections.delete(report.sessionID);
|
|
542
|
-
|
|
543
|
-
|
|
641
|
+
return;
|
|
642
|
+
}
|
|
643
|
+
this.#noteManualSelection(report.sessionID);
|
|
644
|
+
await this.#takeOver(report.sessionID);
|
|
544
645
|
return;
|
|
545
646
|
}
|
|
546
647
|
case "model-request":
|
|
@@ -593,15 +694,31 @@ export class Supervisor {
|
|
|
593
694
|
*/
|
|
594
695
|
async #onPrompt(sessionID, settings) {
|
|
595
696
|
const mode = this.#modeFor(sessionID, settings);
|
|
596
|
-
|
|
697
|
+
const own = this.#continuationPending.delete(sessionID);
|
|
698
|
+
// The host's other plugin instance (ADP-056) may have sent it.
|
|
699
|
+
const foreign = own
|
|
700
|
+
? undefined
|
|
701
|
+
: this.#options.ledger.takeContinuation(sessionID, this.#options.clock());
|
|
702
|
+
if (mode !== "active") {
|
|
703
|
+
// An OFF or OBSERVE session is not routed, but the user's own turn
|
|
704
|
+
// there still supersedes its parked work: turning Adaptive back on
|
|
705
|
+
// must not replay a continuation the user has moved past.
|
|
706
|
+
if (!own && foreign === undefined)
|
|
707
|
+
await this.#takeOver(sessionID);
|
|
597
708
|
return;
|
|
598
|
-
|
|
709
|
+
}
|
|
710
|
+
if (own || foreign !== undefined) {
|
|
599
711
|
// An owned continuation (retry recovery, escalation, parked resume)
|
|
600
712
|
// already owns this unit's routing: re-arm the admission marker from
|
|
601
713
|
// the continuation's tier intent so execution-started skips a fresh
|
|
602
714
|
// (continuation-unaware) admission.
|
|
603
|
-
const tier = this.#continuationTiers.get(sessionID);
|
|
604
|
-
|
|
715
|
+
const tier = own ? this.#continuationTiers.get(sessionID) : foreign?.tier;
|
|
716
|
+
if (foreign?.cycles !== undefined)
|
|
717
|
+
this.#resumeCycles.set(sessionID, foreign.cycles);
|
|
718
|
+
this.#promptAdmission.set(sessionID, {
|
|
719
|
+
...(tier !== undefined ? { tier } : {}),
|
|
720
|
+
continuation: true,
|
|
721
|
+
});
|
|
605
722
|
return;
|
|
606
723
|
}
|
|
607
724
|
// Prompt hooks also admit steering messages into an active episode. They
|
|
@@ -615,6 +732,7 @@ export class Supervisor {
|
|
|
615
732
|
this.#promptAdmission.delete(sessionID);
|
|
616
733
|
this.#escalations.delete(sessionID);
|
|
617
734
|
this.#quotaRecoveriesRemaining.delete(sessionID);
|
|
735
|
+
await this.#takeOver(sessionID);
|
|
618
736
|
// A pending probe ticket from an earlier prompt admission (or a failed
|
|
619
737
|
// recovery) can only be carried by the unit the ORIGINAL prompt opens;
|
|
620
738
|
// a fresh admission means that episode never ran, so the ticket is
|
|
@@ -681,6 +799,11 @@ export class Supervisor {
|
|
|
681
799
|
// it before the close cleanup: a superseding start would otherwise
|
|
682
800
|
// discard it and re-route an already-admitted turn.
|
|
683
801
|
const admitted = this.#promptAdmission.get(report.sessionID);
|
|
802
|
+
// A run Adaptive did not continue — the user's shell or slash command
|
|
803
|
+
// skips the prompt hook — moves the session past any resume waiting to
|
|
804
|
+
// settle on it.
|
|
805
|
+
if (admitted?.continuation !== true)
|
|
806
|
+
this.#options.ledger.noteTakeover(report.sessionID);
|
|
684
807
|
// A superseding start closes the previous episode — release its
|
|
685
808
|
// per-unit bookkeeping (veto budget) too.
|
|
686
809
|
this.#afterUnitClose(opened);
|
|
@@ -691,6 +814,8 @@ export class Supervisor {
|
|
|
691
814
|
this.#continuationTiers.delete(report.sessionID);
|
|
692
815
|
if (admitted.tier !== undefined)
|
|
693
816
|
this.#admissionTiers.set(unit.id, admitted.tier);
|
|
817
|
+
if (admitted.continuation === true)
|
|
818
|
+
this.#continuationUnits.add(unit.id);
|
|
694
819
|
// The prompt-time admission receipt was recorded before this unit
|
|
695
820
|
// existed; adopt it so outcomes attach to it like any unit receipt.
|
|
696
821
|
try {
|
|
@@ -875,6 +1000,7 @@ export class Supervisor {
|
|
|
875
1000
|
at: report.at,
|
|
876
1001
|
});
|
|
877
1002
|
this.#afterUnitClose(closed);
|
|
1003
|
+
this.#resumeCycles.delete(report.sessionID);
|
|
878
1004
|
if (model !== undefined) {
|
|
879
1005
|
this.#observeHealth({ kind: "success", key: refKey(model) });
|
|
880
1006
|
}
|
|
@@ -895,6 +1021,9 @@ export class Supervisor {
|
|
|
895
1021
|
// Tool evidence is captured pre-close like the other per-unit maps.
|
|
896
1022
|
const toolEvidence = unitID !== undefined ? (this.#escalationEvidence.get(unitID) ?? []) : [];
|
|
897
1023
|
const probeTicket = this.#takeProbeTicket(unitID);
|
|
1024
|
+
// A vetoed continuation unit's failure is an echo: its own veto owns the
|
|
1025
|
+
// next continuation.
|
|
1026
|
+
const continuation = unitID !== undefined && this.#continuationUnits.has(unitID) && !this.#vetoes.has(unitID);
|
|
898
1027
|
// The terminal classifies and dedups BEFORE the close: the close clears
|
|
899
1028
|
// the unit's attempt-failure entry, and the terminal must consult it —
|
|
900
1029
|
// it repeats the episode's last attempt, which an attempt signal
|
|
@@ -932,15 +1061,21 @@ export class Supervisor {
|
|
|
932
1061
|
if (unit !== undefined &&
|
|
933
1062
|
model !== undefined &&
|
|
934
1063
|
classification.category === "quota_exhausted") {
|
|
935
|
-
await this.#recoverTerminalQuota(unit, admissionTier, settings);
|
|
1064
|
+
await this.#recoverTerminalQuota(unit, admissionTier, continuation, settings);
|
|
936
1065
|
return;
|
|
937
1066
|
}
|
|
938
1067
|
await this.#escalateOnFailure(report, unitID, model, classification, admissionTier, escalationUsed, toolEvidence, settings);
|
|
939
1068
|
}
|
|
940
|
-
async #recoverTerminalQuota(unit, requiredTier, settings) {
|
|
1069
|
+
async #recoverTerminalQuota(unit, requiredTier, continuation, settings) {
|
|
941
1070
|
const sessionID = unit.sessionID;
|
|
942
|
-
//
|
|
943
|
-
//
|
|
1071
|
+
// The continuation unit itself can run out of quota before the recovery
|
|
1072
|
+
// that started it returns from its prompt call. Its failure is its own:
|
|
1073
|
+
// wait for that recovery to settle, then recover (or park) like any unit.
|
|
1074
|
+
if (continuation)
|
|
1075
|
+
await this.#recoveryPending.get(sessionID);
|
|
1076
|
+
// Otherwise a pending continuation owns the session (a retry veto's
|
|
1077
|
+
// terminal echo must not start another). OBSERVE computes the same
|
|
1078
|
+
// admission without applying it.
|
|
944
1079
|
const mode = this.#modeFor(sessionID, settings);
|
|
945
1080
|
if (mode === "off" ||
|
|
946
1081
|
this.#recoveryPending.has(sessionID) ||
|
|
@@ -964,19 +1099,22 @@ export class Supervisor {
|
|
|
964
1099
|
if (decision.kind === "apply" && remaining === 0) {
|
|
965
1100
|
if (probeTicket !== undefined)
|
|
966
1101
|
this.#health.releaseProbe(probeTicket);
|
|
1102
|
+
// The ladder is spent but the work is not done: park it so the wake
|
|
1103
|
+
// resumes it with a fresh budget instead of leaving the session stuck.
|
|
1104
|
+
if (tier !== undefined) {
|
|
1105
|
+
await this.#recordReceipt({
|
|
1106
|
+
sessionID,
|
|
1107
|
+
tier,
|
|
1108
|
+
decision: { kind: "park", reason: "all-blocked", detail: "quota recovery budget spent" },
|
|
1109
|
+
});
|
|
1110
|
+
await this.#parkUnit(unit.id, sessionID, tier, "quota exhausted");
|
|
1111
|
+
}
|
|
967
1112
|
return;
|
|
968
1113
|
}
|
|
969
1114
|
await this.#recordReceipt({ sessionID, tier, decision });
|
|
970
1115
|
if (decision.kind !== "apply") {
|
|
971
1116
|
if (decision.kind === "park" && tier !== undefined) {
|
|
972
|
-
await this.#
|
|
973
|
-
id: unit.id,
|
|
974
|
-
sessionID,
|
|
975
|
-
tier,
|
|
976
|
-
reason: "all-blocked",
|
|
977
|
-
detail: "quota exhausted",
|
|
978
|
-
parkedAt: this.#options.clock(),
|
|
979
|
-
});
|
|
1117
|
+
await this.#parkUnit(unit.id, sessionID, tier, "quota exhausted");
|
|
980
1118
|
}
|
|
981
1119
|
return;
|
|
982
1120
|
}
|
|
@@ -1095,6 +1233,10 @@ export class Supervisor {
|
|
|
1095
1233
|
}
|
|
1096
1234
|
const unit = this.#tracker.active(report.sessionID);
|
|
1097
1235
|
const probeTicket = this.#takeProbeTicket(unit?.id);
|
|
1236
|
+
// The user cancelling is the user steering: a resume waiting on this
|
|
1237
|
+
// session (on either instance) drops its work instead of re-parking it.
|
|
1238
|
+
if (reason === "user")
|
|
1239
|
+
this.#options.ledger.noteTakeover(report.sessionID);
|
|
1098
1240
|
// An interruption while a recovery is pending on this session aborts it
|
|
1099
1241
|
// at its post-settle guard (the pending recovery's decision predates the
|
|
1100
1242
|
// interruption and must not act after it).
|
|
@@ -1189,6 +1331,7 @@ export class Supervisor {
|
|
|
1189
1331
|
const unitID = closedUnitID(closed);
|
|
1190
1332
|
if (unitID !== undefined) {
|
|
1191
1333
|
this.#vetoes.delete(unitID);
|
|
1334
|
+
this.#continuationUnits.delete(unitID);
|
|
1192
1335
|
this.#admissionTiers.delete(unitID);
|
|
1193
1336
|
this.#escalationEvidence.delete(unitID);
|
|
1194
1337
|
const leftover = this.#probeTickets.get(unitID);
|
|
@@ -1359,19 +1502,7 @@ export class Supervisor {
|
|
|
1359
1502
|
}
|
|
1360
1503
|
if (unitID === undefined || routingTier === undefined)
|
|
1361
1504
|
return;
|
|
1362
|
-
|
|
1363
|
-
await this.#parked.park({
|
|
1364
|
-
id: unitID,
|
|
1365
|
-
sessionID: input.sessionID,
|
|
1366
|
-
tier: routingTier,
|
|
1367
|
-
reason: "all-blocked",
|
|
1368
|
-
detail: "continuation failed",
|
|
1369
|
-
parkedAt: this.#options.clock(),
|
|
1370
|
-
});
|
|
1371
|
-
}
|
|
1372
|
-
catch (parkError) {
|
|
1373
|
-
logFailOpen("parked", parkError);
|
|
1374
|
-
}
|
|
1505
|
+
await this.#parkUnit(unitID, input.sessionID, routingTier, "continuation failed");
|
|
1375
1506
|
});
|
|
1376
1507
|
}
|
|
1377
1508
|
/**
|
|
@@ -1380,12 +1511,22 @@ export class Supervisor {
|
|
|
1380
1511
|
* continuation's prompt hook re-arms the admission marker (the unit must
|
|
1381
1512
|
* not be re-routed: that would discard the recovery/escalation decision).
|
|
1382
1513
|
*/
|
|
1383
|
-
|
|
1514
|
+
#ownRecovery(sessionID, model, tier, stillValid) {
|
|
1515
|
+
const run = this.#continueSession(sessionID, model, tier, stillValid);
|
|
1516
|
+
// The guard window opens in the same tick the recovery starts (before
|
|
1517
|
+
// its first await completes) so an interrupt pumped during the admission
|
|
1518
|
+
// race is still seen.
|
|
1519
|
+
const pending = run.catch(() => undefined);
|
|
1520
|
+
this.#recoveryPending.set(sessionID, pending);
|
|
1521
|
+
return run.finally(() => {
|
|
1522
|
+
// A later recovery on the session owns the entry now.
|
|
1523
|
+
if (this.#recoveryPending.get(sessionID) === pending)
|
|
1524
|
+
this.#recoveryPending.delete(sessionID);
|
|
1525
|
+
});
|
|
1526
|
+
}
|
|
1527
|
+
async #continueSession(sessionID, model, tier, stillValid) {
|
|
1384
1528
|
if (tier !== undefined)
|
|
1385
1529
|
this.#continuationTiers.set(sessionID, tier);
|
|
1386
|
-
// The guard window opens before the first await so an interrupt pumped
|
|
1387
|
-
// during the admission race is still seen.
|
|
1388
|
-
this.#recoveryPending.add(sessionID);
|
|
1389
1530
|
try {
|
|
1390
1531
|
const alive = await this.#isSessionAlive(sessionID);
|
|
1391
1532
|
const recovery = { kind: "switch", model };
|
|
@@ -1416,20 +1557,120 @@ export class Supervisor {
|
|
|
1416
1557
|
this.#continuationTiers.delete(sessionID);
|
|
1417
1558
|
throw error;
|
|
1418
1559
|
}
|
|
1419
|
-
finally {
|
|
1420
|
-
this.#recoveryPending.delete(sessionID);
|
|
1421
|
-
}
|
|
1422
1560
|
}
|
|
1423
|
-
// ---------- parked-work resume (safe
|
|
1561
|
+
// ---------- parked-work resume (safe boundaries: idle, wake) ----------
|
|
1424
1562
|
async #onIdle(settings) {
|
|
1563
|
+
await this.#sweepParked(settings);
|
|
1564
|
+
await this.#armWake();
|
|
1565
|
+
}
|
|
1566
|
+
/**
|
|
1567
|
+
* The wake timer's safe boundary. Fires only when no unit of a parked
|
|
1568
|
+
* session can reach idle on its own; it runs the same sweep (same
|
|
1569
|
+
* safeguards) as idle, then re-plans. An early fire (capped delay) only
|
|
1570
|
+
* re-plans: no sweep, no receipts while a long window is still open.
|
|
1571
|
+
*/
|
|
1572
|
+
async #onWake(dueAt, capped) {
|
|
1573
|
+
this.#setWake(undefined);
|
|
1574
|
+
try {
|
|
1575
|
+
const settings = await this.#ensureSettings();
|
|
1576
|
+
if (!this.#disposed && (!capped || this.#options.clock() >= dueAt)) {
|
|
1577
|
+
if (!this.#live && this.#options.ledger.liveElsewhere()) {
|
|
1578
|
+
// The instance that hears the host has current health and sees
|
|
1579
|
+
// the resumed unit's reports: it resumes, this one backs off.
|
|
1580
|
+
this.#fruitless += 1;
|
|
1581
|
+
}
|
|
1582
|
+
else {
|
|
1583
|
+
if (!this.#live) {
|
|
1584
|
+
// No instance hears the host yet (fresh start): persisted health
|
|
1585
|
+
// is the only current evidence.
|
|
1586
|
+
await this.#healthRecords.load();
|
|
1587
|
+
this.#health.hydrate(this.#healthRecords.snapshot());
|
|
1588
|
+
}
|
|
1589
|
+
// A taken record's delivery settles the backoff itself.
|
|
1590
|
+
if (!(await this.#sweepParked(settings)))
|
|
1591
|
+
this.#fruitless += 1;
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
catch (error) {
|
|
1596
|
+
logFailOpen("wake", error);
|
|
1597
|
+
}
|
|
1598
|
+
await this.#armWake();
|
|
1599
|
+
}
|
|
1600
|
+
/**
|
|
1601
|
+
* Plans the one wake timer from the parked work due for resume (ADP-024).
|
|
1602
|
+
* Never throws: a failed plan leaves parked work to the idle boundary.
|
|
1603
|
+
*/
|
|
1604
|
+
async #armWake() {
|
|
1605
|
+
try {
|
|
1606
|
+
const settings = await this.#ensureSettings();
|
|
1607
|
+
const now = this.#options.clock();
|
|
1608
|
+
// An instance that hears no host has no current health or capacity of
|
|
1609
|
+
// its own: it plans from what the live one persisted, as status does.
|
|
1610
|
+
const usableAt = this.#live
|
|
1611
|
+
? (ref) => resourceUsableAt(resourceChain(refKey(ref)).map((key) => this.#health.recordFor(key)), this.#capacity.reading(refKey(ref)), now)
|
|
1612
|
+
: (await this.#persistedView(settings, now)).usableAt;
|
|
1613
|
+
// Disposal may land during either read above.
|
|
1614
|
+
if (this.#disposed)
|
|
1615
|
+
return;
|
|
1616
|
+
const targets = this.#parked
|
|
1617
|
+
.list()
|
|
1618
|
+
.filter((record) => this.#wakeable(record, settings))
|
|
1619
|
+
.map((record) => parkedResumeAt(record, settings, usableAt));
|
|
1620
|
+
const plan = planWake(targets, now, this.#fruitless);
|
|
1621
|
+
if (plan !== undefined && plan.dueAt === this.#wake?.dueAt)
|
|
1622
|
+
return;
|
|
1623
|
+
this.#wake?.cancel();
|
|
1624
|
+
this.#setWake(undefined);
|
|
1625
|
+
if (plan === undefined)
|
|
1626
|
+
return;
|
|
1627
|
+
const { dueAt, capped } = plan;
|
|
1628
|
+
this.#setWake({
|
|
1629
|
+
dueAt,
|
|
1630
|
+
cancel: this.#options.schedule(() => void this.#onWake(dueAt, capped), plan.delayMs),
|
|
1631
|
+
});
|
|
1632
|
+
}
|
|
1633
|
+
catch (error) {
|
|
1634
|
+
logFailOpen("wake", error);
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
/** Keeps the published wake (what status shows elsewhere) in step. */
|
|
1638
|
+
#setWake(wake) {
|
|
1639
|
+
this.#wake = wake;
|
|
1640
|
+
this.#options.ledger.noteWake(wake?.dueAt);
|
|
1641
|
+
}
|
|
1642
|
+
/** A change the user or a new park made: re-plan without backoff. */
|
|
1643
|
+
async #rearmWake() {
|
|
1644
|
+
this.#fruitless = 0;
|
|
1645
|
+
await this.#armWake();
|
|
1646
|
+
}
|
|
1647
|
+
/**
|
|
1648
|
+
* Parked work the wake serves: an ACTIVE session (observe never resumes,
|
|
1649
|
+
* off is inert), a configured tier (else it is held), attempts left.
|
|
1650
|
+
*/
|
|
1651
|
+
#wakeable(record, settings) {
|
|
1652
|
+
return (this.#modeFor(record.sessionID, settings) === "active" &&
|
|
1653
|
+
configuredTiers(settings).includes(record.tier) &&
|
|
1654
|
+
(record.attempts ?? 0) < MAX_PARKED_ATTEMPTS);
|
|
1655
|
+
}
|
|
1656
|
+
/**
|
|
1657
|
+
* Resumes due parked work once per session; true when any record was
|
|
1658
|
+
* taken for resume. Reloads parked work first: the host's other plugin
|
|
1659
|
+
* instance may have parked or resumed records since this one last looked.
|
|
1660
|
+
*/
|
|
1661
|
+
async #sweepParked(settings) {
|
|
1662
|
+
await this.#parked.refresh();
|
|
1425
1663
|
const records = this.#parked.list();
|
|
1664
|
+
for (const record of records)
|
|
1665
|
+
this.#options.ledger.noteParked(record.sessionID);
|
|
1426
1666
|
// Override holders join the alive sweep: a dead session's override is
|
|
1427
1667
|
// pruned here (its per-session steering is worthless once the session is
|
|
1428
1668
|
// gone), in the same sweep that clears the other per-session maps.
|
|
1429
1669
|
const overrideIDs = this.#sessionModes.snapshot().map((record) => record.sessionID);
|
|
1430
1670
|
const tierRequestIDs = this.#tierRequests.snapshot().map((record) => record.sessionID);
|
|
1431
|
-
if (records.length === 0 && overrideIDs.length === 0 && tierRequestIDs.length === 0)
|
|
1432
|
-
return;
|
|
1671
|
+
if (records.length === 0 && overrideIDs.length === 0 && tierRequestIDs.length === 0) {
|
|
1672
|
+
return false;
|
|
1673
|
+
}
|
|
1433
1674
|
// OFF sessions are not supervised: their parked records are neither
|
|
1434
1675
|
// dropped nor resumed, and no alive probe runs for them.
|
|
1435
1676
|
const supervised = new Set(records
|
|
@@ -1473,7 +1714,10 @@ export class Supervisor {
|
|
|
1473
1714
|
// a safe boundary), and several back-to-back continuation prompts could
|
|
1474
1715
|
// redo work an earlier one already completed (ADP-023).
|
|
1475
1716
|
const attempted = new Set();
|
|
1717
|
+
let resumed = false;
|
|
1476
1718
|
for (const record of plan.attempt) {
|
|
1719
|
+
if (this.#disposed)
|
|
1720
|
+
break;
|
|
1477
1721
|
if (this.#modeFor(record.sessionID, settings) === "off")
|
|
1478
1722
|
continue;
|
|
1479
1723
|
if (this.#tracker.active(record.sessionID) !== undefined)
|
|
@@ -1481,7 +1725,8 @@ export class Supervisor {
|
|
|
1481
1725
|
if (attempted.has(record.sessionID))
|
|
1482
1726
|
continue;
|
|
1483
1727
|
attempted.add(record.sessionID);
|
|
1484
|
-
await this.#resumeAttempt(record, settings)
|
|
1728
|
+
if (await this.#resumeAttempt(record, settings))
|
|
1729
|
+
resumed = true;
|
|
1485
1730
|
}
|
|
1486
1731
|
for (const [sessionID, isAlive] of alive) {
|
|
1487
1732
|
if (!isAlive) {
|
|
@@ -1490,6 +1735,8 @@ export class Supervisor {
|
|
|
1490
1735
|
this.#continuationPending.delete(sessionID);
|
|
1491
1736
|
this.#escalations.delete(sessionID);
|
|
1492
1737
|
this.#quotaRecoveriesRemaining.delete(sessionID);
|
|
1738
|
+
this.#resumeCycles.delete(sessionID);
|
|
1739
|
+
this.#options.ledger.clearParked(sessionID);
|
|
1493
1740
|
if (this.#sessionModes.get(sessionID) !== null) {
|
|
1494
1741
|
try {
|
|
1495
1742
|
await this.#sessionModes.set(sessionID, null);
|
|
@@ -1500,6 +1747,7 @@ export class Supervisor {
|
|
|
1500
1747
|
}
|
|
1501
1748
|
}
|
|
1502
1749
|
}
|
|
1750
|
+
return resumed;
|
|
1503
1751
|
}
|
|
1504
1752
|
async #dropParked(record) {
|
|
1505
1753
|
try {
|
|
@@ -1509,7 +1757,83 @@ export class Supervisor {
|
|
|
1509
1757
|
logFailOpen("parked", error);
|
|
1510
1758
|
}
|
|
1511
1759
|
}
|
|
1512
|
-
/**
|
|
1760
|
+
/**
|
|
1761
|
+
* The user took the session over (own prompt or model pick): a resume in
|
|
1762
|
+
* flight on either plugin instance aborts after its settle, and parked
|
|
1763
|
+
* work is dropped — resuming it would replay a continuation the user has
|
|
1764
|
+
* already moved past.
|
|
1765
|
+
*/
|
|
1766
|
+
async #takeOver(sessionID) {
|
|
1767
|
+
const { ledger } = this.#options;
|
|
1768
|
+
ledger.noteTakeover(sessionID);
|
|
1769
|
+
this.#resumeCycles.delete(sessionID);
|
|
1770
|
+
const parkedHere = () => this.#parked.list().filter((record) => record.sessionID === sessionID);
|
|
1771
|
+
// A session that never parked costs no storage read (OFF stays free).
|
|
1772
|
+
if (parkedHere().length === 0 && !ledger.mayBeParked(sessionID))
|
|
1773
|
+
return;
|
|
1774
|
+
await this.#parked.refresh();
|
|
1775
|
+
const records = parkedHere();
|
|
1776
|
+
ledger.clearParked(sessionID);
|
|
1777
|
+
if (records.length === 0)
|
|
1778
|
+
return;
|
|
1779
|
+
for (const record of records)
|
|
1780
|
+
await this.#dropParked(record);
|
|
1781
|
+
await this.#rearmWake();
|
|
1782
|
+
}
|
|
1783
|
+
/**
|
|
1784
|
+
* Parks a unit whose tier has no usable resource. A unit that was itself
|
|
1785
|
+
* a resume carries the session's resume cycles, so repeated park → wake →
|
|
1786
|
+
* fail rounds exhaust MAX_PARKED_ATTEMPTS instead of looping.
|
|
1787
|
+
*/
|
|
1788
|
+
async #parkUnit(id, sessionID, tier, detail) {
|
|
1789
|
+
const attempts = this.#resumeCycles.get(sessionID);
|
|
1790
|
+
if (await this.#storeParked({
|
|
1791
|
+
id,
|
|
1792
|
+
sessionID,
|
|
1793
|
+
tier,
|
|
1794
|
+
reason: "all-blocked",
|
|
1795
|
+
detail,
|
|
1796
|
+
parkedAt: this.#options.clock(),
|
|
1797
|
+
...(attempts !== undefined ? { attempts } : {}),
|
|
1798
|
+
})) {
|
|
1799
|
+
await this.#rearmWake();
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
/**
|
|
1803
|
+
* Persists parked work; false when nothing waits. Work out of resume
|
|
1804
|
+
* attempts is not kept: it gets a receipt saying so instead of lingering
|
|
1805
|
+
* unresumable behind a waiting chip.
|
|
1806
|
+
*/
|
|
1807
|
+
async #storeParked(record) {
|
|
1808
|
+
if ((record.attempts ?? 0) >= MAX_PARKED_ATTEMPTS) {
|
|
1809
|
+
await this.#recordReceipt({
|
|
1810
|
+
sessionID: record.sessionID,
|
|
1811
|
+
unitID: record.id,
|
|
1812
|
+
tier: record.tier,
|
|
1813
|
+
decision: {
|
|
1814
|
+
kind: "park",
|
|
1815
|
+
reason: "all-blocked",
|
|
1816
|
+
detail: `gave up after ${String(MAX_PARKED_ATTEMPTS)} resume attempts`,
|
|
1817
|
+
},
|
|
1818
|
+
});
|
|
1819
|
+
return false;
|
|
1820
|
+
}
|
|
1821
|
+
this.#options.ledger.noteParked(record.sessionID);
|
|
1822
|
+
try {
|
|
1823
|
+
await this.#parked.park(record);
|
|
1824
|
+
}
|
|
1825
|
+
catch (error) {
|
|
1826
|
+
logFailOpen("parked", error);
|
|
1827
|
+
return false;
|
|
1828
|
+
}
|
|
1829
|
+
return true;
|
|
1830
|
+
}
|
|
1831
|
+
/**
|
|
1832
|
+
* One parked record's fresh routing attempt at a safe boundary; true when
|
|
1833
|
+
* the record was taken for resume. Its continuation is delivered in the
|
|
1834
|
+
* background: the prompt call stays open while the continued unit runs,
|
|
1835
|
+
* and the sweep must not hold every other due session behind it.
|
|
1836
|
+
*/
|
|
1513
1837
|
async #resumeAttempt(record, settings) {
|
|
1514
1838
|
// The parked session resumes under ITS effective mode, not the idle
|
|
1515
1839
|
// report's session's mode.
|
|
@@ -1523,57 +1847,91 @@ export class Supervisor {
|
|
|
1523
1847
|
}
|
|
1524
1848
|
catch (error) {
|
|
1525
1849
|
logFailOpen("resume", error);
|
|
1526
|
-
return;
|
|
1850
|
+
return false;
|
|
1527
1851
|
}
|
|
1528
|
-
if (mode === "observe") {
|
|
1852
|
+
if (mode === "observe" || decision.kind !== "apply") {
|
|
1853
|
+
// Observed, or still no usable resource (or the routing deferred) —
|
|
1854
|
+
// stay parked. A non-apply admission held no probe ticket: #admit keeps
|
|
1855
|
+
// a ticket only for apply decisions and releases every other one.
|
|
1529
1856
|
if (probeTicket !== undefined)
|
|
1530
1857
|
this.#health.releaseProbe(probeTicket);
|
|
1531
|
-
await this.#
|
|
1532
|
-
|
|
1533
|
-
unitID: record.id,
|
|
1534
|
-
tier: record.tier,
|
|
1535
|
-
decision: { kind: "observe-only", would: decision },
|
|
1536
|
-
});
|
|
1537
|
-
return;
|
|
1538
|
-
}
|
|
1539
|
-
if (decision.kind !== "apply") {
|
|
1540
|
-
// Still no usable resource (or the routing deferred) — stay parked. The
|
|
1541
|
-
// admission held no probe ticket: #admit keeps a ticket only for apply
|
|
1542
|
-
// decisions and releases every other one itself.
|
|
1543
|
-
await this.#recordReceipt({
|
|
1544
|
-
sessionID: record.sessionID,
|
|
1545
|
-
unitID: record.id,
|
|
1546
|
-
tier: record.tier,
|
|
1547
|
-
decision,
|
|
1548
|
-
});
|
|
1549
|
-
return;
|
|
1858
|
+
await this.#noteStillParked(record, mode === "observe" ? { kind: "observe-only", would: decision } : decision);
|
|
1859
|
+
return false;
|
|
1550
1860
|
}
|
|
1551
|
-
|
|
1552
|
-
|
|
1861
|
+
// The takeover baseline is read before the claim, synchronously with it:
|
|
1862
|
+
// a user prompt landing while the record is popped still counts. The
|
|
1863
|
+
// claim spans the pop and the continuation's delivery: the host's other
|
|
1864
|
+
// plugin instance sweeping the same record backs off, then finds it gone
|
|
1865
|
+
// once it re-reads the parked store.
|
|
1866
|
+
const { ledger } = this.#options;
|
|
1867
|
+
const takeovers = ledger.takeovers(record.sessionID);
|
|
1868
|
+
if (!ledger.claim(record.sessionID)) {
|
|
1553
1869
|
if (probeTicket !== undefined)
|
|
1554
1870
|
this.#health.releaseProbe(probeTicket);
|
|
1555
|
-
return;
|
|
1871
|
+
return false;
|
|
1556
1872
|
}
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1873
|
+
let delivering = false;
|
|
1874
|
+
try {
|
|
1875
|
+
const popped = await this.#popParked(record);
|
|
1876
|
+
if (popped === undefined) {
|
|
1877
|
+
if (probeTicket !== undefined)
|
|
1878
|
+
this.#health.releaseProbe(probeTicket);
|
|
1879
|
+
return false;
|
|
1880
|
+
}
|
|
1881
|
+
// The resumed unit is a new round of the same task: a fresh quota
|
|
1882
|
+
// ladder, and one more spent resume cycle should it park again.
|
|
1883
|
+
this.#resumeCycles.set(record.sessionID, (record.attempts ?? 0) + 1);
|
|
1884
|
+
this.#quotaRecoveriesRemaining.delete(record.sessionID);
|
|
1885
|
+
if (probeTicket !== undefined)
|
|
1886
|
+
this.#sessionProbeTickets.set(record.sessionID, probeTicket);
|
|
1887
|
+
// The resume continuation is not re-admitted (owned continuation), so
|
|
1888
|
+
// the freshly routed admission is recorded here, unit-less; the unit
|
|
1889
|
+
// that opens next adopts it and carries the outcome.
|
|
1890
|
+
await this.#recordReceipt({ sessionID: record.sessionID, tier: record.tier, decision });
|
|
1891
|
+
delivering = true;
|
|
1892
|
+
const takenOver = () => ledger.takeovers(record.sessionID) !== takeovers;
|
|
1893
|
+
void this.#deliverResume(record, decision.model, settings, probeTicket, takenOver).finally(() => {
|
|
1894
|
+
ledger.release(record.sessionID);
|
|
1895
|
+
});
|
|
1896
|
+
return true;
|
|
1897
|
+
}
|
|
1898
|
+
finally {
|
|
1899
|
+
if (!delivering)
|
|
1900
|
+
ledger.release(record.sessionID);
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
/**
|
|
1904
|
+
* Records why parked work stays parked. A wake that finds nothing changed
|
|
1905
|
+
* adds no receipt: a tier with no stated recovery would otherwise log the
|
|
1906
|
+
* same decision every few minutes.
|
|
1907
|
+
*/
|
|
1908
|
+
async #noteStillParked(record, decision) {
|
|
1909
|
+
const last = this.#receipts.snapshot().findLast((receipt) => receipt.unitID === record.id);
|
|
1910
|
+
if (last !== undefined && JSON.stringify(last.decision) === JSON.stringify(decision))
|
|
1911
|
+
return;
|
|
1560
1912
|
await this.#recordReceipt({
|
|
1561
1913
|
sessionID: record.sessionID,
|
|
1914
|
+
unitID: record.id,
|
|
1562
1915
|
tier: record.tier,
|
|
1563
1916
|
decision,
|
|
1564
1917
|
});
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1918
|
+
}
|
|
1919
|
+
/**
|
|
1920
|
+
* The resume continuation is an owned recovery: settle first, apply the
|
|
1921
|
+
* freshly routed model, then prompt — the episode's first LLM call must
|
|
1922
|
+
* not hit the model the work was parked away from. Never rejects.
|
|
1923
|
+
*/
|
|
1924
|
+
async #deliverResume(record, model, settings, probeTicket, takenOver) {
|
|
1568
1925
|
try {
|
|
1569
|
-
|
|
1570
|
-
this.#sessionProbeTickets.set(record.sessionID, probeTicket);
|
|
1571
|
-
await this.#ownRecovery(record.sessionID, decision.model, record.tier,
|
|
1926
|
+
await this.#ownRecovery(record.sessionID, model, record.tier,
|
|
1572
1927
|
// The parked session's own effective mode and its still-open unit
|
|
1573
1928
|
// window are re-checked after the settle; a session that gained an
|
|
1574
|
-
// active unit must not be prompted (not a safe boundary)
|
|
1575
|
-
|
|
1929
|
+
// active unit must not be prompted (not a safe boundary), and one the
|
|
1930
|
+
// user took over (on either instance) has moved past the work.
|
|
1931
|
+
() => !(takenOver() || this.#disposed) &&
|
|
1932
|
+
this.#modeFor(record.sessionID, settings) !== "off" &&
|
|
1576
1933
|
this.#tracker.active(record.sessionID) === undefined);
|
|
1934
|
+
this.#fruitless = 0;
|
|
1577
1935
|
}
|
|
1578
1936
|
catch (error) {
|
|
1579
1937
|
if (probeTicket !== undefined &&
|
|
@@ -1583,7 +1941,15 @@ export class Supervisor {
|
|
|
1583
1941
|
}
|
|
1584
1942
|
if (!(error instanceof RecoveryAbortedError))
|
|
1585
1943
|
logFailOpen("resume", error);
|
|
1944
|
+
this.#resumeCycles.delete(record.sessionID);
|
|
1945
|
+
// Work the user took over is gone for good; anything else waits again,
|
|
1946
|
+
// and a failed resume backs off like a fruitless wake rather than
|
|
1947
|
+
// retrying a second later.
|
|
1948
|
+
if (takenOver())
|
|
1949
|
+
return;
|
|
1950
|
+
this.#fruitless += 1;
|
|
1586
1951
|
await this.#rePark(record);
|
|
1952
|
+
await this.#armWake();
|
|
1587
1953
|
}
|
|
1588
1954
|
}
|
|
1589
1955
|
async #popParked(record) {
|
|
@@ -1596,16 +1962,11 @@ export class Supervisor {
|
|
|
1596
1962
|
}
|
|
1597
1963
|
}
|
|
1598
1964
|
async #rePark(record) {
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
});
|
|
1605
|
-
}
|
|
1606
|
-
catch (error) {
|
|
1607
|
-
logFailOpen("parked", error);
|
|
1608
|
-
}
|
|
1965
|
+
await this.#storeParked({
|
|
1966
|
+
...record,
|
|
1967
|
+
attempts: (record.attempts ?? 0) + 1,
|
|
1968
|
+
parkedAt: this.#options.clock(),
|
|
1969
|
+
});
|
|
1609
1970
|
}
|
|
1610
1971
|
// ---------- shared helpers ----------
|
|
1611
1972
|
async #isSessionAlive(sessionID) {
|
|
@@ -1695,6 +2056,14 @@ function candidateRef(candidate) {
|
|
|
1695
2056
|
...(candidate.variant !== undefined ? { variant: candidate.variant } : {}),
|
|
1696
2057
|
};
|
|
1697
2058
|
}
|
|
2059
|
+
/**
|
|
2060
|
+
* When a parked record can resume: the earliest usable candidate its
|
|
2061
|
+
* resume admission would route to (a standing pin wins over the tier).
|
|
2062
|
+
*/
|
|
2063
|
+
function parkedResumeAt(record, settings, usableAt) {
|
|
2064
|
+
const candidates = settings.pin !== null ? [settings.pin] : settings.tiers[record.tier];
|
|
2065
|
+
return earliestUsable(candidates.map(usableAt));
|
|
2066
|
+
}
|
|
1698
2067
|
/** ResourceKey for a model ref (no account level is observable in v1). */
|
|
1699
2068
|
function refKey(ref) {
|
|
1700
2069
|
// The host reports base-model calls with variant "default" (verified on
|