gestalt-mobile 0.35.1 → 0.35.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/assets/{index-BkQMGFPJ.js → index-SZM7lpix.js} +23 -23
- package/dist/client/index.html +1 -1
- package/dist/server/server/composition.contracts.js +936 -3
- package/dist/server/server/composition.js +175 -11
- package/dist/server/server/features/autopilot/application/service.js +944 -102
- package/dist/server/server/features/autopilot/domain/autopilot-session.js +42 -36
- package/dist/server/server/features/autopilot/domain/supervised-lifecycle.js +196 -1
- package/dist/server/server/features/autopilot/domain/supervision-protocol.js +98 -0
- package/dist/server/server/platform/codex/json-rpc-client.js +19 -2
- package/dist/server/server/platform/codex/session-runtime.js +87 -10
- package/dist/server/shared/contracts/autopilot-recovery.js +7 -0
- package/dist/server/shared/contracts/autopilot-wait-lease.js +13 -3
- package/package.json +1 -1
|
@@ -7,9 +7,9 @@ import { deriveSessionStatus } from '../../sessions/session-status.js';
|
|
|
7
7
|
import { createHash } from 'node:crypto';
|
|
8
8
|
import { AUTOPILOT_WAIT_MAX_MS, AUTOPILOT_WAIT_MIN_MS, } from '../../../../shared/contracts/autopilot-wait-lease.js';
|
|
9
9
|
import { autopilotSnapshot, disabledAutopilot, } from '../domain/autopilot-session.js';
|
|
10
|
-
import { classifyExecutorOutcome, decideSupervisedLifecycle, executorIdentity, validStructuredBlock, } from '../domain/supervised-lifecycle.js';
|
|
10
|
+
import { classifyExecutorOutcome, decideSupervisedLifecycle, executorIdentity, validStructuredBlock, checkpointTarget, } from '../domain/supervised-lifecycle.js';
|
|
11
11
|
import { classifyAgentActivity, decideAutopilot, executionComplete, } from './policy.js';
|
|
12
|
-
import { consumeObservableWake, consumeWaitDeadline, recordAutomaticContinuation, recoverSafetyPause, registerProactiveWait as registerProtocolWait, reportProbe, semanticProgressKey, startSupervisionProtocol, } from '../domain/supervision-protocol.js';
|
|
12
|
+
import { consumeCheckpointBoundary, consumeObsoleteWait, consumeObservableWake, consumeWaitDeadline, recordAutomaticContinuation, recoverSafetyPause, registerProactiveWait as registerProtocolWait, reportProbe, semanticProgressKey, startSupervisionProtocol, } from '../domain/supervision-protocol.js';
|
|
13
13
|
export class AutopilotCoordinator {
|
|
14
14
|
deps;
|
|
15
15
|
timers = new Map();
|
|
@@ -272,8 +272,10 @@ export class AutopilotCoordinator {
|
|
|
272
272
|
const prior = this.deps.store.find(sessionId) ?? disabledAutopilot(sessionId, now);
|
|
273
273
|
const nextFingerprint = fingerprint(currentPlan.plan);
|
|
274
274
|
const replacing = Boolean(prior.planIdentity && prior.planIdentity !== currentPlan.identity);
|
|
275
|
-
if (replacing)
|
|
275
|
+
if (replacing) {
|
|
276
276
|
this.cancelWaitTimer(sessionId);
|
|
277
|
+
this.parkedSubscriptions.delete(sessionId);
|
|
278
|
+
}
|
|
277
279
|
if (prior.requestedEnabled &&
|
|
278
280
|
prior.planIdentity === currentPlan.identity &&
|
|
279
281
|
prior.state !== 'attentionRequired')
|
|
@@ -290,12 +292,9 @@ export class AutopilotCoordinator {
|
|
|
290
292
|
stopReason: null,
|
|
291
293
|
executor: undefined,
|
|
292
294
|
blocking: undefined,
|
|
293
|
-
|
|
294
|
-
// budget from a different semantic plan identity. Re-enabling the same
|
|
295
|
-
// plan still preserves its non-terminal protocol state.
|
|
296
|
-
supervision: replacing
|
|
295
|
+
supervision: recoverSafetyPause(consumeObsoleteWait(replacing
|
|
297
296
|
? startSupervisionProtocol(this.progressKey(sessionId))
|
|
298
|
-
:
|
|
297
|
+
: (prior.supervision ?? startSupervisionProtocol(this.progressKey(sessionId))), this.progressKey(sessionId)), this.progressKey(sessionId)),
|
|
299
298
|
updatedAt: now,
|
|
300
299
|
};
|
|
301
300
|
this.persist(next);
|
|
@@ -314,6 +313,7 @@ export class AutopilotCoordinator {
|
|
|
314
313
|
if (!prior.requestedEnabled &&
|
|
315
314
|
prior.state === 'disabled' &&
|
|
316
315
|
!cancelled &&
|
|
316
|
+
!prior.supervision?.waitLease &&
|
|
317
317
|
(!currentPlan || prior.planIdentity === currentPlan.identity))
|
|
318
318
|
return this.snapshot(sessionId);
|
|
319
319
|
const next = {
|
|
@@ -326,12 +326,13 @@ export class AutopilotCoordinator {
|
|
|
326
326
|
...(cancelled ? { lastControlId: null } : {}),
|
|
327
327
|
stopReason: 'manualDisabled',
|
|
328
328
|
blocking: undefined,
|
|
329
|
-
supervision:
|
|
329
|
+
supervision: consumeObsoleteWait(prior.supervision ?? startSupervisionProtocol(this.progressKey(sessionId)), this.progressKey(sessionId)),
|
|
330
330
|
updatedAt: now,
|
|
331
331
|
};
|
|
332
332
|
this.persist(next, cancelled);
|
|
333
333
|
this.cancelTimer(sessionId);
|
|
334
334
|
this.cancelWaitTimer(sessionId);
|
|
335
|
+
this.parkedSubscriptions.delete(sessionId);
|
|
335
336
|
return this.snapshot(sessionId);
|
|
336
337
|
}
|
|
337
338
|
cancel(sessionId, reason) {
|
|
@@ -349,10 +350,12 @@ export class AutopilotCoordinator {
|
|
|
349
350
|
...(cancelled ? { lastControlId: null } : {}),
|
|
350
351
|
stopReason: reason,
|
|
351
352
|
blocking: undefined,
|
|
353
|
+
supervision: consumeObsoleteWait(prior.supervision ?? startSupervisionProtocol(this.progressKey(sessionId)), this.progressKey(sessionId)),
|
|
352
354
|
updatedAt: now,
|
|
353
355
|
}, cancelled);
|
|
354
356
|
this.cancelTimer(sessionId);
|
|
355
357
|
this.cancelWaitTimer(sessionId);
|
|
358
|
+
this.parkedSubscriptions.delete(sessionId);
|
|
356
359
|
}
|
|
357
360
|
evaluate(sessionId) {
|
|
358
361
|
const prior = this.deps.store.find(sessionId);
|
|
@@ -526,7 +529,15 @@ export class AutopilotCoordinator {
|
|
|
526
529
|
(state.checkpoints.terminalReviewAccepted ? 'terminalReviewAccepted' : 'l1Accepted');
|
|
527
530
|
this.persist({
|
|
528
531
|
...state,
|
|
529
|
-
|
|
532
|
+
// The failure audit is durable history, but it is not a permanent
|
|
533
|
+
// health condition. A matching final or reconstructed runtime has
|
|
534
|
+
// completed the bounded recovery, so clear the live indicator.
|
|
535
|
+
checkpoints: {
|
|
536
|
+
...state.checkpoints,
|
|
537
|
+
pendingTurnId: null,
|
|
538
|
+
pendingKind: null,
|
|
539
|
+
checkpointHandoffFailed: false,
|
|
540
|
+
},
|
|
530
541
|
updatedAt: occurredAt,
|
|
531
542
|
}, undefined, [
|
|
532
543
|
{
|
|
@@ -555,7 +566,12 @@ export class AutopilotCoordinator {
|
|
|
555
566
|
});
|
|
556
567
|
this.flushOutbox(sessionId);
|
|
557
568
|
}
|
|
558
|
-
|
|
569
|
+
// The persisted checkpoint is the continuation trigger. It becomes
|
|
570
|
+
// actionable only after the root final has closed its owning turn.
|
|
571
|
+
if (checkpointBoundary)
|
|
572
|
+
this.evaluate(sessionId);
|
|
573
|
+
else
|
|
574
|
+
this.activitySettled(sessionId, 'rootFinalAttempt');
|
|
559
575
|
return finalAllowed;
|
|
560
576
|
}
|
|
561
577
|
/** Records an already validated root-only checkpoint; it never changes Org state. */
|
|
@@ -569,28 +585,50 @@ export class AutopilotCoordinator {
|
|
|
569
585
|
return false;
|
|
570
586
|
const reportedL1Ids = previous?.reportedL1Ids ?? [];
|
|
571
587
|
const reportedL2Ids = previous?.reportedL2Ids ?? [];
|
|
572
|
-
const
|
|
588
|
+
const canonicalPosition = checkpoint.kind === 'terminalReviewAccepted' ? 'terminal' : checkpoint.position;
|
|
589
|
+
const target = checkpoint.kind === 'l2Completed'
|
|
590
|
+
? checkpointTarget('l2', checkpoint.l1Id, checkpoint.l2Id)
|
|
591
|
+
: checkpoint.kind === 'l1Accepted'
|
|
592
|
+
? checkpointTarget('l1', checkpoint.l1Id)
|
|
593
|
+
: checkpointTarget('terminal');
|
|
594
|
+
const completionEpoch = previous?.completionEpochs?.find((entry) => entry.target === target);
|
|
595
|
+
const epoch = completionEpoch?.epoch ?? 0;
|
|
596
|
+
const key = createHash('sha256')
|
|
597
|
+
.update(JSON.stringify([retained.identity, checkpoint.kind, canonicalPosition, epoch]))
|
|
598
|
+
.digest('hex');
|
|
573
599
|
const acceptedKeys = previous?.acceptedKeys ?? [];
|
|
574
600
|
if (acceptedKeys.includes(key))
|
|
575
|
-
return
|
|
601
|
+
return previous?.pendingTurnId === turnId && previous.pendingKind === checkpoint.kind;
|
|
576
602
|
const l2Key = checkpoint.kind === 'l2Completed' ? JSON.stringify([checkpoint.l1Id, checkpoint.l2Id]) : null;
|
|
577
|
-
if (l2Key && reportedL2Ids.includes(l2Key))
|
|
578
|
-
return false;
|
|
579
|
-
if (checkpoint.kind === 'l1Accepted' && reportedL1Ids.includes(checkpoint.l1Id))
|
|
580
|
-
return false;
|
|
581
603
|
if (checkpoint.kind === 'terminalReviewAccepted' && previous?.terminalReviewAccepted)
|
|
582
604
|
return false;
|
|
605
|
+
if (completionEpoch?.completed && !completionEpoch.reopened)
|
|
606
|
+
return false;
|
|
607
|
+
const completionEpochs = [
|
|
608
|
+
...(previous?.completionEpochs ?? []).filter((entry) => entry.target !== target),
|
|
609
|
+
{ target, epoch, reopened: false, completed: true },
|
|
610
|
+
];
|
|
583
611
|
const checkpoints = {
|
|
584
612
|
protocolVersion: 1,
|
|
585
613
|
planIdentity: retained.identity,
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
614
|
+
completionEpochs: boundEpochs(completionEpochs, target),
|
|
615
|
+
reportedL2Ids: l2Key ? boundedUnique([...reportedL2Ids, l2Key], 512) : reportedL2Ids,
|
|
616
|
+
reportedL1Ids: checkpoint.kind === 'l1Accepted'
|
|
617
|
+
? boundedUnique([...reportedL1Ids, checkpoint.l1Id], 128)
|
|
618
|
+
: reportedL1Ids,
|
|
619
|
+
acceptedKeys: boundedUnique([...acceptedKeys, key], 768),
|
|
589
620
|
pendingTurnId: turnId,
|
|
590
621
|
pendingKind: checkpoint.kind,
|
|
622
|
+
checkpointHandoffFailed: false,
|
|
591
623
|
terminalReviewAccepted: checkpoint.kind === 'terminalReviewAccepted' || previous?.terminalReviewAccepted === true,
|
|
592
624
|
};
|
|
593
|
-
|
|
625
|
+
const checkpointed = { ...prior, checkpoints, updatedAt: occurredAt };
|
|
626
|
+
this.persist({
|
|
627
|
+
...checkpointed,
|
|
628
|
+
supervision: checkpointed.supervision
|
|
629
|
+
? consumeCheckpointBoundary(checkpointed.supervision, this.progressKeyFor(sessionId, checkpointed, retained))
|
|
630
|
+
: checkpointed.supervision,
|
|
631
|
+
}, undefined, [
|
|
594
632
|
{
|
|
595
633
|
sessionId,
|
|
596
634
|
type: checkpoint.kind === 'l2Completed'
|
|
@@ -611,13 +649,40 @@ export class AutopilotCoordinator {
|
|
|
611
649
|
occurredAt,
|
|
612
650
|
},
|
|
613
651
|
]);
|
|
614
|
-
//
|
|
615
|
-
//
|
|
616
|
-
//
|
|
617
|
-
|
|
618
|
-
|
|
652
|
+
// Persistence and acknowledgement stay on the short checkpoint path. Any
|
|
653
|
+
// pre-boundary lease is already obsolete, but continuation is deliberately
|
|
654
|
+
// deferred until the matching root final closes the turn.
|
|
655
|
+
this.cancelWaitTimer(sessionId);
|
|
656
|
+
this.parkedSubscriptions.delete(sessionId);
|
|
657
|
+
this.cancelTimer(sessionId);
|
|
619
658
|
return true;
|
|
620
659
|
}
|
|
660
|
+
/**
|
|
661
|
+
* Records a failed checkpoint transport handoff without releasing its root
|
|
662
|
+
* boundary. The matching final (or a runtime recovery) remains the only
|
|
663
|
+
* authority allowed to schedule the next root continuation.
|
|
664
|
+
*/
|
|
665
|
+
checkpointHandoffFailed(sessionId, requestTurnId) {
|
|
666
|
+
const prior = this.deps.store.find(sessionId);
|
|
667
|
+
if (!prior?.checkpoints?.pendingTurnId ||
|
|
668
|
+
prior.checkpoints.pendingTurnId !== requestTurnId ||
|
|
669
|
+
prior.checkpoints.checkpointHandoffFailed)
|
|
670
|
+
return false;
|
|
671
|
+
this.persist({
|
|
672
|
+
...prior,
|
|
673
|
+
checkpoints: { ...prior.checkpoints, checkpointHandoffFailed: true },
|
|
674
|
+
updatedAt: this.deps.now(),
|
|
675
|
+
});
|
|
676
|
+
return true;
|
|
677
|
+
}
|
|
678
|
+
/** A replacement runtime is the explicit substitute for the lost root final. */
|
|
679
|
+
recoverCheckpointHandoff(sessionId) {
|
|
680
|
+
const pendingTurnId = this.deps.store.find(sessionId)?.checkpoints?.pendingTurnId;
|
|
681
|
+
if (!pendingTurnId)
|
|
682
|
+
return false;
|
|
683
|
+
this.checkpointHandoffFailed(sessionId, pendingTurnId);
|
|
684
|
+
return this.turnCompleted(sessionId);
|
|
685
|
+
}
|
|
621
686
|
/** Handles only plan lifecycle safety; ordinary plan mutations are ignored. */
|
|
622
687
|
planStatusChanged(sessionId) {
|
|
623
688
|
const prior = this.deps.store.find(sessionId);
|
|
@@ -629,9 +694,21 @@ export class AutopilotCoordinator {
|
|
|
629
694
|
return;
|
|
630
695
|
}
|
|
631
696
|
if (prior.planIdentity && prior.planIdentity !== plan.identity) {
|
|
697
|
+
this.supersedeExecutorCommands(sessionId, plan.identity, plan.plan);
|
|
632
698
|
this.cancel(sessionId, 'planReplaced');
|
|
633
699
|
return;
|
|
634
700
|
}
|
|
701
|
+
this.supersedeExecutorCommands(sessionId, plan.identity, plan.plan);
|
|
702
|
+
// Supersession persists independently. Re-read before reopening epochs so
|
|
703
|
+
// this update cannot restore commands from the stale pre-supersession row.
|
|
704
|
+
const current = this.deps.store.find(sessionId) ?? prior;
|
|
705
|
+
const checkpointEpochs = this.reopenCheckpointEpochs(current, plan.plan);
|
|
706
|
+
if (checkpointEpochs !== current.checkpoints?.completionEpochs)
|
|
707
|
+
this.persist({
|
|
708
|
+
...current,
|
|
709
|
+
checkpoints: { ...current.checkpoints, completionEpochs: checkpointEpochs },
|
|
710
|
+
updatedAt: this.deps.now(),
|
|
711
|
+
});
|
|
635
712
|
if (this.semanticEvent(sessionId, 'planChanged') ||
|
|
636
713
|
this.semanticEvent(sessionId, 'reviewChanged'))
|
|
637
714
|
return;
|
|
@@ -645,6 +722,31 @@ export class AutopilotCoordinator {
|
|
|
645
722
|
this.planEventKeys.set(sessionId, eventKey);
|
|
646
723
|
this.evaluate(sessionId);
|
|
647
724
|
}
|
|
725
|
+
/**
|
|
726
|
+
* Plan refinements are not a new completion. Only a completed target that
|
|
727
|
+
* becomes ineligible opens its next reportable epoch.
|
|
728
|
+
*/
|
|
729
|
+
reopenCheckpointEpochs(state, plan) {
|
|
730
|
+
const epochs = state.checkpoints?.completionEpochs;
|
|
731
|
+
if (!epochs?.length)
|
|
732
|
+
return epochs;
|
|
733
|
+
let changed = false;
|
|
734
|
+
const next = epochs.map((entry) => {
|
|
735
|
+
const [kind, l1Id, l2Id] = JSON.parse(entry.target);
|
|
736
|
+
const l1 = plan.steps.find((step) => step.id === l1Id);
|
|
737
|
+
const eligible = kind === 'l2'
|
|
738
|
+
? Boolean(l1?.children.some((child) => child.id === l2Id && child.state === 'DONE'))
|
|
739
|
+
: kind === 'l1'
|
|
740
|
+
? l1?.state === 'DONE' && l1.reviewStatus === 'REVIEWED'
|
|
741
|
+
: plan.executionComplete === true;
|
|
742
|
+
if (!eligible && !entry.reopened) {
|
|
743
|
+
changed = true;
|
|
744
|
+
return { ...entry, epoch: entry.epoch + 1, reopened: true, completed: false };
|
|
745
|
+
}
|
|
746
|
+
return entry;
|
|
747
|
+
});
|
|
748
|
+
return changed ? next : epochs;
|
|
749
|
+
}
|
|
648
750
|
/**
|
|
649
751
|
* Records a validated, session-private supervision request. It intentionally
|
|
650
752
|
* does not discover plans: composition must first bind the update to this
|
|
@@ -670,6 +772,7 @@ export class AutopilotCoordinator {
|
|
|
670
772
|
if (replacing) {
|
|
671
773
|
this.cancelTimer(sessionId);
|
|
672
774
|
this.cancelWaitTimer(sessionId);
|
|
775
|
+
this.parkedSubscriptions.delete(sessionId);
|
|
673
776
|
}
|
|
674
777
|
const next = {
|
|
675
778
|
...prior,
|
|
@@ -684,9 +787,9 @@ export class AutopilotCoordinator {
|
|
|
684
787
|
stopReason: null,
|
|
685
788
|
executor: undefined,
|
|
686
789
|
blocking: undefined,
|
|
687
|
-
supervision: replacing
|
|
790
|
+
supervision: recoverSafetyPause(consumeObsoleteWait(replacing
|
|
688
791
|
? startSupervisionProtocol(this.progressKey(sessionId))
|
|
689
|
-
:
|
|
792
|
+
: (prior.supervision ?? startSupervisionProtocol(this.progressKey(sessionId))), this.progressKey(sessionId)), this.progressKey(sessionId)),
|
|
690
793
|
updatedAt: now,
|
|
691
794
|
};
|
|
692
795
|
this.persist(next, cancelled);
|
|
@@ -707,20 +810,10 @@ export class AutopilotCoordinator {
|
|
|
707
810
|
this.evaluate(sessionId);
|
|
708
811
|
return;
|
|
709
812
|
}
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
child.state,
|
|
715
|
-
child.reason,
|
|
716
|
-
child.continuationGeneration ?? 0,
|
|
717
|
-
(child.ownedProcesses ?? []).map((process) => [
|
|
718
|
-
process.processId,
|
|
719
|
-
process.state,
|
|
720
|
-
process.ownership,
|
|
721
|
-
]),
|
|
722
|
-
]),
|
|
723
|
-
});
|
|
813
|
+
// Reuse the canonical semantic projection: child order, timestamps, and
|
|
814
|
+
// activity prose cannot manufacture an extra wake, while a child-only
|
|
815
|
+
// lifecycle transition remains visible with an unchanged root.
|
|
816
|
+
const eventKey = this.progressKey(sessionId);
|
|
724
817
|
if (this.activityEventKeys.get(sessionId) === eventKey)
|
|
725
818
|
return;
|
|
726
819
|
this.activityEventKeys.set(sessionId, eventKey);
|
|
@@ -765,10 +858,10 @@ export class AutopilotCoordinator {
|
|
|
765
858
|
this.completionTimers.get(sessionId)?.();
|
|
766
859
|
this.completionTimers.set(sessionId, this.deps.schedule(() => {
|
|
767
860
|
this.completionTimers.delete(sessionId);
|
|
768
|
-
const current = this.deps.store.find(sessionId);
|
|
769
|
-
if (!current?.requestedEnabled || current.state !== 'monitoring')
|
|
770
|
-
return;
|
|
771
861
|
this.enqueue(sessionId, async () => {
|
|
862
|
+
const current = this.deps.store.find(sessionId);
|
|
863
|
+
if (!current?.requestedEnabled || current.state !== 'monitoring')
|
|
864
|
+
return;
|
|
772
865
|
if (!(await this.enforceSupervisedLifecycle(sessionId, event)))
|
|
773
866
|
this.evaluate(sessionId);
|
|
774
867
|
});
|
|
@@ -776,6 +869,8 @@ export class AutopilotCoordinator {
|
|
|
776
869
|
}
|
|
777
870
|
manualSend(sessionId) {
|
|
778
871
|
this.cancelTimer(sessionId);
|
|
872
|
+
this.cancelWaitTimer(sessionId);
|
|
873
|
+
this.parkedSubscriptions.delete(sessionId);
|
|
779
874
|
const prior = this.deps.store.find(sessionId);
|
|
780
875
|
if (!prior || !prior.requestedEnabled)
|
|
781
876
|
return;
|
|
@@ -787,6 +882,7 @@ export class AutopilotCoordinator {
|
|
|
787
882
|
generation: prior.generation + 1,
|
|
788
883
|
nextEvaluationAt: null,
|
|
789
884
|
lastControlId: null,
|
|
885
|
+
supervision: consumeObsoleteWait(prior.supervision ?? startSupervisionProtocol(this.progressKey(sessionId)), this.progressKey(sessionId)),
|
|
790
886
|
updatedAt: now,
|
|
791
887
|
}, cancelled);
|
|
792
888
|
}
|
|
@@ -843,6 +939,28 @@ export class AutopilotCoordinator {
|
|
|
843
939
|
const current = this.deps.store.find(sessionId);
|
|
844
940
|
if (!current || current.generation !== generation || !current.requestedEnabled)
|
|
845
941
|
return;
|
|
942
|
+
if (current.executor?.replacement &&
|
|
943
|
+
!this.validReplacement(sessionId, current.executor.replacement)) {
|
|
944
|
+
const now = this.deps.now();
|
|
945
|
+
const cancelled = this.cancelScheduledControl(current, now);
|
|
946
|
+
this.persist({
|
|
947
|
+
...current,
|
|
948
|
+
state: 'monitoring',
|
|
949
|
+
executor: {
|
|
950
|
+
...current.executor,
|
|
951
|
+
replacement: undefined,
|
|
952
|
+
commands: current.executor.commands?.map((command) => ['scheduled', 'issued'].includes(command.status)
|
|
953
|
+
? { ...command, status: 'superseded', updatedAt: now }
|
|
954
|
+
: command),
|
|
955
|
+
},
|
|
956
|
+
...(cancelled ? { generation: current.generation + 1, lastControlId: null } : {}),
|
|
957
|
+
nextEvaluationAt: null,
|
|
958
|
+
updatedAt: now,
|
|
959
|
+
}, cancelled);
|
|
960
|
+
this.audit(sessionId, 'autopilot.executor-replacement-superseded', {});
|
|
961
|
+
this.evaluate(sessionId);
|
|
962
|
+
return;
|
|
963
|
+
}
|
|
846
964
|
const session = this.deps.session(sessionId);
|
|
847
965
|
if (session?.activeTurnId ||
|
|
848
966
|
this.deps.pendingInteraction(sessionId) ||
|
|
@@ -862,6 +980,7 @@ export class AutopilotCoordinator {
|
|
|
862
980
|
const retained = this.deps.plan(sessionId);
|
|
863
981
|
if (this.deps.executorController &&
|
|
864
982
|
retained &&
|
|
983
|
+
!current.executor?.replacement &&
|
|
865
984
|
this.currentExecutor(sessionId, retained.plan, current.consecutiveNoProgress, current.executor)) {
|
|
866
985
|
const now = this.deps.now();
|
|
867
986
|
const cancelled = this.cancelScheduledControl(current, now);
|
|
@@ -955,6 +1074,19 @@ export class AutopilotCoordinator {
|
|
|
955
1074
|
this.completionTimers.delete(sessionId);
|
|
956
1075
|
this.executorTimers.get(sessionId)?.();
|
|
957
1076
|
this.executorTimers.delete(sessionId);
|
|
1077
|
+
const state = this.deps.store.find(sessionId);
|
|
1078
|
+
const executor = state?.executor;
|
|
1079
|
+
if (state && executor?.commands?.some((command) => command.status === 'scheduled'))
|
|
1080
|
+
this.persist({
|
|
1081
|
+
...state,
|
|
1082
|
+
executor: {
|
|
1083
|
+
...executor,
|
|
1084
|
+
commands: executor.commands.map((command) => command.status === 'scheduled'
|
|
1085
|
+
? { ...command, status: 'cancelled', updatedAt: this.deps.now() }
|
|
1086
|
+
: command),
|
|
1087
|
+
},
|
|
1088
|
+
updatedAt: this.deps.now(),
|
|
1089
|
+
});
|
|
958
1090
|
}
|
|
959
1091
|
cancelWaitTimer(sessionId) {
|
|
960
1092
|
this.waitTimers.get(sessionId)?.();
|
|
@@ -967,6 +1099,12 @@ export class AutopilotCoordinator {
|
|
|
967
1099
|
const state = this.deps.store.find(sessionId);
|
|
968
1100
|
if (!state?.requestedEnabled || !state.supervision)
|
|
969
1101
|
return;
|
|
1102
|
+
const lease = state.supervision.waitLease;
|
|
1103
|
+
if (state.supervision.outcome !== 'parked' ||
|
|
1104
|
+
!lease ||
|
|
1105
|
+
lease.id !== leaseId ||
|
|
1106
|
+
lease.resumeAt !== resumeAt)
|
|
1107
|
+
return;
|
|
970
1108
|
const protocol = consumeWaitDeadline(state.supervision, leaseId, this.deps.now());
|
|
971
1109
|
if (protocol === state.supervision) {
|
|
972
1110
|
this.armWaitDeadline(sessionId, leaseId, resumeAt);
|
|
@@ -988,6 +1126,10 @@ export class AutopilotCoordinator {
|
|
|
988
1126
|
return false;
|
|
989
1127
|
const executor = this.currentExecutor(sessionId, retained.plan, state.consecutiveNoProgress, state.executor);
|
|
990
1128
|
const executorChanged = executor ? this.persistExecutor(sessionId, executor) : false;
|
|
1129
|
+
if (executor && (await this.reconcileSplitBrain(sessionId, executor)))
|
|
1130
|
+
return true;
|
|
1131
|
+
if (executor)
|
|
1132
|
+
this.supersedeExecutorCommands(sessionId, retained.identity, retained.plan, executor);
|
|
991
1133
|
if (executorChanged && this.wakeForExecutorChange(sessionId, executor))
|
|
992
1134
|
return true;
|
|
993
1135
|
const decision = decideSupervisedLifecycle({
|
|
@@ -1017,16 +1159,10 @@ export class AutopilotCoordinator {
|
|
|
1017
1159
|
const alreadyMonitoring = executor?.ownedProcesses.some((process) => process.processId === processId &&
|
|
1018
1160
|
process.ownership === 'supervisor' &&
|
|
1019
1161
|
process.state === 'detached-active');
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
...executor,
|
|
1025
|
-
ownedProcesses: executor.ownedProcesses.map((process) => process.processId === processId
|
|
1026
|
-
? { ...process, ownership: 'supervisor', state: 'detached-active' }
|
|
1027
|
-
: process),
|
|
1028
|
-
});
|
|
1029
|
-
if (!alreadyMonitoring)
|
|
1162
|
+
const transferred = alreadyMonitoring || !executor
|
|
1163
|
+
? alreadyMonitoring
|
|
1164
|
+
: await this.performProcessAction(sessionId, executor, decision.action.process, 'transfer');
|
|
1165
|
+
if (transferred && !alreadyMonitoring)
|
|
1030
1166
|
this.audit(sessionId, 'autopilot.process-monitoring', {
|
|
1031
1167
|
threadId: decision.action.process.ownerThreadId,
|
|
1032
1168
|
processId,
|
|
@@ -1035,42 +1171,41 @@ export class AutopilotCoordinator {
|
|
|
1035
1171
|
return true;
|
|
1036
1172
|
}
|
|
1037
1173
|
case 'consumeProcessResult': {
|
|
1038
|
-
const
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1174
|
+
const processAction = decision.action;
|
|
1175
|
+
const processId = processAction.processId;
|
|
1176
|
+
const process = executor?.ownedProcesses.find((candidate) => candidate.processId === processId &&
|
|
1177
|
+
candidate.ownerThreadId === processAction.threadId &&
|
|
1178
|
+
candidate.state === 'exited-awaiting-result');
|
|
1179
|
+
const consumed = process && executor
|
|
1180
|
+
? await this.performProcessAction(sessionId, executor, process, 'consume')
|
|
1181
|
+
: false;
|
|
1182
|
+
if (consumed)
|
|
1183
|
+
this.audit(sessionId, 'autopilot.process-result-consumed', {
|
|
1184
|
+
threadId: processAction.threadId,
|
|
1185
|
+
processId,
|
|
1186
|
+
resultArtifact: processAction.resultArtifact,
|
|
1044
1187
|
});
|
|
1045
|
-
|
|
1046
|
-
threadId: decision.action.threadId,
|
|
1047
|
-
processId,
|
|
1048
|
-
resultArtifact: decision.action.resultArtifact,
|
|
1049
|
-
});
|
|
1050
|
-
if (executor)
|
|
1188
|
+
if (consumed && executor)
|
|
1051
1189
|
this.armExecutorContinuation(sessionId, this.deps.policy.executorContinuationBaseMs, executor.threadId, executor.continuationGeneration + 1, {
|
|
1052
1190
|
kind: 'processExited',
|
|
1053
1191
|
processId,
|
|
1054
|
-
resultArtifact:
|
|
1192
|
+
resultArtifact: processAction.resultArtifact,
|
|
1055
1193
|
});
|
|
1056
1194
|
return true;
|
|
1057
1195
|
}
|
|
1058
1196
|
case 'terminateProcess': {
|
|
1059
|
-
const
|
|
1060
|
-
const
|
|
1197
|
+
const processAction = decision.action;
|
|
1198
|
+
const processId = processAction.processId;
|
|
1199
|
+
const process = executor?.ownedProcesses.find((candidate) => candidate.processId === processId && candidate.ownerThreadId === processAction.threadId);
|
|
1200
|
+
const terminated = process && executor
|
|
1201
|
+
? await this.performProcessAction(sessionId, executor, process, 'terminate')
|
|
1202
|
+
: false;
|
|
1061
1203
|
if (terminated)
|
|
1062
1204
|
this.audit(sessionId, 'autopilot.process-terminated', {
|
|
1063
|
-
threadId:
|
|
1205
|
+
threadId: processAction.threadId,
|
|
1064
1206
|
processId,
|
|
1065
1207
|
reason: 'resourceBudget',
|
|
1066
1208
|
});
|
|
1067
|
-
if (terminated && executor)
|
|
1068
|
-
this.persistExecutor(sessionId, {
|
|
1069
|
-
...executor,
|
|
1070
|
-
ownedProcesses: executor.ownedProcesses.map((process) => process.processId === processId
|
|
1071
|
-
? { ...process, state: 'terminated-for-budget' }
|
|
1072
|
-
: process),
|
|
1073
|
-
});
|
|
1074
1209
|
if (terminated && executor)
|
|
1075
1210
|
this.armExecutorContinuation(sessionId, this.deps.policy.executorContinuationBaseMs, executor.threadId, executor.continuationGeneration + 1, { kind: 'processResourceLimit', processId });
|
|
1076
1211
|
else
|
|
@@ -1090,12 +1225,9 @@ export class AutopilotCoordinator {
|
|
|
1090
1225
|
.activity(sessionId)
|
|
1091
1226
|
?.subagents.filter((candidate) => candidate.canonicalPosition === canonicalPosition)
|
|
1092
1227
|
.sort((left, right) => (right.continuationGeneration ?? 1) - (left.continuationGeneration ?? 1) ||
|
|
1093
|
-
Date.parse(right.lastActivityAt) - Date.parse(left.lastActivityAt)
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
child.state === 'disconnected' ||
|
|
1097
|
-
child.outcome === 'cancelled' ||
|
|
1098
|
-
child.outcome === 'failed') {
|
|
1228
|
+
Date.parse(right.lastActivityAt) - Date.parse(left.lastActivityAt) ||
|
|
1229
|
+
(left.threadId ?? left.id).localeCompare(right.threadId ?? right.id))[0];
|
|
1230
|
+
if (!child) {
|
|
1099
1231
|
if (persisted?.canonicalPosition !== canonicalPosition ||
|
|
1100
1232
|
persisted.outcome === 'cancelled' ||
|
|
1101
1233
|
persisted.outcome === 'failed')
|
|
@@ -1107,6 +1239,13 @@ export class AutopilotCoordinator {
|
|
|
1107
1239
|
ownedProcesses: refreshPersistedProcesses(persisted.ownedProcesses, [], this.deps.now()),
|
|
1108
1240
|
};
|
|
1109
1241
|
}
|
|
1242
|
+
// A roster entry for a newer generation is authoritative even when that
|
|
1243
|
+
// generation is disconnected. Never fall back to an older persisted
|
|
1244
|
+
// writer; incomplete owner metadata instead fences all continuation.
|
|
1245
|
+
if (!child.taskPath || !child.canonicalTaskName)
|
|
1246
|
+
return undefined;
|
|
1247
|
+
if (child.outcome === 'cancelled' || child.outcome === 'failed')
|
|
1248
|
+
return undefined;
|
|
1110
1249
|
const activeL2 = step.children.find((candidate) => candidate.state === 'WIP');
|
|
1111
1250
|
const outcome = classifyExecutorOutcome({
|
|
1112
1251
|
objectiveComplete: step.state === 'DONE',
|
|
@@ -1127,8 +1266,270 @@ export class AutopilotCoordinator {
|
|
|
1127
1266
|
continuationCount: persisted?.canonicalPosition === canonicalPosition
|
|
1128
1267
|
? persisted.continuationCount
|
|
1129
1268
|
: continuationCount,
|
|
1269
|
+
...(persisted?.replacement &&
|
|
1270
|
+
(child.continuationGeneration ?? 1) < persisted.replacement.generation
|
|
1271
|
+
? { replacement: persisted.replacement }
|
|
1272
|
+
: {}),
|
|
1273
|
+
...(persisted?.replacement &&
|
|
1274
|
+
(child.continuationGeneration ?? 1) >= persisted.replacement.generation
|
|
1275
|
+
? { resumeFailures: 0 }
|
|
1276
|
+
: persisted?.resumeFailures
|
|
1277
|
+
? { resumeFailures: persisted.resumeFailures }
|
|
1278
|
+
: {}),
|
|
1279
|
+
...(persisted?.commands ? { commands: persisted.commands } : {}),
|
|
1130
1280
|
};
|
|
1131
1281
|
}
|
|
1282
|
+
/** A fresh roster may expose two physical writers for one canonical L1. */
|
|
1283
|
+
async reconcileSplitBrain(sessionId, executor) {
|
|
1284
|
+
const activity = this.deps.activity(sessionId);
|
|
1285
|
+
if (activity?.confidence !== 'fresh')
|
|
1286
|
+
return true;
|
|
1287
|
+
const candidates = activity.subagents.filter((child) => child.canonicalPosition === executor.canonicalPosition);
|
|
1288
|
+
if (candidates.length < 2)
|
|
1289
|
+
return false;
|
|
1290
|
+
if (candidates.some((child) => !child.threadId ||
|
|
1291
|
+
!child.taskPath ||
|
|
1292
|
+
child.canonicalTaskName !== executor.canonicalTaskName ||
|
|
1293
|
+
child.continuationGeneration === undefined)) {
|
|
1294
|
+
this.audit(sessionId, 'autopilot.executor-reconciliation-incomplete', {});
|
|
1295
|
+
return true;
|
|
1296
|
+
}
|
|
1297
|
+
const ordered = [...candidates].sort((left, right) => (right.continuationGeneration ?? 0) - (left.continuationGeneration ?? 0) ||
|
|
1298
|
+
Date.parse(right.lastActivityAt) - Date.parse(left.lastActivityAt) ||
|
|
1299
|
+
(left.threadId ?? left.id).localeCompare(right.threadId ?? right.id));
|
|
1300
|
+
const owner = ordered[0];
|
|
1301
|
+
const obsolete = ordered
|
|
1302
|
+
.slice(1)
|
|
1303
|
+
.filter((child) => child.state === 'working' ||
|
|
1304
|
+
child.state === 'awaitingAgent' ||
|
|
1305
|
+
child.ownedProcesses?.some((process) => process.state === 'running' || process.state === 'detached-active'));
|
|
1306
|
+
const controller = this.deps.executorController;
|
|
1307
|
+
const ownerUnavailable = owner.state === 'disconnected' || owner.outcome === 'cancelled' || owner.outcome === 'failed';
|
|
1308
|
+
if (!obsolete.length) {
|
|
1309
|
+
if (!ownerUnavailable)
|
|
1310
|
+
return false;
|
|
1311
|
+
if (controller)
|
|
1312
|
+
await controller.refresh(sessionId);
|
|
1313
|
+
return true;
|
|
1314
|
+
}
|
|
1315
|
+
if (!controller)
|
|
1316
|
+
return true;
|
|
1317
|
+
for (const child of obsolete) {
|
|
1318
|
+
try {
|
|
1319
|
+
const interrupted = await controller.interrupt(sessionId, child.threadId);
|
|
1320
|
+
this.audit(sessionId, interrupted ? 'autopilot.executor-superseded' : 'autopilot.executor-interrupt-failed', {
|
|
1321
|
+
ownerThreadId: owner.threadId,
|
|
1322
|
+
obsoleteThreadId: child.threadId,
|
|
1323
|
+
});
|
|
1324
|
+
}
|
|
1325
|
+
catch {
|
|
1326
|
+
this.audit(sessionId, 'autopilot.executor-interrupt-failed', {
|
|
1327
|
+
ownerThreadId: owner.threadId,
|
|
1328
|
+
obsoleteThreadId: child.threadId,
|
|
1329
|
+
});
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
await controller.refresh(sessionId);
|
|
1333
|
+
return true;
|
|
1334
|
+
}
|
|
1335
|
+
executorCommand(fence, executor, generation, trigger) {
|
|
1336
|
+
const commandId = createHash('sha256')
|
|
1337
|
+
.update(JSON.stringify([
|
|
1338
|
+
fence.planIdentity,
|
|
1339
|
+
fence.planFingerprint,
|
|
1340
|
+
fence.canonicalPosition,
|
|
1341
|
+
executor.canonicalTaskName,
|
|
1342
|
+
executor.taskPath,
|
|
1343
|
+
fence.executorThreadId,
|
|
1344
|
+
generation,
|
|
1345
|
+
...(executor.resumeFailures ? [executor.resumeFailures] : []),
|
|
1346
|
+
trigger.kind,
|
|
1347
|
+
]))
|
|
1348
|
+
.digest('hex');
|
|
1349
|
+
const now = this.deps.now();
|
|
1350
|
+
return {
|
|
1351
|
+
commandId,
|
|
1352
|
+
status: 'scheduled',
|
|
1353
|
+
planIdentity: fence.planIdentity,
|
|
1354
|
+
planFingerprint: fence.planFingerprint,
|
|
1355
|
+
canonicalPosition: fence.canonicalPosition ?? executor.canonicalPosition,
|
|
1356
|
+
canonicalTaskName: executor.canonicalTaskName,
|
|
1357
|
+
taskPath: executor.taskPath,
|
|
1358
|
+
threadId: fence.executorThreadId,
|
|
1359
|
+
generation,
|
|
1360
|
+
...(executor.resumeFailures ? { attempt: executor.resumeFailures } : {}),
|
|
1361
|
+
trigger: trigger.kind,
|
|
1362
|
+
createdAt: now,
|
|
1363
|
+
updatedAt: now,
|
|
1364
|
+
};
|
|
1365
|
+
}
|
|
1366
|
+
/** Process actions share the durable executor-command journal but carry a process-instance key. */
|
|
1367
|
+
processActionCommand(sessionId, executor, process, kind) {
|
|
1368
|
+
const retained = this.deps.plan(sessionId);
|
|
1369
|
+
const processKey = executorProcessKey(process);
|
|
1370
|
+
const now = this.deps.now();
|
|
1371
|
+
return {
|
|
1372
|
+
commandId: createHash('sha256')
|
|
1373
|
+
.update(JSON.stringify([
|
|
1374
|
+
retained?.identity ?? 'none',
|
|
1375
|
+
retained ? fingerprint(retained.plan) : 'none',
|
|
1376
|
+
executor.canonicalPosition,
|
|
1377
|
+
executor.canonicalTaskName,
|
|
1378
|
+
executor.taskPath,
|
|
1379
|
+
executor.threadId,
|
|
1380
|
+
executor.continuationGeneration,
|
|
1381
|
+
kind,
|
|
1382
|
+
processKey,
|
|
1383
|
+
]))
|
|
1384
|
+
.digest('hex'),
|
|
1385
|
+
status: 'scheduled',
|
|
1386
|
+
planIdentity: retained?.identity ?? 'none',
|
|
1387
|
+
planFingerprint: retained ? fingerprint(retained.plan) : 'none',
|
|
1388
|
+
canonicalPosition: executor.canonicalPosition,
|
|
1389
|
+
canonicalTaskName: executor.canonicalTaskName,
|
|
1390
|
+
taskPath: executor.taskPath,
|
|
1391
|
+
threadId: executor.threadId,
|
|
1392
|
+
generation: executor.continuationGeneration,
|
|
1393
|
+
// Process work never launches a writer itself; this preserves the command
|
|
1394
|
+
// envelope without conflating it with a continuation prompt.
|
|
1395
|
+
trigger: 'partial',
|
|
1396
|
+
processAction: { kind, processKey },
|
|
1397
|
+
createdAt: now,
|
|
1398
|
+
updatedAt: now,
|
|
1399
|
+
};
|
|
1400
|
+
}
|
|
1401
|
+
processActionCurrent(sessionId, command) {
|
|
1402
|
+
const state = this.deps.store.find(sessionId);
|
|
1403
|
+
const retained = this.deps.plan(sessionId);
|
|
1404
|
+
const executor = state?.executor;
|
|
1405
|
+
const child = this.deps
|
|
1406
|
+
.activity(sessionId)
|
|
1407
|
+
?.subagents.find((candidate) => (candidate.threadId ?? candidate.id) === command.threadId);
|
|
1408
|
+
if (!state?.requestedEnabled ||
|
|
1409
|
+
!retained ||
|
|
1410
|
+
!executor ||
|
|
1411
|
+
!command.processAction ||
|
|
1412
|
+
retained.identity !== command.planIdentity ||
|
|
1413
|
+
fingerprint(retained.plan) !== command.planFingerprint ||
|
|
1414
|
+
executor.canonicalPosition !== command.canonicalPosition ||
|
|
1415
|
+
executor.canonicalTaskName !== command.canonicalTaskName ||
|
|
1416
|
+
executor.taskPath !== command.taskPath ||
|
|
1417
|
+
executor.threadId !== command.threadId ||
|
|
1418
|
+
executor.continuationGeneration !== command.generation ||
|
|
1419
|
+
child?.canonicalPosition !== command.canonicalPosition ||
|
|
1420
|
+
child?.canonicalTaskName !== command.canonicalTaskName ||
|
|
1421
|
+
(child.continuationGeneration ?? 1) !== command.generation)
|
|
1422
|
+
return null;
|
|
1423
|
+
const process = executor.ownedProcesses.find((candidate) => executorProcessKey(candidate) === command.processAction.processKey);
|
|
1424
|
+
return process ? { state, executor, process } : null;
|
|
1425
|
+
}
|
|
1426
|
+
/**
|
|
1427
|
+
* Records issued before crossing the external boundary. Repeating an issued
|
|
1428
|
+
* action passes the same command id to an idempotent adapter, so a process
|
|
1429
|
+
* loss after the external effect cannot duplicate ownership or consumption.
|
|
1430
|
+
*/
|
|
1431
|
+
async performProcessAction(sessionId, executor, process, kind) {
|
|
1432
|
+
const controller = this.deps.executorController;
|
|
1433
|
+
if (!controller)
|
|
1434
|
+
return false;
|
|
1435
|
+
const requested = this.processActionCommand(sessionId, executor, process, kind);
|
|
1436
|
+
const command = this.scheduleExecutorCommand(sessionId, requested);
|
|
1437
|
+
if (!command || ['cancelled', 'superseded'].includes(command.status))
|
|
1438
|
+
return false;
|
|
1439
|
+
if (command.status === 'accepted')
|
|
1440
|
+
return true;
|
|
1441
|
+
const current = this.processActionCurrent(sessionId, command);
|
|
1442
|
+
if (!current) {
|
|
1443
|
+
this.executorCommandTransition(sessionId, command.commandId, 'superseded');
|
|
1444
|
+
return false;
|
|
1445
|
+
}
|
|
1446
|
+
const issued = command.status === 'issued'
|
|
1447
|
+
? command
|
|
1448
|
+
: this.executorCommandTransition(sessionId, command.commandId, 'issued');
|
|
1449
|
+
if (!issued || issued.status !== 'issued')
|
|
1450
|
+
return false;
|
|
1451
|
+
try {
|
|
1452
|
+
const accepted = kind === 'transfer'
|
|
1453
|
+
? (await controller.transferProcess(sessionId, current.process.ownerThreadId, current.process.processId, issued.commandId),
|
|
1454
|
+
true)
|
|
1455
|
+
: kind === 'consume'
|
|
1456
|
+
? (await controller.consumeProcess(sessionId, current.process.ownerThreadId, current.process.processId, issued.commandId),
|
|
1457
|
+
true)
|
|
1458
|
+
: await controller.terminateProcess(sessionId, current.process.ownerThreadId, current.process.processId, issued.commandId, {
|
|
1459
|
+
itemId: current.process.itemId,
|
|
1460
|
+
...(current.process.osPid === undefined ? {} : { osPid: current.process.osPid }),
|
|
1461
|
+
});
|
|
1462
|
+
if (!accepted) {
|
|
1463
|
+
this.executorCommandTransition(sessionId, issued.commandId, 'failed');
|
|
1464
|
+
return false;
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
catch (error) {
|
|
1468
|
+
// The command remains issued: the same idempotency key is the only retry.
|
|
1469
|
+
this.containOperationFailure(sessionId, error);
|
|
1470
|
+
return false;
|
|
1471
|
+
}
|
|
1472
|
+
const latest = this.processActionCurrent(sessionId, issued);
|
|
1473
|
+
if (!latest) {
|
|
1474
|
+
this.executorCommandTransition(sessionId, issued.commandId, 'superseded');
|
|
1475
|
+
return false;
|
|
1476
|
+
}
|
|
1477
|
+
this.persist({
|
|
1478
|
+
...latest.state,
|
|
1479
|
+
executor: {
|
|
1480
|
+
...latest.executor,
|
|
1481
|
+
commands: latest.executor.commands.map((candidate) => candidate.commandId === issued.commandId
|
|
1482
|
+
? { ...candidate, status: 'accepted', updatedAt: this.deps.now() }
|
|
1483
|
+
: candidate),
|
|
1484
|
+
ownedProcesses: latest.executor.ownedProcesses.map((candidate) => executorProcessKey(candidate) !== issued.processAction.processKey
|
|
1485
|
+
? candidate
|
|
1486
|
+
: kind === 'transfer'
|
|
1487
|
+
? {
|
|
1488
|
+
...candidate,
|
|
1489
|
+
ownership: 'supervisor',
|
|
1490
|
+
state: 'detached-active',
|
|
1491
|
+
}
|
|
1492
|
+
: kind === 'consume'
|
|
1493
|
+
? { ...candidate, state: 'result-consumed' }
|
|
1494
|
+
: { ...candidate, state: 'terminated-for-budget' }),
|
|
1495
|
+
},
|
|
1496
|
+
updatedAt: this.deps.now(),
|
|
1497
|
+
});
|
|
1498
|
+
return true;
|
|
1499
|
+
}
|
|
1500
|
+
executorCommandTransition(sessionId, commandId, status) {
|
|
1501
|
+
const state = this.deps.store.find(sessionId);
|
|
1502
|
+
const executor = state?.executor;
|
|
1503
|
+
const command = executor?.commands?.find((candidate) => candidate.commandId === commandId);
|
|
1504
|
+
if (!state || !executor || !command || command.status === status)
|
|
1505
|
+
return command;
|
|
1506
|
+
const next = { ...command, status, updatedAt: this.deps.now() };
|
|
1507
|
+
this.persist({
|
|
1508
|
+
...state,
|
|
1509
|
+
executor: {
|
|
1510
|
+
...executor,
|
|
1511
|
+
commands: executor.commands.map((candidate) => candidate.commandId === commandId ? next : candidate),
|
|
1512
|
+
},
|
|
1513
|
+
updatedAt: this.deps.now(),
|
|
1514
|
+
});
|
|
1515
|
+
return next;
|
|
1516
|
+
}
|
|
1517
|
+
scheduleExecutorCommand(sessionId, command) {
|
|
1518
|
+
const state = this.deps.store.find(sessionId);
|
|
1519
|
+
const executor = state?.executor;
|
|
1520
|
+
if (!state || !executor)
|
|
1521
|
+
return undefined;
|
|
1522
|
+
const existing = executor.commands?.find((candidate) => candidate.commandId === command.commandId);
|
|
1523
|
+
if (existing)
|
|
1524
|
+
return existing;
|
|
1525
|
+
const commands = [...(executor.commands ?? []), command].slice(-32);
|
|
1526
|
+
this.persist({
|
|
1527
|
+
...state,
|
|
1528
|
+
executor: { ...executor, commands },
|
|
1529
|
+
updatedAt: this.deps.now(),
|
|
1530
|
+
});
|
|
1531
|
+
return command;
|
|
1532
|
+
}
|
|
1132
1533
|
persistExecutor(sessionId, executor) {
|
|
1133
1534
|
const current = this.deps.store.find(sessionId);
|
|
1134
1535
|
if (!current || JSON.stringify(current.executor) === JSON.stringify(executor))
|
|
@@ -1136,6 +1537,40 @@ export class AutopilotCoordinator {
|
|
|
1136
1537
|
this.persist({ ...current, executor, updatedAt: this.deps.now() });
|
|
1137
1538
|
return true;
|
|
1138
1539
|
}
|
|
1540
|
+
/** Retain stale commands as evidence; they can never regain writer authority. */
|
|
1541
|
+
supersedeExecutorCommands(sessionId, planIdentity, plan, current) {
|
|
1542
|
+
const state = this.deps.store.find(sessionId);
|
|
1543
|
+
const executor = state?.executor;
|
|
1544
|
+
if (!state || !executor?.commands?.length)
|
|
1545
|
+
return false;
|
|
1546
|
+
const planFingerprint = fingerprint(plan);
|
|
1547
|
+
let changed = false;
|
|
1548
|
+
const commands = executor.commands.map((command) => {
|
|
1549
|
+
const ownershipMismatch = Boolean(current) &&
|
|
1550
|
+
(command.canonicalPosition !== current.canonicalPosition ||
|
|
1551
|
+
command.canonicalTaskName !== current.canonicalTaskName ||
|
|
1552
|
+
command.taskPath !== current.taskPath ||
|
|
1553
|
+
command.threadId !== current.threadId);
|
|
1554
|
+
const stale = command.planIdentity !== planIdentity ||
|
|
1555
|
+
command.planFingerprint !== planFingerprint ||
|
|
1556
|
+
ownershipMismatch;
|
|
1557
|
+
if (!stale || !['scheduled', 'issued'].includes(command.status))
|
|
1558
|
+
return command;
|
|
1559
|
+
changed = true;
|
|
1560
|
+
return { ...command, status: 'superseded', updatedAt: this.deps.now() };
|
|
1561
|
+
});
|
|
1562
|
+
if (!changed)
|
|
1563
|
+
return false;
|
|
1564
|
+
this.persist({
|
|
1565
|
+
...state,
|
|
1566
|
+
executor: { ...executor, commands },
|
|
1567
|
+
updatedAt: this.deps.now(),
|
|
1568
|
+
});
|
|
1569
|
+
this.audit(sessionId, 'autopilot.executor-command-superseded', {
|
|
1570
|
+
count: commands.filter((command) => command.status === 'superseded').length,
|
|
1571
|
+
});
|
|
1572
|
+
return true;
|
|
1573
|
+
}
|
|
1139
1574
|
wakeForExecutorChange(sessionId, executor) {
|
|
1140
1575
|
const processes = executor.ownedProcesses;
|
|
1141
1576
|
const conditions = [
|
|
@@ -1150,6 +1585,9 @@ export class AutopilotCoordinator {
|
|
|
1150
1585
|
return conditions.some((condition) => this.semanticEvent(sessionId, condition));
|
|
1151
1586
|
}
|
|
1152
1587
|
freshExecutorIdentity(sessionId) {
|
|
1588
|
+
const replacement = this.deps.store.find(sessionId)?.executor?.replacement;
|
|
1589
|
+
if (replacement && this.validReplacement(sessionId, replacement))
|
|
1590
|
+
return replacement;
|
|
1153
1591
|
const plan = this.deps.plan(sessionId)?.plan;
|
|
1154
1592
|
if (!plan)
|
|
1155
1593
|
return undefined;
|
|
@@ -1164,29 +1602,232 @@ export class AutopilotCoordinator {
|
|
|
1164
1602
|
.map((child) => child.continuationGeneration ?? 1) ?? [];
|
|
1165
1603
|
return executorIdentity(canonicalTaskName, Math.max(0, ...generations) + 1);
|
|
1166
1604
|
}
|
|
1605
|
+
validReplacement(sessionId, replacement) {
|
|
1606
|
+
const retained = this.deps.plan(sessionId);
|
|
1607
|
+
if (!retained ||
|
|
1608
|
+
retained.identity !== replacement.planIdentity ||
|
|
1609
|
+
fingerprint(retained.plan) !== replacement.planFingerprint)
|
|
1610
|
+
return false;
|
|
1611
|
+
const index = retained.plan.steps.findIndex((step) => step.id === retained.plan.currentStepId || step.state === 'WIP');
|
|
1612
|
+
return index >= 0 && replacement.canonicalPosition === `L${index + 1}`;
|
|
1613
|
+
}
|
|
1614
|
+
/** Converts an exhausted physical executor into one durable root-owned handoff. */
|
|
1615
|
+
scheduleExecutorReplacement(sessionId, executor) {
|
|
1616
|
+
const current = this.deps.store.find(sessionId);
|
|
1617
|
+
const retained = this.deps.plan(sessionId);
|
|
1618
|
+
if (!current?.requestedEnabled || !retained || current.executor?.replacement)
|
|
1619
|
+
return;
|
|
1620
|
+
const identity = executorIdentity(executor.canonicalTaskName, executor.continuationGeneration + 1);
|
|
1621
|
+
const now = this.deps.now();
|
|
1622
|
+
const controlId = this.deps.nextControlId(sessionId, current.generation);
|
|
1623
|
+
const replacement = {
|
|
1624
|
+
...executor,
|
|
1625
|
+
outcome: 'failed',
|
|
1626
|
+
replacement: {
|
|
1627
|
+
...identity,
|
|
1628
|
+
planIdentity: retained.identity,
|
|
1629
|
+
planFingerprint: fingerprint(retained.plan),
|
|
1630
|
+
},
|
|
1631
|
+
commands: executor.commands?.map((command) => ['scheduled', 'issued'].includes(command.status)
|
|
1632
|
+
? { ...command, status: 'superseded', updatedAt: now }
|
|
1633
|
+
: command),
|
|
1634
|
+
};
|
|
1635
|
+
this.arm(sessionId, current.generation, now);
|
|
1636
|
+
this.persist({
|
|
1637
|
+
...current,
|
|
1638
|
+
state: 'backoff',
|
|
1639
|
+
executor: replacement,
|
|
1640
|
+
nextEvaluationAt: now,
|
|
1641
|
+
lastControlId: controlId,
|
|
1642
|
+
updatedAt: now,
|
|
1643
|
+
}, {
|
|
1644
|
+
sessionId,
|
|
1645
|
+
controlId,
|
|
1646
|
+
status: 'scheduled',
|
|
1647
|
+
createdAt: now,
|
|
1648
|
+
updatedAt: now,
|
|
1649
|
+
failureCode: null,
|
|
1650
|
+
}, [
|
|
1651
|
+
{
|
|
1652
|
+
sessionId,
|
|
1653
|
+
type: 'autopilot.executor-replacement-scheduled',
|
|
1654
|
+
payload: {
|
|
1655
|
+
canonicalPosition: identity.canonicalPosition,
|
|
1656
|
+
taskName: identity.taskName,
|
|
1657
|
+
generation: identity.generation,
|
|
1658
|
+
},
|
|
1659
|
+
occurredAt: now,
|
|
1660
|
+
},
|
|
1661
|
+
]);
|
|
1662
|
+
}
|
|
1167
1663
|
armExecutorRefresh(sessionId, delayMs) {
|
|
1168
1664
|
if (this.executorTimers.has(sessionId))
|
|
1169
1665
|
return;
|
|
1666
|
+
const state = this.deps.store.find(sessionId);
|
|
1667
|
+
const plan = this.deps.plan(sessionId);
|
|
1668
|
+
const executor = state && plan
|
|
1669
|
+
? this.currentExecutor(sessionId, plan.plan, state.consecutiveNoProgress, state.executor)
|
|
1670
|
+
: undefined;
|
|
1671
|
+
const fence = state && plan && !this.deps.session(sessionId)?.activeTurnId
|
|
1672
|
+
? {
|
|
1673
|
+
generation: state.generation,
|
|
1674
|
+
identity: plan.identity,
|
|
1675
|
+
fingerprint: fingerprint(plan.plan),
|
|
1676
|
+
state: state.state,
|
|
1677
|
+
supervision: state.supervision?.outcome ?? null,
|
|
1678
|
+
checkpoint: JSON.stringify(state.checkpoints ?? null),
|
|
1679
|
+
canonicalPosition: executor?.canonicalPosition ?? null,
|
|
1680
|
+
executorThreadId: executor?.threadId ?? null,
|
|
1681
|
+
executorGeneration: executor?.continuationGeneration ?? null,
|
|
1682
|
+
}
|
|
1683
|
+
: null;
|
|
1684
|
+
if (!fence)
|
|
1685
|
+
return;
|
|
1686
|
+
let delivered = false;
|
|
1170
1687
|
this.executorTimers.set(sessionId, this.deps.schedule(() => {
|
|
1171
1688
|
this.executorTimers.delete(sessionId);
|
|
1689
|
+
if (delivered) {
|
|
1690
|
+
this.audit(sessionId, 'autopilot.executor-refresh-stale', { fence });
|
|
1691
|
+
return;
|
|
1692
|
+
}
|
|
1693
|
+
delivered = true;
|
|
1172
1694
|
this.enqueue(sessionId, async () => {
|
|
1695
|
+
const current = this.deps.store.find(sessionId);
|
|
1696
|
+
const currentPlan = this.deps.plan(sessionId);
|
|
1697
|
+
const currentChild = fence.executorThreadId
|
|
1698
|
+
? this.deps
|
|
1699
|
+
.activity(sessionId)
|
|
1700
|
+
?.subagents.find((child) => (child.threadId ?? child.id) === fence.executorThreadId)
|
|
1701
|
+
: null;
|
|
1702
|
+
if (!current?.requestedEnabled ||
|
|
1703
|
+
this.deps.pendingInteraction(sessionId) ||
|
|
1704
|
+
current.generation !== fence.generation ||
|
|
1705
|
+
current.state !== fence.state ||
|
|
1706
|
+
(current.supervision?.outcome ?? null) !== fence.supervision ||
|
|
1707
|
+
JSON.stringify(current.checkpoints ?? null) !== fence.checkpoint ||
|
|
1708
|
+
currentPlan?.identity !== fence.identity ||
|
|
1709
|
+
!currentPlan ||
|
|
1710
|
+
fingerprint(currentPlan.plan) !== fence.fingerprint ||
|
|
1711
|
+
Boolean(this.deps.session(sessionId)?.activeTurnId) ||
|
|
1712
|
+
(fence.executorThreadId !== null &&
|
|
1713
|
+
(currentChild?.canonicalPosition !== fence.canonicalPosition ||
|
|
1714
|
+
(currentChild.continuationGeneration ?? 1) !== fence.executorGeneration))) {
|
|
1715
|
+
this.audit(sessionId, 'autopilot.executor-refresh-stale', { fence });
|
|
1716
|
+
return;
|
|
1717
|
+
}
|
|
1173
1718
|
await this.deps.executorController?.refresh(sessionId);
|
|
1174
1719
|
if (!(await this.enforceSupervisedLifecycle(sessionId, 'processObserved')))
|
|
1175
1720
|
this.evaluate(sessionId);
|
|
1176
1721
|
});
|
|
1177
1722
|
}, delayMs));
|
|
1178
1723
|
}
|
|
1724
|
+
/**
|
|
1725
|
+
* A store read can fail after the timer that exposed it has already fired.
|
|
1726
|
+
* This retry deliberately needs no lifecycle read to arm; once the store is
|
|
1727
|
+
* available again, normal evaluation re-establishes a fully fenced timer.
|
|
1728
|
+
*/
|
|
1729
|
+
armStoreRecovery(sessionId) {
|
|
1730
|
+
if (this.executorTimers.has(sessionId))
|
|
1731
|
+
return;
|
|
1732
|
+
let delivered = false;
|
|
1733
|
+
const cancel = this.deps.schedule(() => {
|
|
1734
|
+
this.executorTimers.delete(sessionId);
|
|
1735
|
+
if (delivered)
|
|
1736
|
+
return;
|
|
1737
|
+
delivered = true;
|
|
1738
|
+
this.enqueue(sessionId, async () => {
|
|
1739
|
+
const current = this.deps.store.find(sessionId);
|
|
1740
|
+
if (!current?.requestedEnabled)
|
|
1741
|
+
return;
|
|
1742
|
+
this.evaluate(sessionId);
|
|
1743
|
+
});
|
|
1744
|
+
}, this.deps.policy.executorContinuationMaxMs);
|
|
1745
|
+
this.executorTimers.set(sessionId, cancel);
|
|
1746
|
+
}
|
|
1179
1747
|
armExecutorContinuation(sessionId, delayMs, threadId, generation, trigger) {
|
|
1180
1748
|
if (this.executorTimers.has(sessionId))
|
|
1181
1749
|
return;
|
|
1750
|
+
const state = this.deps.store.find(sessionId);
|
|
1751
|
+
const retained = this.deps.plan(sessionId);
|
|
1752
|
+
const armedChild = this.deps
|
|
1753
|
+
.activity(sessionId)
|
|
1754
|
+
?.subagents.find((child) => (child.threadId ?? child.id) === threadId);
|
|
1755
|
+
const fence = state && retained
|
|
1756
|
+
? {
|
|
1757
|
+
sessionGeneration: state.generation,
|
|
1758
|
+
planIdentity: retained.identity,
|
|
1759
|
+
planFingerprint: fingerprint(retained.plan),
|
|
1760
|
+
canonicalPosition: armedChild?.canonicalPosition ?? state.executor?.canonicalPosition ?? null,
|
|
1761
|
+
executorThreadId: threadId,
|
|
1762
|
+
executorGeneration: generation - 1,
|
|
1763
|
+
supervisionOutcome: state.supervision?.outcome ?? null,
|
|
1764
|
+
checkpoint: JSON.stringify(state.checkpoints ?? null),
|
|
1765
|
+
boundaryState: state.state,
|
|
1766
|
+
}
|
|
1767
|
+
: null;
|
|
1768
|
+
if (!fence || this.deps.session(sessionId)?.activeTurnId)
|
|
1769
|
+
return;
|
|
1770
|
+
const commandExecutor = state?.executor;
|
|
1771
|
+
if (!commandExecutor)
|
|
1772
|
+
return;
|
|
1773
|
+
const requested = this.executorCommand(fence, commandExecutor, generation, trigger);
|
|
1774
|
+
const command = this.scheduleExecutorCommand(sessionId, requested);
|
|
1775
|
+
// A command that crossed the durable issue boundary is deliberately not
|
|
1776
|
+
// replayed: app-server acceptance may have happened before a crash.
|
|
1777
|
+
if (!command || command.status !== 'scheduled')
|
|
1778
|
+
return;
|
|
1779
|
+
let delivered = false;
|
|
1182
1780
|
this.executorTimers.set(sessionId, this.deps.schedule(() => {
|
|
1183
1781
|
this.executorTimers.delete(sessionId);
|
|
1782
|
+
if (delivered) {
|
|
1783
|
+
this.audit(sessionId, 'autopilot.executor-continuation-stale', {
|
|
1784
|
+
threadId,
|
|
1785
|
+
generation,
|
|
1786
|
+
trigger: trigger.kind,
|
|
1787
|
+
fence,
|
|
1788
|
+
});
|
|
1789
|
+
return;
|
|
1790
|
+
}
|
|
1791
|
+
delivered = true;
|
|
1184
1792
|
this.enqueue(sessionId, async () => {
|
|
1185
1793
|
const current = this.deps.store.find(sessionId);
|
|
1186
|
-
|
|
1794
|
+
const latestPlan = this.deps.plan(sessionId);
|
|
1795
|
+
const currentChild = this.deps
|
|
1796
|
+
.activity(sessionId)
|
|
1797
|
+
?.subagents.find((child) => (child.threadId ?? child.id) === fence.executorThreadId);
|
|
1798
|
+
const valid = current?.requestedEnabled &&
|
|
1799
|
+
!this.deps.pendingInteraction(sessionId) &&
|
|
1800
|
+
current.generation === fence.sessionGeneration &&
|
|
1801
|
+
current.state === fence.boundaryState &&
|
|
1802
|
+
(current.supervision?.outcome ?? null) === fence.supervisionOutcome &&
|
|
1803
|
+
JSON.stringify(current.checkpoints ?? null) === fence.checkpoint &&
|
|
1804
|
+
!this.deps.session(sessionId)?.activeTurnId &&
|
|
1805
|
+
latestPlan?.identity === fence.planIdentity &&
|
|
1806
|
+
latestPlan &&
|
|
1807
|
+
fingerprint(latestPlan.plan) === fence.planFingerprint &&
|
|
1808
|
+
currentChild?.canonicalPosition === fence.canonicalPosition &&
|
|
1809
|
+
(currentChild.continuationGeneration ?? 1) === fence.executorGeneration;
|
|
1810
|
+
if (!valid) {
|
|
1811
|
+
const ownershipChanged = latestPlan?.identity !== fence.planIdentity ||
|
|
1812
|
+
!latestPlan ||
|
|
1813
|
+
fingerprint(latestPlan.plan) !== fence.planFingerprint ||
|
|
1814
|
+
currentChild?.canonicalPosition !== fence.canonicalPosition ||
|
|
1815
|
+
(currentChild.continuationGeneration ?? 1) !== fence.executorGeneration;
|
|
1816
|
+
this.executorCommandTransition(sessionId, command.commandId, ownershipChanged ? 'superseded' : 'cancelled');
|
|
1817
|
+
this.audit(sessionId, 'autopilot.executor-continuation-stale', {
|
|
1818
|
+
threadId,
|
|
1819
|
+
generation,
|
|
1820
|
+
trigger: trigger.kind,
|
|
1821
|
+
fence,
|
|
1822
|
+
});
|
|
1823
|
+
return;
|
|
1824
|
+
}
|
|
1825
|
+
const issued = this.executorCommandTransition(sessionId, command.commandId, 'issued');
|
|
1826
|
+
if (issued?.status !== 'issued')
|
|
1187
1827
|
return;
|
|
1188
1828
|
try {
|
|
1189
1829
|
await this.deps.executorController?.resume(sessionId, threadId, generation, trigger);
|
|
1830
|
+
this.executorCommandTransition(sessionId, command.commandId, 'accepted');
|
|
1190
1831
|
this.audit(sessionId, 'autopilot.executor-resumed', {
|
|
1191
1832
|
threadId,
|
|
1192
1833
|
generation,
|
|
@@ -1213,8 +1854,30 @@ export class AutopilotCoordinator {
|
|
|
1213
1854
|
updatedAt: this.deps.now(),
|
|
1214
1855
|
});
|
|
1215
1856
|
}
|
|
1216
|
-
catch {
|
|
1217
|
-
|
|
1857
|
+
catch (error) {
|
|
1858
|
+
// Only an explicit app-server rejection is safe to retry. A lost
|
|
1859
|
+
// response may conceal accepted work, so retain its issued fence.
|
|
1860
|
+
if (!explicitExecutorRejection(error)) {
|
|
1861
|
+
this.containOperationFailure(sessionId, error);
|
|
1862
|
+
this.armExecutorRefresh(sessionId, this.deps.policy.executorContinuationMaxMs);
|
|
1863
|
+
return;
|
|
1864
|
+
}
|
|
1865
|
+
this.executorCommandTransition(sessionId, command.commandId, 'failed');
|
|
1866
|
+
const latest = this.deps.store.find(sessionId);
|
|
1867
|
+
const failedExecutor = latest?.executor;
|
|
1868
|
+
if (!latest?.requestedEnabled || !failedExecutor)
|
|
1869
|
+
return;
|
|
1870
|
+
const resumeFailures = (failedExecutor.resumeFailures ?? 0) + 1;
|
|
1871
|
+
const next = {
|
|
1872
|
+
...latest,
|
|
1873
|
+
executor: { ...failedExecutor, resumeFailures },
|
|
1874
|
+
updatedAt: this.deps.now(),
|
|
1875
|
+
};
|
|
1876
|
+
this.persist(next);
|
|
1877
|
+
if (resumeFailures >= this.deps.policy.retryLimit)
|
|
1878
|
+
this.scheduleExecutorReplacement(sessionId, next.executor);
|
|
1879
|
+
else
|
|
1880
|
+
this.armExecutorRefresh(sessionId, this.deps.policy.executorContinuationMaxMs);
|
|
1218
1881
|
}
|
|
1219
1882
|
});
|
|
1220
1883
|
}, delayMs));
|
|
@@ -1405,6 +2068,10 @@ export class AutopilotCoordinator {
|
|
|
1405
2068
|
}
|
|
1406
2069
|
progressKey(sessionId) {
|
|
1407
2070
|
const retained = this.deps.plan(sessionId);
|
|
2071
|
+
const state = this.deps.store.find(sessionId);
|
|
2072
|
+
return this.progressKeyFor(sessionId, state, retained);
|
|
2073
|
+
}
|
|
2074
|
+
progressKeyFor(sessionId, state, retained) {
|
|
1408
2075
|
const activity = this.deps.activity(sessionId);
|
|
1409
2076
|
const interactions = this.deps.pendingInteraction(sessionId)
|
|
1410
2077
|
? [{ id: 'pending', kind: 'session', state: 'pending' }]
|
|
@@ -1417,18 +2084,32 @@ export class AutopilotCoordinator {
|
|
|
1417
2084
|
},
|
|
1418
2085
|
review: { status: null },
|
|
1419
2086
|
checkpoint: {
|
|
1420
|
-
pendingTurnId:
|
|
1421
|
-
terminalReviewAccepted:
|
|
2087
|
+
pendingTurnId: state?.checkpoints?.pendingTurnId ?? null,
|
|
2088
|
+
terminalReviewAccepted: state?.checkpoints?.terminalReviewAccepted ?? false,
|
|
1422
2089
|
},
|
|
1423
2090
|
pendingInteractions: interactions,
|
|
1424
2091
|
executor: {
|
|
1425
|
-
generation:
|
|
1426
|
-
state:
|
|
2092
|
+
generation: state?.executor?.continuationGeneration ?? 0,
|
|
2093
|
+
state: state?.executor?.outcome ?? null,
|
|
1427
2094
|
},
|
|
1428
|
-
ownedProcesses: (
|
|
2095
|
+
ownedProcesses: (state?.executor?.ownedProcesses ?? []).map((process) => ({
|
|
1429
2096
|
id: process.processId,
|
|
1430
2097
|
state: process.state,
|
|
1431
|
-
ownerGeneration:
|
|
2098
|
+
ownerGeneration: state?.executor?.continuationGeneration ?? 0,
|
|
2099
|
+
})),
|
|
2100
|
+
childActivity: (activity?.subagents ?? []).map((child) => ({
|
|
2101
|
+
id: child.id,
|
|
2102
|
+
threadId: child.threadId ?? null,
|
|
2103
|
+
taskName: child.canonicalTaskName ?? null,
|
|
2104
|
+
position: child.canonicalPosition ?? null,
|
|
2105
|
+
generation: child.continuationGeneration ?? 0,
|
|
2106
|
+
state: child.state,
|
|
2107
|
+
outcome: child.outcome ?? null,
|
|
2108
|
+
ownedProcesses: (child.ownedProcesses ?? []).map((process) => ({
|
|
2109
|
+
id: process.processId,
|
|
2110
|
+
state: process.state,
|
|
2111
|
+
ownership: process.ownership,
|
|
2112
|
+
})),
|
|
1432
2113
|
})),
|
|
1433
2114
|
agentActivity: [
|
|
1434
2115
|
...(activity
|
|
@@ -1485,16 +2166,142 @@ export class AutopilotCoordinator {
|
|
|
1485
2166
|
}
|
|
1486
2167
|
enqueue(sessionId, operation) {
|
|
1487
2168
|
const previous = this.operations.get(sessionId);
|
|
1488
|
-
//
|
|
1489
|
-
//
|
|
1490
|
-
|
|
2169
|
+
// Every asynchronous boundary settles locally. In particular, do not
|
|
2170
|
+
// discard the promise returned by finally(): that would turn a rejected
|
|
2171
|
+
// controller, store, or publisher call into an unhandled rejection and
|
|
2172
|
+
// poison the next serial operation.
|
|
2173
|
+
const run = async () => {
|
|
2174
|
+
try {
|
|
2175
|
+
await operation();
|
|
2176
|
+
}
|
|
2177
|
+
catch (error) {
|
|
2178
|
+
this.containOperationFailure(sessionId, error);
|
|
2179
|
+
}
|
|
2180
|
+
};
|
|
2181
|
+
const next = previous ? previous.then(run, run) : run();
|
|
1491
2182
|
this.operations.set(sessionId, next);
|
|
1492
|
-
void next.
|
|
2183
|
+
void next.then(() => {
|
|
2184
|
+
if (this.operations.get(sessionId) === next)
|
|
2185
|
+
this.operations.delete(sessionId);
|
|
2186
|
+
}, () => {
|
|
2187
|
+
// run() contains failures, but retain a rejection observer as a final
|
|
2188
|
+
// guard if a future containment change itself becomes asynchronous.
|
|
1493
2189
|
if (this.operations.get(sessionId) === next)
|
|
1494
2190
|
this.operations.delete(sessionId);
|
|
1495
2191
|
});
|
|
1496
2192
|
return next;
|
|
1497
2193
|
}
|
|
2194
|
+
/**
|
|
2195
|
+
* Converts an unexpected queued-operation failure into one opaque durable
|
|
2196
|
+
* recovery outcome. The recovery path deliberately avoids publish(): an
|
|
2197
|
+
* outbox publication failure must not recursively manufacture another
|
|
2198
|
+
* rejected promise before a later journal flush can replay the event.
|
|
2199
|
+
*/
|
|
2200
|
+
containOperationFailure(sessionId, error) {
|
|
2201
|
+
const code = operationFailureCode(error);
|
|
2202
|
+
try {
|
|
2203
|
+
this.deps.diagnostic?.(sessionId, `operationFailed:${code}`);
|
|
2204
|
+
}
|
|
2205
|
+
catch {
|
|
2206
|
+
// Diagnostics are advisory and cannot compromise queue liveness.
|
|
2207
|
+
}
|
|
2208
|
+
let current = null;
|
|
2209
|
+
try {
|
|
2210
|
+
current = this.deps.store.find(sessionId);
|
|
2211
|
+
}
|
|
2212
|
+
catch {
|
|
2213
|
+
try {
|
|
2214
|
+
this.armStoreRecovery(sessionId);
|
|
2215
|
+
}
|
|
2216
|
+
catch {
|
|
2217
|
+
try {
|
|
2218
|
+
this.deps.diagnostic?.(sessionId, `operationRecoveryUnavailable:${code}`);
|
|
2219
|
+
}
|
|
2220
|
+
catch {
|
|
2221
|
+
// A later external lifecycle input may retry after both dependencies recover.
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2224
|
+
return;
|
|
2225
|
+
}
|
|
2226
|
+
if (!current?.requestedEnabled)
|
|
2227
|
+
return;
|
|
2228
|
+
const now = this.deps.now();
|
|
2229
|
+
const recovery = {
|
|
2230
|
+
...current,
|
|
2231
|
+
state: 'monitoring',
|
|
2232
|
+
generation: current.generation + 1,
|
|
2233
|
+
nextEvaluationAt: null,
|
|
2234
|
+
stopReason: 'reconcileFailed',
|
|
2235
|
+
updatedAt: now,
|
|
2236
|
+
};
|
|
2237
|
+
try {
|
|
2238
|
+
this.commit({
|
|
2239
|
+
state: recovery,
|
|
2240
|
+
events: [
|
|
2241
|
+
{
|
|
2242
|
+
sessionId,
|
|
2243
|
+
type: 'autopilot.operation-failed',
|
|
2244
|
+
payload: { code },
|
|
2245
|
+
occurredAt: now,
|
|
2246
|
+
},
|
|
2247
|
+
],
|
|
2248
|
+
});
|
|
2249
|
+
}
|
|
2250
|
+
catch {
|
|
2251
|
+
// A failing persistence adapter has no durable surface available in this
|
|
2252
|
+
// process. Its observable diagnostic is deliberately independent of
|
|
2253
|
+
// that adapter; retain one bounded runtime recovery rather than treating
|
|
2254
|
+
// a failed write as if the operation had safely settled.
|
|
2255
|
+
try {
|
|
2256
|
+
this.armExecutorRefresh(sessionId, this.deps.policy.executorContinuationMaxMs);
|
|
2257
|
+
}
|
|
2258
|
+
catch {
|
|
2259
|
+
try {
|
|
2260
|
+
this.deps.diagnostic?.(sessionId, `operationRecoveryUnavailable:${code}`);
|
|
2261
|
+
}
|
|
2262
|
+
catch {
|
|
2263
|
+
// The queue itself remains settled for a future external event.
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
2266
|
+
return;
|
|
2267
|
+
}
|
|
2268
|
+
try {
|
|
2269
|
+
this.armExecutorRefresh(sessionId, this.deps.policy.executorContinuationMaxMs);
|
|
2270
|
+
}
|
|
2271
|
+
catch {
|
|
2272
|
+
// Scheduling is the last recovery capability. If it is unavailable,
|
|
2273
|
+
// leave a safe terminal state rather than claim enabled supervision with
|
|
2274
|
+
// no reachable continuation.
|
|
2275
|
+
try {
|
|
2276
|
+
this.commit({
|
|
2277
|
+
state: {
|
|
2278
|
+
...recovery,
|
|
2279
|
+
state: 'safetyPaused',
|
|
2280
|
+
requestedEnabled: false,
|
|
2281
|
+
stopReason: 'safetyPaused',
|
|
2282
|
+
updatedAt: this.deps.now(),
|
|
2283
|
+
},
|
|
2284
|
+
events: [
|
|
2285
|
+
{
|
|
2286
|
+
sessionId,
|
|
2287
|
+
type: 'autopilot.operation-recovery-unavailable',
|
|
2288
|
+
payload: { code },
|
|
2289
|
+
occurredAt: this.deps.now(),
|
|
2290
|
+
},
|
|
2291
|
+
],
|
|
2292
|
+
});
|
|
2293
|
+
}
|
|
2294
|
+
catch {
|
|
2295
|
+
// There is no additional in-process action that can safely restore a
|
|
2296
|
+
// failed store and scheduler; importantly the serial queue remains
|
|
2297
|
+
// settled for a future external recovery event.
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
2300
|
+
}
|
|
2301
|
+
}
|
|
2302
|
+
function explicitExecutorRejection(error) {
|
|
2303
|
+
return (error instanceof Error &&
|
|
2304
|
+
['AUTOPILOT_EXECUTOR_UNAVAILABLE', 'AUTOPILOT_EXECUTOR_REJECTED'].includes(error.message));
|
|
1498
2305
|
}
|
|
1499
2306
|
function startFailureCode(error) {
|
|
1500
2307
|
const code = error && typeof error === 'object' && 'code' in error && typeof error.code === 'string'
|
|
@@ -1506,6 +2313,19 @@ function startFailureCode(error) {
|
|
|
1506
2313
|
? 'START_UNAVAILABLE'
|
|
1507
2314
|
: 'START_FAILED';
|
|
1508
2315
|
}
|
|
2316
|
+
/** Keep failure telemetry bounded and free of prompts, paths, and error prose. */
|
|
2317
|
+
function operationFailureCode(error) {
|
|
2318
|
+
const code = error && typeof error === 'object' && 'code' in error && typeof error.code === 'string'
|
|
2319
|
+
? error.code
|
|
2320
|
+
: error instanceof Error
|
|
2321
|
+
? error.message
|
|
2322
|
+
: '';
|
|
2323
|
+
if (/PERSIST|SQLITE|DATABASE|STORE/i.test(code))
|
|
2324
|
+
return 'PERSISTENCE';
|
|
2325
|
+
if (/PUBLISH|OUTBOX|JOURNAL/i.test(code))
|
|
2326
|
+
return 'PUBLICATION';
|
|
2327
|
+
return 'OPERATION';
|
|
2328
|
+
}
|
|
1509
2329
|
function fingerprint(plan) {
|
|
1510
2330
|
return JSON.stringify(plan.steps.map((step) => [
|
|
1511
2331
|
step.id,
|
|
@@ -1514,12 +2334,22 @@ function fingerprint(plan) {
|
|
|
1514
2334
|
step.children.map((child) => [child.id, child.state]),
|
|
1515
2335
|
]));
|
|
1516
2336
|
}
|
|
2337
|
+
function boundedUnique(values, limit) {
|
|
2338
|
+
return [...new Set(values)].slice(-limit);
|
|
2339
|
+
}
|
|
2340
|
+
function boundEpochs(epochs, currentTarget) {
|
|
2341
|
+
const current = epochs.find((entry) => entry.target === currentTarget);
|
|
2342
|
+
const retained = epochs
|
|
2343
|
+
.filter((entry) => entry.target !== currentTarget)
|
|
2344
|
+
.sort((left, right) => left.target.localeCompare(right.target));
|
|
2345
|
+
return [...retained.slice(0, current ? 639 : 640), ...(current ? [current] : [])].sort((left, right) => left.target.localeCompare(right.target));
|
|
2346
|
+
}
|
|
1517
2347
|
function refreshPersistedProcesses(persisted, observed, now) {
|
|
1518
2348
|
if (!observed.length)
|
|
1519
2349
|
return persisted;
|
|
1520
|
-
const priorById = new Map(persisted.map((process) => [process
|
|
2350
|
+
const priorById = new Map(persisted.map((process) => [executorProcessKey(process), process]));
|
|
1521
2351
|
return observed.map((process) => {
|
|
1522
|
-
const prior = priorById.get(process
|
|
2352
|
+
const prior = priorById.get(executorProcessKey(process));
|
|
1523
2353
|
if (!prior)
|
|
1524
2354
|
return process;
|
|
1525
2355
|
const observedAt = prior.observedAt;
|
|
@@ -1528,9 +2358,21 @@ function refreshPersistedProcesses(persisted, observed, now) {
|
|
|
1528
2358
|
observedAt,
|
|
1529
2359
|
elapsedMs: Math.max(process.elapsedMs, Date.parse(now) - Date.parse(observedAt)),
|
|
1530
2360
|
ownership: prior.ownership === 'supervisor' ? 'supervisor' : process.ownership,
|
|
1531
|
-
state: prior.
|
|
1532
|
-
?
|
|
1533
|
-
: process.state
|
|
2361
|
+
state: prior.state === 'result-consumed' || prior.state === 'terminated-for-budget'
|
|
2362
|
+
? prior.state
|
|
2363
|
+
: prior.ownership === 'supervisor' && process.state === 'running'
|
|
2364
|
+
? 'detached-active'
|
|
2365
|
+
: process.state,
|
|
1534
2366
|
};
|
|
1535
2367
|
});
|
|
1536
2368
|
}
|
|
2369
|
+
/** A numeric OS PID or process id alone can be reused by a later child operation. */
|
|
2370
|
+
function executorProcessKey(process) {
|
|
2371
|
+
return JSON.stringify([
|
|
2372
|
+
process.ownerThreadId,
|
|
2373
|
+
process.ownerTaskPath,
|
|
2374
|
+
process.processId,
|
|
2375
|
+
process.itemId,
|
|
2376
|
+
process.osPid ?? null,
|
|
2377
|
+
]);
|
|
2378
|
+
}
|