owenloop 0.5.22 → 0.5.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/packages/work/src/agent/brief.d.ts +16 -0
- package/dist/packages/work/src/agent/brief.js +23 -0
- package/dist/packages/work/src/agent/loop.d.ts +11 -1
- package/dist/packages/work/src/agent/loop.js +520 -8
- package/dist/packages/work/src/harness/claude.d.ts +43 -3
- package/dist/packages/work/src/harness/claude.js +520 -115
- package/dist/packages/work/src/harness/contract.d.ts +36 -1
- package/dist/packages/work/src/harness/contract.js +34 -0
- package/dist/packages/work/src/harness/gatekeeper.d.ts +21 -0
- package/dist/packages/work/src/harness/gatekeeper.js +55 -8
- package/dist/packages/work/src/harness/session-store.d.ts +18 -0
- package/dist/packages/work/src/harness/session-store.js +30 -1
- package/dist/packages/work/src/roles/agent-run.js +5 -1
- package/dist/packages/work/src/roles/sessions.d.ts +7 -0
- package/dist/packages/work/src/roles/sessions.js +56 -5
- package/docs/agent-runner.md +42 -3
- package/docs/cli.md +4 -0
- package/package.json +1 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
|
@@ -60,15 +60,17 @@ import { existsSync } from 'node:fs';
|
|
|
60
60
|
import { isWorkdirAllowed } from "./workdir.js";
|
|
61
61
|
import { createApprovalRequester } from "./approvals.js";
|
|
62
62
|
import { createLeaseLoop } from "../lease/loop.js";
|
|
63
|
-
import { isResumeUnavailable } from "../harness/contract.js";
|
|
63
|
+
import { HarnessTurnError, isHarnessTurnError, isResumeUnavailable } from "../harness/contract.js";
|
|
64
64
|
import { preflightStepPermissions } from "../harness/permissions.js";
|
|
65
65
|
import { orderId, } from "../harness/session-store.js";
|
|
66
|
-
import { buildOwenloopMcp, renderBrief, renderRejection, renderReplayBrief, } from "./brief.js";
|
|
66
|
+
import { buildOwenloopMcp, renderColdRecoveryAppendix, renderBrief, renderRecoveryWake, renderRejection, renderReplayBrief, } from "./brief.js";
|
|
67
67
|
import { resolveCapabilityCandidates, selectCandidate, } from "./capability-model.js";
|
|
68
68
|
/** Default confirm-phase cadence: one `get_order` per second. */
|
|
69
69
|
export const DEFAULT_CONFIRM_INTERVAL_MS = 1_000;
|
|
70
70
|
/** Default confirm-phase bound: give the hub 15s to show the submit's outcome. */
|
|
71
71
|
export const DEFAULT_SUBMIT_GRACE_MS = 15_000;
|
|
72
|
+
/** Recovery cleanup is best-effort and must never hold a settled worker forever. */
|
|
73
|
+
const DEFAULT_RECOVERY_STOP_GRACE_MS = 5_000;
|
|
72
74
|
/** Keep capability-silent release diagnostics useful without growing worker logs without bound. */
|
|
73
75
|
const HARNESS_FAILURE_TAIL_SIZE = 4;
|
|
74
76
|
const HARNESS_FAILURE_FRAGMENT_CAP = 700;
|
|
@@ -253,6 +255,7 @@ export function createAgentRunLoop(opts) {
|
|
|
253
255
|
const order = orderId(workflow, runId);
|
|
254
256
|
let signalled = false;
|
|
255
257
|
let leaseSettled = false;
|
|
258
|
+
let settledLeaseOutcome;
|
|
256
259
|
let torndown = false;
|
|
257
260
|
let leasePromise;
|
|
258
261
|
let adapter;
|
|
@@ -270,8 +273,19 @@ export function createAgentRunLoop(opts) {
|
|
|
270
273
|
* `SessionRecord.deliveredReasonAt`. Carried forward from the prior record
|
|
271
274
|
* when this firing delivered no new reasons, so it can never regress. */
|
|
272
275
|
let deliveredReasonAt;
|
|
276
|
+
/** Recovery control is deliberately the only extra data persisted on recovery rows. */
|
|
277
|
+
let recovery;
|
|
278
|
+
let lastDurableActivityAt = 0;
|
|
273
279
|
/** Captures a thrown safety-gate append from the synchronous `started` event. */
|
|
274
280
|
let activePersistenceFailure;
|
|
281
|
+
/**
|
|
282
|
+
* Recovery control rows are a pre-dispatch safety boundary too. Activity is
|
|
283
|
+
* delivered from inside an adapter, so it cannot await cleanup; this signal
|
|
284
|
+
* lets the recovery race stop the exact active session and release instead of
|
|
285
|
+
* misclassifying the writer failure as a provider retry.
|
|
286
|
+
*/
|
|
287
|
+
let recoveryPersistenceFailure;
|
|
288
|
+
let reportRecoveryPersistenceFailure;
|
|
275
289
|
/** Provider-selected model, when a harness reports it after its synchronous start gate. */
|
|
276
290
|
let runtimeModel;
|
|
277
291
|
/** Recent unique harness diagnostics for the capability-silent no-submit log only. */
|
|
@@ -317,7 +331,7 @@ export function createAgentRunLoop(opts) {
|
|
|
317
331
|
const orderReady = new Promise((r) => {
|
|
318
332
|
resolveOrder = r;
|
|
319
333
|
});
|
|
320
|
-
const lease = createLeaseLoop({
|
|
334
|
+
const lease = (opts.leaseFactory ?? createLeaseLoop)({
|
|
321
335
|
hub,
|
|
322
336
|
workflow,
|
|
323
337
|
run: runId,
|
|
@@ -367,9 +381,36 @@ export function createAgentRunLoop(opts) {
|
|
|
367
381
|
status,
|
|
368
382
|
createdAt,
|
|
369
383
|
...(deliveredReasonAt !== undefined ? { deliveredReasonAt } : {}),
|
|
384
|
+
...(recovery !== undefined ? { recovery } : {}),
|
|
370
385
|
updatedAt: at,
|
|
371
386
|
});
|
|
372
387
|
}
|
|
388
|
+
/** Stop one recovery session without allowing provider cleanup to hang the worker. */
|
|
389
|
+
async function stopRecoverySession(ref) {
|
|
390
|
+
if (adapter === undefined)
|
|
391
|
+
return;
|
|
392
|
+
const activeAdapter = adapter;
|
|
393
|
+
let timedOut = false;
|
|
394
|
+
const stopping = Promise.resolve()
|
|
395
|
+
.then(() => activeAdapter.stop(ref))
|
|
396
|
+
.then(() => 'stopped', () => {
|
|
397
|
+
if (!timedOut) {
|
|
398
|
+
opts.err('owenloop work agent-run: recovery session stop failed (details redacted; ignored)');
|
|
399
|
+
}
|
|
400
|
+
return 'failed';
|
|
401
|
+
});
|
|
402
|
+
let timer;
|
|
403
|
+
const timeout = new Promise((resolve) => {
|
|
404
|
+
timer = setTimeout(() => resolve('timeout'), opts.recoveryStopGraceMs ?? DEFAULT_RECOVERY_STOP_GRACE_MS);
|
|
405
|
+
});
|
|
406
|
+
const result = await Promise.race([stopping, timeout]);
|
|
407
|
+
if (timer !== undefined)
|
|
408
|
+
clearTimeout(timer);
|
|
409
|
+
if (result === 'timeout') {
|
|
410
|
+
timedOut = true;
|
|
411
|
+
opts.err('owenloop work agent-run: recovery session stop timed out (details redacted; ignored)');
|
|
412
|
+
}
|
|
413
|
+
}
|
|
373
414
|
/**
|
|
374
415
|
* Tear the harness session down. Guarded so the signal path and the normal
|
|
375
416
|
* end-of-run path together still stop the session exactly once; never throws.
|
|
@@ -378,6 +419,12 @@ export function createAgentRunLoop(opts) {
|
|
|
378
419
|
if (adapter === undefined || sessionRef === undefined || torndown)
|
|
379
420
|
return;
|
|
380
421
|
torndown = true;
|
|
422
|
+
if (recovery !== undefined) {
|
|
423
|
+
const ref = sessionRef;
|
|
424
|
+
sessionRef = undefined;
|
|
425
|
+
await stopRecoverySession(ref);
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
381
428
|
try {
|
|
382
429
|
await adapter.stop(sessionRef);
|
|
383
430
|
}
|
|
@@ -424,7 +471,8 @@ export function createAgentRunLoop(opts) {
|
|
|
424
471
|
switch (e.kind) {
|
|
425
472
|
case 'started':
|
|
426
473
|
sessionRef = e.ref;
|
|
427
|
-
|
|
474
|
+
if (recovery === undefined)
|
|
475
|
+
rememberRuntimeModel(e.model);
|
|
428
476
|
opts.err(`owenloop work agent-run: session started for ${order} (harness '${e.ref.harness}')`);
|
|
429
477
|
// This synchronous event is the cold-start gate. The adapter must not
|
|
430
478
|
// begin provider work until the callback returns. A thrown durable append
|
|
@@ -439,6 +487,11 @@ export function createAgentRunLoop(opts) {
|
|
|
439
487
|
}
|
|
440
488
|
return;
|
|
441
489
|
case 'progress':
|
|
490
|
+
// Recovery logs are a payload-free control surface. The selected adapter
|
|
491
|
+
// suppresses provider prose at source; this second boundary prevents a future
|
|
492
|
+
// adapter regression from printing it through the generic worker logger.
|
|
493
|
+
if (recovery !== undefined)
|
|
494
|
+
return;
|
|
442
495
|
rememberRuntimeModel(e.model);
|
|
443
496
|
rememberHarnessFailure(e.failure);
|
|
444
497
|
opts.err(`owenloop work agent-run: ${e.text}`);
|
|
@@ -451,12 +504,36 @@ export function createAgentRunLoop(opts) {
|
|
|
451
504
|
opts.err('owenloop work agent-run: final response evidence received (redacted)');
|
|
452
505
|
return;
|
|
453
506
|
case 'needs_input':
|
|
507
|
+
if (recovery !== undefined) {
|
|
508
|
+
opts.err('owenloop work agent-run: recovery harness requested input (details redacted)');
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
454
511
|
opts.err(`owenloop work agent-run: WARNING the step agent asked for input and this contract has no reply channel — ${e.question}`);
|
|
455
512
|
return;
|
|
513
|
+
case 'activity':
|
|
514
|
+
if (recovery !== undefined) {
|
|
515
|
+
recovery = { ...recovery, lastActivityAt: e.at, deadlineAt: e.deadlineAt };
|
|
516
|
+
// SDK partial events may arrive per token. Keep the in-memory deadline
|
|
517
|
+
// exact while bounding durable control writes to one per five seconds.
|
|
518
|
+
if (e.at - lastDurableActivityAt >= 5_000) {
|
|
519
|
+
lastDurableActivityAt = e.at;
|
|
520
|
+
try {
|
|
521
|
+
record('active');
|
|
522
|
+
}
|
|
523
|
+
catch (error) {
|
|
524
|
+
recoveryPersistenceFailure ??= error;
|
|
525
|
+
reportRecoveryPersistenceFailure?.(error);
|
|
526
|
+
void teardown();
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
return;
|
|
456
531
|
case 'turn_ended':
|
|
457
532
|
opts.err(`owenloop work agent-run: turn ended for ${order} (telemetry — the hub decides the outcome)`);
|
|
458
533
|
return;
|
|
459
534
|
case 'exited':
|
|
535
|
+
if (recovery !== undefined)
|
|
536
|
+
return;
|
|
460
537
|
rememberHarnessFailure(e.error);
|
|
461
538
|
if (e.error === undefined && e.exitCode !== null && e.exitCode !== 0) {
|
|
462
539
|
rememberHarnessFailure(`harness exited with code ${String(e.exitCode)}`);
|
|
@@ -507,8 +584,9 @@ export function createAgentRunLoop(opts) {
|
|
|
507
584
|
}
|
|
508
585
|
async function run() {
|
|
509
586
|
leasePromise = lease.run();
|
|
510
|
-
void leasePromise.then(() => {
|
|
587
|
+
void leasePromise.then((outcome) => {
|
|
511
588
|
leaseSettled = true;
|
|
589
|
+
settledLeaseOutcome = outcome;
|
|
512
590
|
});
|
|
513
591
|
// First contact race: the order arrives (hold established), or the lease
|
|
514
592
|
// resolves terminally before we ever established it.
|
|
@@ -648,17 +726,54 @@ export function createAgentRunLoop(opts) {
|
|
|
648
726
|
owes: briefOwes(packet),
|
|
649
727
|
};
|
|
650
728
|
const permissions = step.permissions;
|
|
729
|
+
// Adapter-owned configuration is resolved once per worker life. Passing
|
|
730
|
+
// this object through every dispatch prevents a changing environment from
|
|
731
|
+
// changing recovery policy midway through a claimed order.
|
|
732
|
+
let recoveryPolicy;
|
|
733
|
+
let recoveryConfigurationFailure;
|
|
734
|
+
try {
|
|
735
|
+
recoveryPolicy = active.recoveryPolicy?.();
|
|
736
|
+
}
|
|
737
|
+
catch (error) {
|
|
738
|
+
recoveryConfigurationFailure = error;
|
|
739
|
+
}
|
|
740
|
+
// The live owed set is the only legitimate recovery target. `briefOwes`
|
|
741
|
+
// preserves the legacy packet.outputs fallback for old hub projections.
|
|
742
|
+
const recoveryPaths = (briefOwes(packet) ?? [])
|
|
743
|
+
.map((owed) => owed.path)
|
|
744
|
+
.filter((path) => path !== '');
|
|
745
|
+
const recoveryConfigurationIsTerminal = isHarnessTurnError(recoveryConfigurationFailure);
|
|
746
|
+
const recoveryEnabled = recoveryPaths.length === 1 &&
|
|
747
|
+
(recoveryPolicy !== undefined || recoveryConfigurationIsTerminal);
|
|
748
|
+
// A present-but-invalid adapter setting must stop before *any* provider
|
|
749
|
+
// delivery, including multi-output orders that cannot use bounded recovery.
|
|
750
|
+
// The one-output opt-in path turns that terminal configuration failure into
|
|
751
|
+
// the same durable producer-side hold as other terminal failures below.
|
|
752
|
+
if (recoveryConfigurationFailure !== undefined && !recoveryEnabled) {
|
|
753
|
+
opts.err(`owenloop work agent-run: harness recovery configuration is invalid: ${errMsg(recoveryConfigurationFailure)}`);
|
|
754
|
+
return releaseWith('invalid-harness-recovery-configuration', 'incompatible-harness-policy');
|
|
755
|
+
}
|
|
651
756
|
// Mandatory final-boundary preflight. Common checks run first, followed by
|
|
652
|
-
// the selected adapter's exact capability check.
|
|
757
|
+
// the selected adapter's exact capability check. In an eligible recovery
|
|
758
|
+
// firing this is a terminal policy failure and must hold, not re-offer.
|
|
653
759
|
const policyIssues = [
|
|
654
760
|
...preflightStepPermissions(permissions),
|
|
655
761
|
...active.preflight(permissions),
|
|
656
762
|
];
|
|
657
|
-
if (policyIssues.length > 0) {
|
|
763
|
+
if (policyIssues.length > 0 && recoveryEnabled) {
|
|
764
|
+
opts.err('owenloop work agent-run: recovery harness failure category=permission-policy (details redacted)');
|
|
765
|
+
}
|
|
766
|
+
else {
|
|
658
767
|
for (const issue of policyIssues) {
|
|
659
768
|
opts.err(`owenloop work agent-run: harness policy refusal for ${order} on '${resolution.id}'` +
|
|
660
769
|
`${issue.field !== undefined ? ` (${issue.field})` : ''}: ${issue.message}`);
|
|
661
770
|
}
|
|
771
|
+
}
|
|
772
|
+
let recoveryPreflightFailure;
|
|
773
|
+
if (policyIssues.length > 0) {
|
|
774
|
+
recoveryPreflightFailure = new HarnessTurnError('permission-policy', true, 'the selected harness refused the configured permission policy');
|
|
775
|
+
}
|
|
776
|
+
if (recoveryPreflightFailure !== undefined && !recoveryEnabled) {
|
|
662
777
|
return releaseWith('incompatible-harness-policy', 'incompatible-harness-policy');
|
|
663
778
|
}
|
|
664
779
|
if (routing.kind === 'unrouted') {
|
|
@@ -761,7 +876,9 @@ export function createAgentRunLoop(opts) {
|
|
|
761
876
|
client: opts.hub,
|
|
762
877
|
workflow: opts.workflow,
|
|
763
878
|
run: opts.run,
|
|
764
|
-
onEvent: (e) => opts.out(
|
|
879
|
+
onEvent: (e) => opts.out(recoveryEnabled
|
|
880
|
+
? `owenloop work agent-run: recovery approval ${e.kind} (details redacted)`
|
|
881
|
+
: `owenloop work agent-run: approval ${e.kind} — ${e.text}`),
|
|
765
882
|
});
|
|
766
883
|
// A resolved crew-roster candidate wins over the step's authored harness,
|
|
767
884
|
// model, and effort. The harness was already selected above; these values
|
|
@@ -775,6 +892,7 @@ export function createAgentRunLoop(opts) {
|
|
|
775
892
|
permissions,
|
|
776
893
|
approvals,
|
|
777
894
|
...(resolvedModel ?? {}),
|
|
895
|
+
...(recoveryEnabled && recoveryPolicy !== undefined ? { recoveryPolicy } : {}),
|
|
778
896
|
};
|
|
779
897
|
/** Built lazily: a cold start after a refused resume needs a FRESH one. */
|
|
780
898
|
const coldArgs = () => ({
|
|
@@ -791,7 +909,401 @@ export function createAgentRunLoop(opts) {
|
|
|
791
909
|
permissions,
|
|
792
910
|
approvals,
|
|
793
911
|
...(resolvedModel ?? {}),
|
|
912
|
+
...(recoveryEnabled && recoveryPolicy !== undefined ? { recoveryPolicy } : {}),
|
|
794
913
|
});
|
|
914
|
+
if (recoveryEnabled) {
|
|
915
|
+
const recoveryPath = recoveryPaths[0];
|
|
916
|
+
/**
|
|
917
|
+
* The opt-in branch is intentionally self-contained. The ordinary path
|
|
918
|
+
* below is left untouched for adapters/workflows that do not opt in.
|
|
919
|
+
*/
|
|
920
|
+
const runRecovery = async () => {
|
|
921
|
+
const exact = opts.latestRunSession?.(workflow, runId, packet.step) ?? null;
|
|
922
|
+
const saved = exact?.recovery;
|
|
923
|
+
recovery =
|
|
924
|
+
saved !== undefined && saved.generation === runId
|
|
925
|
+
? { ...saved }
|
|
926
|
+
: {
|
|
927
|
+
generation: runId,
|
|
928
|
+
phase: 'primary',
|
|
929
|
+
wakeUsed: false,
|
|
930
|
+
coldRestartUsed: false,
|
|
931
|
+
};
|
|
932
|
+
let resolveRecoveryPersistenceFailure;
|
|
933
|
+
const recoveryPersistenceFailed = new Promise((resolve) => {
|
|
934
|
+
resolveRecoveryPersistenceFailure = resolve;
|
|
935
|
+
});
|
|
936
|
+
let recoveryPersistenceSignalled = false;
|
|
937
|
+
const failRecoveryPersistence = (error) => {
|
|
938
|
+
recoveryPersistenceFailure ??= error;
|
|
939
|
+
if (recoveryPersistenceSignalled)
|
|
940
|
+
return;
|
|
941
|
+
recoveryPersistenceSignalled = true;
|
|
942
|
+
resolveRecoveryPersistenceFailure?.();
|
|
943
|
+
};
|
|
944
|
+
reportRecoveryPersistenceFailure = failRecoveryPersistence;
|
|
945
|
+
if (recoveryPersistenceFailure !== undefined)
|
|
946
|
+
failRecoveryPersistence(recoveryPersistenceFailure);
|
|
947
|
+
const sessionStoreFailed = async () => {
|
|
948
|
+
await teardown();
|
|
949
|
+
return releaseWith('session-store-failed', 'session-store-failed');
|
|
950
|
+
};
|
|
951
|
+
// A saved phase may have dispatched in a prior worker even though this
|
|
952
|
+
// process cannot distinguish that from a crash just after checkpointing.
|
|
953
|
+
// Treat it as post-dispatch so a late hub outcome is still confirmed.
|
|
954
|
+
let providerDispatchStarted = saved !== undefined && saved.generation === runId;
|
|
955
|
+
const setFailure = (failure) => {
|
|
956
|
+
if (!isHarnessTurnError(failure))
|
|
957
|
+
return;
|
|
958
|
+
recovery = {
|
|
959
|
+
...recovery,
|
|
960
|
+
lastFailure: { category: failure.category, at: opts.now() },
|
|
961
|
+
};
|
|
962
|
+
};
|
|
963
|
+
const checkpoint = (phase, changes = {}) => {
|
|
964
|
+
const nextRecovery = {
|
|
965
|
+
...recovery,
|
|
966
|
+
...changes,
|
|
967
|
+
phase,
|
|
968
|
+
phaseStartedAt: opts.now(),
|
|
969
|
+
};
|
|
970
|
+
// A failure describes the phase that just settled. Starting a later
|
|
971
|
+
// provider phase supersedes it so a clean wake/cold turn cannot leave
|
|
972
|
+
// a stale primary timeout in the eventual hold question. The held
|
|
973
|
+
// transition preserves the current phase's failure, when there is one.
|
|
974
|
+
if (phase !== 'held')
|
|
975
|
+
delete nextRecovery.lastFailure;
|
|
976
|
+
recovery = nextRecovery;
|
|
977
|
+
try {
|
|
978
|
+
record('active'); // recovery rows are fsynced by the store.
|
|
979
|
+
return true;
|
|
980
|
+
}
|
|
981
|
+
catch (error) {
|
|
982
|
+
failRecoveryPersistence(error);
|
|
983
|
+
return false;
|
|
984
|
+
}
|
|
985
|
+
};
|
|
986
|
+
const recordRecovery = (status) => {
|
|
987
|
+
try {
|
|
988
|
+
record(status);
|
|
989
|
+
return true;
|
|
990
|
+
}
|
|
991
|
+
catch (error) {
|
|
992
|
+
failRecoveryPersistence(error);
|
|
993
|
+
return false;
|
|
994
|
+
}
|
|
995
|
+
};
|
|
996
|
+
const stopCurrent = async () => {
|
|
997
|
+
if (sessionRef === undefined)
|
|
998
|
+
return;
|
|
999
|
+
const ref = sessionRef;
|
|
1000
|
+
sessionRef = undefined;
|
|
1001
|
+
await stopRecoverySession(ref);
|
|
1002
|
+
};
|
|
1003
|
+
const finishLeaseOutcome = async (outcome) => {
|
|
1004
|
+
if (outcome === 'completed') {
|
|
1005
|
+
if (!recordRecovery('submitted')) {
|
|
1006
|
+
opts.err('owenloop work agent-run: could not persist the submitted recovery diagnostic (hub outcome remains authoritative)');
|
|
1007
|
+
}
|
|
1008
|
+
await stopCurrent();
|
|
1009
|
+
return 'submitted';
|
|
1010
|
+
}
|
|
1011
|
+
if (!recordRecovery('dead')) {
|
|
1012
|
+
opts.err('owenloop work agent-run: could not persist the dead recovery diagnostic (lease outcome remains authoritative)');
|
|
1013
|
+
}
|
|
1014
|
+
const mapped = signalled ? 'killed' : mapLeaseDuringTurn(outcome);
|
|
1015
|
+
await stopCurrent();
|
|
1016
|
+
return mapped;
|
|
1017
|
+
};
|
|
1018
|
+
const finishConfirmedOutcome = async (outcome) => {
|
|
1019
|
+
if (outcome === 'submitted') {
|
|
1020
|
+
lease.stop('submitted', { release: false });
|
|
1021
|
+
await leasePromise;
|
|
1022
|
+
if (!recordRecovery('submitted')) {
|
|
1023
|
+
opts.err('owenloop work agent-run: could not persist the submitted recovery diagnostic (hub outcome remains authoritative)');
|
|
1024
|
+
}
|
|
1025
|
+
await stopCurrent();
|
|
1026
|
+
return 'submitted';
|
|
1027
|
+
}
|
|
1028
|
+
lease.stop('lease-lost', { release: false });
|
|
1029
|
+
await leasePromise;
|
|
1030
|
+
if (!recordRecovery('dead')) {
|
|
1031
|
+
opts.err('owenloop work agent-run: could not persist the dead recovery diagnostic (claim loss remains authoritative)');
|
|
1032
|
+
}
|
|
1033
|
+
await stopCurrent();
|
|
1034
|
+
return 'lease-lost';
|
|
1035
|
+
};
|
|
1036
|
+
const confirmPhase = async () => {
|
|
1037
|
+
if (settledLeaseOutcome !== undefined)
|
|
1038
|
+
return finishLeaseOutcome(settledLeaseOutcome);
|
|
1039
|
+
const confirmed = await Promise.race([
|
|
1040
|
+
confirmOutcome({
|
|
1041
|
+
hub,
|
|
1042
|
+
workflow,
|
|
1043
|
+
run: runId,
|
|
1044
|
+
holder: opts.holder,
|
|
1045
|
+
sleep: opts.sleep,
|
|
1046
|
+
now: opts.now,
|
|
1047
|
+
err: opts.err,
|
|
1048
|
+
intervalMs: opts.confirmIntervalMs ?? DEFAULT_CONFIRM_INTERVAL_MS,
|
|
1049
|
+
graceMs: opts.submitGraceMs ?? DEFAULT_SUBMIT_GRACE_MS,
|
|
1050
|
+
cancelled: () => signalled || leaseSettled,
|
|
1051
|
+
}).then((v) => ({ t: 'confirm', v })),
|
|
1052
|
+
leasePromise.then((o) => ({ t: 'lease', o })),
|
|
1053
|
+
]);
|
|
1054
|
+
if (confirmed.t === 'lease')
|
|
1055
|
+
return finishLeaseOutcome(confirmed.o);
|
|
1056
|
+
if (confirmed.v !== 'no-submit')
|
|
1057
|
+
return finishConfirmedOutcome(confirmed.v);
|
|
1058
|
+
return 'continue';
|
|
1059
|
+
};
|
|
1060
|
+
const sessionStoreFailedAfterDispatch = async () => {
|
|
1061
|
+
// Give the hub authority before cleanup: a hung provider stop can neither
|
|
1062
|
+
// hide a submit nor delay recognition that the claim is already gone.
|
|
1063
|
+
const confirmed = await confirmPhase();
|
|
1064
|
+
if (confirmed !== 'continue')
|
|
1065
|
+
return confirmed;
|
|
1066
|
+
await stopCurrent();
|
|
1067
|
+
return releaseWith('session-store-failed', 'session-store-failed');
|
|
1068
|
+
};
|
|
1069
|
+
const persistenceFailureOutcome = () => providerDispatchStarted ? sessionStoreFailedAfterDispatch() : sessionStoreFailed();
|
|
1070
|
+
const dispatch = async (phase, changes, perform) => {
|
|
1071
|
+
if (!checkpoint(phase, changes)) {
|
|
1072
|
+
return { outcome: await persistenceFailureOutcome() };
|
|
1073
|
+
}
|
|
1074
|
+
// Give the independently running lease one event-loop turn to publish an
|
|
1075
|
+
// outcome that landed after the prior confirm. There is no await between
|
|
1076
|
+
// this check and perform(), making the provider-dispatch boundary atomic
|
|
1077
|
+
// with respect to this worker.
|
|
1078
|
+
const beforeDispatch = await Promise.race([
|
|
1079
|
+
leasePromise.then((outcome) => ({ t: 'lease', outcome })),
|
|
1080
|
+
opts.sleep(0).then(() => ({ t: 'continue' })),
|
|
1081
|
+
]);
|
|
1082
|
+
if (beforeDispatch.t === 'lease') {
|
|
1083
|
+
return { outcome: await finishLeaseOutcome(beforeDispatch.outcome) };
|
|
1084
|
+
}
|
|
1085
|
+
if (signalled || leaseSettled) {
|
|
1086
|
+
const outcome = settledLeaseOutcome ?? await leasePromise;
|
|
1087
|
+
return { outcome: await finishLeaseOutcome(outcome) };
|
|
1088
|
+
}
|
|
1089
|
+
if (activePersistenceFailure !== undefined || recoveryPersistenceFailure !== undefined) {
|
|
1090
|
+
return { outcome: await sessionStoreFailedAfterDispatch() };
|
|
1091
|
+
}
|
|
1092
|
+
providerDispatchStarted = true;
|
|
1093
|
+
const turn = perform().then(() => ({ t: 'turn' }), (failure) => ({ t: 'turn', failure }));
|
|
1094
|
+
const raced = await Promise.race([
|
|
1095
|
+
turn,
|
|
1096
|
+
leasePromise.then((o) => ({ t: 'lease', o })),
|
|
1097
|
+
recoveryPersistenceFailed.then(() => ({ t: 'persistence' })),
|
|
1098
|
+
]);
|
|
1099
|
+
if (raced.t === 'persistence') {
|
|
1100
|
+
return { outcome: await sessionStoreFailedAfterDispatch() };
|
|
1101
|
+
}
|
|
1102
|
+
if (raced.t === 'lease') {
|
|
1103
|
+
return { outcome: await finishLeaseOutcome(raced.o) };
|
|
1104
|
+
}
|
|
1105
|
+
if (activePersistenceFailure !== undefined || recoveryPersistenceFailure !== undefined) {
|
|
1106
|
+
return { outcome: await sessionStoreFailedAfterDispatch() };
|
|
1107
|
+
}
|
|
1108
|
+
const failure = 'failure' in raced ? raced.failure : undefined;
|
|
1109
|
+
if (failure !== undefined) {
|
|
1110
|
+
setFailure(failure);
|
|
1111
|
+
if (isHarnessTurnError(failure)) {
|
|
1112
|
+
opts.err(`owenloop work agent-run: recovery harness failure category=${failure.category}`);
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
if (!recordRecovery('turn-ended'))
|
|
1116
|
+
return { outcome: await sessionStoreFailedAfterDispatch() };
|
|
1117
|
+
const confirmed = await confirmPhase();
|
|
1118
|
+
if (confirmed !== 'continue')
|
|
1119
|
+
return { outcome: confirmed };
|
|
1120
|
+
return failure === undefined ? {} : { failure };
|
|
1121
|
+
};
|
|
1122
|
+
const askHeld = async (heldAfter) => {
|
|
1123
|
+
const facts = recovery;
|
|
1124
|
+
const question = `Harness recovery held ${recoveryPath} after ${heldAfter}; a human decision is required.`;
|
|
1125
|
+
const context = JSON.stringify({
|
|
1126
|
+
generation: facts.generation,
|
|
1127
|
+
phase: facts.phase,
|
|
1128
|
+
wakeUsed: facts.wakeUsed,
|
|
1129
|
+
coldRestartUsed: facts.coldRestartUsed,
|
|
1130
|
+
...(facts.lastFailure !== undefined ? { lastFailure: facts.lastFailure } : {}),
|
|
1131
|
+
...(facts.lastActivityAt !== undefined ? { lastActivityAt: facts.lastActivityAt } : {}),
|
|
1132
|
+
...(facts.deadlineAt !== undefined ? { deadlineAt: facts.deadlineAt } : {}),
|
|
1133
|
+
});
|
|
1134
|
+
// A transport error is ambiguous. Check the hub before one safe ask
|
|
1135
|
+
// retry; neither path can construct a fourth provider dispatch.
|
|
1136
|
+
for (let attemptAsk = 0; attemptAsk < 2; attemptAsk += 1) {
|
|
1137
|
+
try {
|
|
1138
|
+
const asked = await hub.ask({ workflow, run: runId, path: recoveryPath, question, context });
|
|
1139
|
+
if (asked.ok === true && asked.closed === true) {
|
|
1140
|
+
if (!recordRecovery('turn-ended')) {
|
|
1141
|
+
opts.err('owenloop work agent-run: could not persist the held recovery diagnostic (accepted ask remains authoritative)');
|
|
1142
|
+
}
|
|
1143
|
+
lease.stop('recovery-held', { release: false });
|
|
1144
|
+
await leasePromise;
|
|
1145
|
+
return 'held';
|
|
1146
|
+
}
|
|
1147
|
+
if (asked.ok !== true) {
|
|
1148
|
+
opts.err('owenloop work agent-run: recovery ask was refused (details redacted)');
|
|
1149
|
+
if (asked.closed === true) {
|
|
1150
|
+
if (!recordRecovery('dead')) {
|
|
1151
|
+
opts.err('owenloop work agent-run: could not persist the dead recovery diagnostic (closed ask refusal remains authoritative)');
|
|
1152
|
+
}
|
|
1153
|
+
lease.stop('recovery-ask-refused', { release: false });
|
|
1154
|
+
await leasePromise;
|
|
1155
|
+
return 'no-submit';
|
|
1156
|
+
}
|
|
1157
|
+
if (!recordRecovery('dead'))
|
|
1158
|
+
return sessionStoreFailed();
|
|
1159
|
+
return releaseWith('recovery-ask-refused', 'no-submit');
|
|
1160
|
+
}
|
|
1161
|
+
// `ask` is contractually closing. A success envelope that does not
|
|
1162
|
+
// close is transport-ambiguous, so use the same authoritative
|
|
1163
|
+
// outcome check/retry path as a thrown request below.
|
|
1164
|
+
throw new Error('recovery ask was accepted without closing the run');
|
|
1165
|
+
}
|
|
1166
|
+
catch {
|
|
1167
|
+
try {
|
|
1168
|
+
const current = await hub.getOrder({ workflow, run: runId, holder: opts.holder });
|
|
1169
|
+
if (current.lease.outcome !== undefined) {
|
|
1170
|
+
if (!recordRecovery('submitted')) {
|
|
1171
|
+
opts.err('owenloop work agent-run: could not persist the submitted recovery diagnostic (hub outcome remains authoritative)');
|
|
1172
|
+
}
|
|
1173
|
+
lease.stop('submitted', { release: false });
|
|
1174
|
+
await leasePromise;
|
|
1175
|
+
return 'submitted';
|
|
1176
|
+
}
|
|
1177
|
+
if (current.lease.claimed === false) {
|
|
1178
|
+
if (!recordRecovery('dead')) {
|
|
1179
|
+
opts.err('owenloop work agent-run: could not persist the dead recovery diagnostic (claim loss remains authoritative)');
|
|
1180
|
+
}
|
|
1181
|
+
lease.stop('lease-lost', { release: false });
|
|
1182
|
+
await leasePromise;
|
|
1183
|
+
return 'lease-lost';
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
catch {
|
|
1187
|
+
// The second ask is still the only recoverable transport action.
|
|
1188
|
+
}
|
|
1189
|
+
if (attemptAsk === 1) {
|
|
1190
|
+
opts.err('owenloop work agent-run: recovery ask failed (details redacted)');
|
|
1191
|
+
if (!recordRecovery('dead'))
|
|
1192
|
+
return sessionStoreFailed();
|
|
1193
|
+
return releaseWith('recovery-ask-failed', 'hub-unreachable');
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
return 'hub-unreachable';
|
|
1198
|
+
};
|
|
1199
|
+
const hold = async (failure) => {
|
|
1200
|
+
if (failure !== undefined)
|
|
1201
|
+
setFailure(failure);
|
|
1202
|
+
const heldAfter = recovery.lastFailure?.category ?? (recovery.phase !== 'held'
|
|
1203
|
+
? recovery.phase
|
|
1204
|
+
: recovery.coldRestartUsed
|
|
1205
|
+
? 'cold-restart'
|
|
1206
|
+
: recovery.wakeUsed
|
|
1207
|
+
? 'wake'
|
|
1208
|
+
: 'primary');
|
|
1209
|
+
// Persist the terminal transition before stopping a query or attempting
|
|
1210
|
+
// an ask. A process death can underspend recovery, never repeat it.
|
|
1211
|
+
if (!checkpoint('held'))
|
|
1212
|
+
return persistenceFailureOutcome();
|
|
1213
|
+
await stopCurrent();
|
|
1214
|
+
return askHeld(heldAfter);
|
|
1215
|
+
};
|
|
1216
|
+
const prepareColdRestart = () => checkpoint('cold-restart', { wakeUsed: true, coldRestartUsed: true });
|
|
1217
|
+
if (recoveryConfigurationFailure !== undefined) {
|
|
1218
|
+
return hold(recoveryConfigurationFailure);
|
|
1219
|
+
}
|
|
1220
|
+
if (recoveryPreflightFailure !== undefined)
|
|
1221
|
+
return hold(recoveryPreflightFailure);
|
|
1222
|
+
// A re-dispatch of this concrete run consumes the checkpointed phase;
|
|
1223
|
+
// it never repeats a provider call that was already prepared durably.
|
|
1224
|
+
let next = 'primary';
|
|
1225
|
+
if (saved?.generation === runId) {
|
|
1226
|
+
const terminal = saved.lastFailure !== undefined &&
|
|
1227
|
+
['authentication', 'permission-policy', 'model-unavailable', 'configuration'].includes(saved.lastFailure.category);
|
|
1228
|
+
next = terminal || saved.phase === 'held' || saved.coldRestartUsed || saved.phase === 'cold-restart'
|
|
1229
|
+
? 'held'
|
|
1230
|
+
: saved.wakeUsed || saved.phase === 'wake'
|
|
1231
|
+
? 'cold-restart'
|
|
1232
|
+
: 'wake';
|
|
1233
|
+
if (next === 'wake' && exact !== null && exact.token !== '') {
|
|
1234
|
+
sessionRef = { harness: exact.harness, token: exact.token };
|
|
1235
|
+
createdAt = exact.createdAt;
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
if (next === 'primary') {
|
|
1239
|
+
activePersistenceFailure = undefined;
|
|
1240
|
+
const primary = await dispatch('primary', {}, async () => {
|
|
1241
|
+
if (resumable && prev !== null) {
|
|
1242
|
+
sessionRef = { harness: prev.harness, token: prev.token };
|
|
1243
|
+
createdAt = prev.createdAt;
|
|
1244
|
+
await active.deliver(sessionRef, delta.message, deliverArgs, onEvent);
|
|
1245
|
+
markDelivered();
|
|
1246
|
+
return;
|
|
1247
|
+
}
|
|
1248
|
+
const ref = await active.start(coldArgs(), onEvent);
|
|
1249
|
+
sessionRef = ref;
|
|
1250
|
+
markDelivered();
|
|
1251
|
+
});
|
|
1252
|
+
if ('outcome' in primary)
|
|
1253
|
+
return primary.outcome;
|
|
1254
|
+
if (isHarnessTurnError(primary.failure) && primary.failure.terminal) {
|
|
1255
|
+
return hold();
|
|
1256
|
+
}
|
|
1257
|
+
// A primary resume that no longer exists has no live native session to wake.
|
|
1258
|
+
next = isResumeUnavailable(primary.failure) ? 'cold-restart' : 'wake';
|
|
1259
|
+
}
|
|
1260
|
+
if (next === 'wake') {
|
|
1261
|
+
if (sessionRef === undefined && exact?.token !== undefined && exact.token !== '') {
|
|
1262
|
+
sessionRef = { harness: exact.harness, token: exact.token };
|
|
1263
|
+
createdAt = exact.createdAt;
|
|
1264
|
+
}
|
|
1265
|
+
if (sessionRef === undefined) {
|
|
1266
|
+
next = 'cold-restart';
|
|
1267
|
+
}
|
|
1268
|
+
else {
|
|
1269
|
+
const wake = await dispatch('wake', { wakeUsed: true }, async () => {
|
|
1270
|
+
await active.deliver(sessionRef, renderRecoveryWake(recoveryPath, recovery), deliverArgs, onEvent);
|
|
1271
|
+
});
|
|
1272
|
+
if ('outcome' in wake)
|
|
1273
|
+
return wake.outcome;
|
|
1274
|
+
if (isHarnessTurnError(wake.failure) && wake.failure.terminal) {
|
|
1275
|
+
return hold();
|
|
1276
|
+
}
|
|
1277
|
+
next = 'cold-restart';
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
if (next === 'cold-restart') {
|
|
1281
|
+
// This includes the ResumeUnavailable skip. Both the skipped wake
|
|
1282
|
+
// and the cold dispatch are durably consumed before any stop await.
|
|
1283
|
+
if (!prepareColdRestart())
|
|
1284
|
+
return persistenceFailureOutcome();
|
|
1285
|
+
await stopCurrent();
|
|
1286
|
+
createdAt = 0;
|
|
1287
|
+
const cold = await dispatch('cold-restart', { wakeUsed: true, coldRestartUsed: true }, async () => {
|
|
1288
|
+
activePersistenceFailure = undefined;
|
|
1289
|
+
const base = renderReplayBrief(renderBrief(step.brief, spec), {
|
|
1290
|
+
packet,
|
|
1291
|
+
...(prev?.deliveredReasonAt !== undefined ? { deliveredReasonAt: prev.deliveredReasonAt } : {}),
|
|
1292
|
+
});
|
|
1293
|
+
const ref = await active.start({ ...coldArgs(), brief: `${base}\n\n---\n\n${renderColdRecoveryAppendix(recovery)}` }, onEvent);
|
|
1294
|
+
sessionRef = ref;
|
|
1295
|
+
markDelivered();
|
|
1296
|
+
});
|
|
1297
|
+
if ('outcome' in cold)
|
|
1298
|
+
return cold.outcome;
|
|
1299
|
+
if (isHarnessTurnError(cold.failure) && cold.failure.terminal) {
|
|
1300
|
+
return hold();
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
return hold();
|
|
1304
|
+
};
|
|
1305
|
+
return runRecovery();
|
|
1306
|
+
}
|
|
795
1307
|
const path = resumable ? 'resume' : delta.message !== '' ? 'cold replay' : 'cold start';
|
|
796
1308
|
opts.out(`owenloop work agent-run: hosting ${order} (step '${packet.step}', harness '${resolution.id}', attempt ${attempt}, ${path})`);
|
|
797
1309
|
/** Cold-start this firing. Shared by the ordinary path and the fallback. */
|