owenloop 0.5.21 → 0.5.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -60,15 +60,17 @@ import { existsSync } from 'node:fs';
60
60
  import { isWorkdirAllowed } from "./workdir.js";
61
61
  import { createApprovalRequester } from "./approvals.js";
62
62
  import { createLeaseLoop } from "../lease/loop.js";
63
- import { isResumeUnavailable } from "../harness/contract.js";
63
+ import { HarnessTurnError, isHarnessTurnError, isResumeUnavailable } from "../harness/contract.js";
64
64
  import { preflightStepPermissions } from "../harness/permissions.js";
65
65
  import { orderId, } from "../harness/session-store.js";
66
- import { buildOwenloopMcp, renderBrief, renderRejection, renderReplayBrief, } from "./brief.js";
66
+ import { buildOwenloopMcp, renderColdRecoveryAppendix, renderBrief, renderRecoveryWake, renderRejection, renderReplayBrief, } from "./brief.js";
67
67
  import { resolveCapabilityCandidates, selectCandidate, } from "./capability-model.js";
68
68
  /** Default confirm-phase cadence: one `get_order` per second. */
69
69
  export const DEFAULT_CONFIRM_INTERVAL_MS = 1_000;
70
70
  /** Default confirm-phase bound: give the hub 15s to show the submit's outcome. */
71
71
  export const DEFAULT_SUBMIT_GRACE_MS = 15_000;
72
+ /** Recovery cleanup is best-effort and must never hold a settled worker forever. */
73
+ const DEFAULT_RECOVERY_STOP_GRACE_MS = 5_000;
72
74
  /** Keep capability-silent release diagnostics useful without growing worker logs without bound. */
73
75
  const HARNESS_FAILURE_TAIL_SIZE = 4;
74
76
  const HARNESS_FAILURE_FRAGMENT_CAP = 700;
@@ -253,6 +255,7 @@ export function createAgentRunLoop(opts) {
253
255
  const order = orderId(workflow, runId);
254
256
  let signalled = false;
255
257
  let leaseSettled = false;
258
+ let settledLeaseOutcome;
256
259
  let torndown = false;
257
260
  let leasePromise;
258
261
  let adapter;
@@ -270,8 +273,19 @@ export function createAgentRunLoop(opts) {
270
273
  * `SessionRecord.deliveredReasonAt`. Carried forward from the prior record
271
274
  * when this firing delivered no new reasons, so it can never regress. */
272
275
  let deliveredReasonAt;
276
+ /** Recovery control is deliberately the only extra data persisted on recovery rows. */
277
+ let recovery;
278
+ let lastDurableActivityAt = 0;
273
279
  /** Captures a thrown safety-gate append from the synchronous `started` event. */
274
280
  let activePersistenceFailure;
281
+ /**
282
+ * Recovery control rows are a pre-dispatch safety boundary too. Activity is
283
+ * delivered from inside an adapter, so it cannot await cleanup; this signal
284
+ * lets the recovery race stop the exact active session and release instead of
285
+ * misclassifying the writer failure as a provider retry.
286
+ */
287
+ let recoveryPersistenceFailure;
288
+ let reportRecoveryPersistenceFailure;
275
289
  /** Provider-selected model, when a harness reports it after its synchronous start gate. */
276
290
  let runtimeModel;
277
291
  /** Recent unique harness diagnostics for the capability-silent no-submit log only. */
@@ -317,7 +331,7 @@ export function createAgentRunLoop(opts) {
317
331
  const orderReady = new Promise((r) => {
318
332
  resolveOrder = r;
319
333
  });
320
- const lease = createLeaseLoop({
334
+ const lease = (opts.leaseFactory ?? createLeaseLoop)({
321
335
  hub,
322
336
  workflow,
323
337
  run: runId,
@@ -367,9 +381,36 @@ export function createAgentRunLoop(opts) {
367
381
  status,
368
382
  createdAt,
369
383
  ...(deliveredReasonAt !== undefined ? { deliveredReasonAt } : {}),
384
+ ...(recovery !== undefined ? { recovery } : {}),
370
385
  updatedAt: at,
371
386
  });
372
387
  }
388
+ /** Stop one recovery session without allowing provider cleanup to hang the worker. */
389
+ async function stopRecoverySession(ref) {
390
+ if (adapter === undefined)
391
+ return;
392
+ const activeAdapter = adapter;
393
+ let timedOut = false;
394
+ const stopping = Promise.resolve()
395
+ .then(() => activeAdapter.stop(ref))
396
+ .then(() => 'stopped', () => {
397
+ if (!timedOut) {
398
+ opts.err('owenloop work agent-run: recovery session stop failed (details redacted; ignored)');
399
+ }
400
+ return 'failed';
401
+ });
402
+ let timer;
403
+ const timeout = new Promise((resolve) => {
404
+ timer = setTimeout(() => resolve('timeout'), opts.recoveryStopGraceMs ?? DEFAULT_RECOVERY_STOP_GRACE_MS);
405
+ });
406
+ const result = await Promise.race([stopping, timeout]);
407
+ if (timer !== undefined)
408
+ clearTimeout(timer);
409
+ if (result === 'timeout') {
410
+ timedOut = true;
411
+ opts.err('owenloop work agent-run: recovery session stop timed out (details redacted; ignored)');
412
+ }
413
+ }
373
414
  /**
374
415
  * Tear the harness session down. Guarded so the signal path and the normal
375
416
  * end-of-run path together still stop the session exactly once; never throws.
@@ -378,6 +419,12 @@ export function createAgentRunLoop(opts) {
378
419
  if (adapter === undefined || sessionRef === undefined || torndown)
379
420
  return;
380
421
  torndown = true;
422
+ if (recovery !== undefined) {
423
+ const ref = sessionRef;
424
+ sessionRef = undefined;
425
+ await stopRecoverySession(ref);
426
+ return;
427
+ }
381
428
  try {
382
429
  await adapter.stop(sessionRef);
383
430
  }
@@ -424,7 +471,8 @@ export function createAgentRunLoop(opts) {
424
471
  switch (e.kind) {
425
472
  case 'started':
426
473
  sessionRef = e.ref;
427
- rememberRuntimeModel(e.model);
474
+ if (recovery === undefined)
475
+ rememberRuntimeModel(e.model);
428
476
  opts.err(`owenloop work agent-run: session started for ${order} (harness '${e.ref.harness}')`);
429
477
  // This synchronous event is the cold-start gate. The adapter must not
430
478
  // begin provider work until the callback returns. A thrown durable append
@@ -439,6 +487,11 @@ export function createAgentRunLoop(opts) {
439
487
  }
440
488
  return;
441
489
  case 'progress':
490
+ // Recovery logs are a payload-free control surface. The selected adapter
491
+ // suppresses provider prose at source; this second boundary prevents a future
492
+ // adapter regression from printing it through the generic worker logger.
493
+ if (recovery !== undefined)
494
+ return;
442
495
  rememberRuntimeModel(e.model);
443
496
  rememberHarnessFailure(e.failure);
444
497
  opts.err(`owenloop work agent-run: ${e.text}`);
@@ -451,12 +504,36 @@ export function createAgentRunLoop(opts) {
451
504
  opts.err('owenloop work agent-run: final response evidence received (redacted)');
452
505
  return;
453
506
  case 'needs_input':
507
+ if (recovery !== undefined) {
508
+ opts.err('owenloop work agent-run: recovery harness requested input (details redacted)');
509
+ return;
510
+ }
454
511
  opts.err(`owenloop work agent-run: WARNING the step agent asked for input and this contract has no reply channel — ${e.question}`);
455
512
  return;
513
+ case 'activity':
514
+ if (recovery !== undefined) {
515
+ recovery = { ...recovery, lastActivityAt: e.at, deadlineAt: e.deadlineAt };
516
+ // SDK partial events may arrive per token. Keep the in-memory deadline
517
+ // exact while bounding durable control writes to one per five seconds.
518
+ if (e.at - lastDurableActivityAt >= 5_000) {
519
+ lastDurableActivityAt = e.at;
520
+ try {
521
+ record('active');
522
+ }
523
+ catch (error) {
524
+ recoveryPersistenceFailure ??= error;
525
+ reportRecoveryPersistenceFailure?.(error);
526
+ void teardown();
527
+ }
528
+ }
529
+ }
530
+ return;
456
531
  case 'turn_ended':
457
532
  opts.err(`owenloop work agent-run: turn ended for ${order} (telemetry — the hub decides the outcome)`);
458
533
  return;
459
534
  case 'exited':
535
+ if (recovery !== undefined)
536
+ return;
460
537
  rememberHarnessFailure(e.error);
461
538
  if (e.error === undefined && e.exitCode !== null && e.exitCode !== 0) {
462
539
  rememberHarnessFailure(`harness exited with code ${String(e.exitCode)}`);
@@ -507,8 +584,9 @@ export function createAgentRunLoop(opts) {
507
584
  }
508
585
  async function run() {
509
586
  leasePromise = lease.run();
510
- void leasePromise.then(() => {
587
+ void leasePromise.then((outcome) => {
511
588
  leaseSettled = true;
589
+ settledLeaseOutcome = outcome;
512
590
  });
513
591
  // First contact race: the order arrives (hold established), or the lease
514
592
  // resolves terminally before we ever established it.
@@ -648,17 +726,54 @@ export function createAgentRunLoop(opts) {
648
726
  owes: briefOwes(packet),
649
727
  };
650
728
  const permissions = step.permissions;
729
+ // Adapter-owned configuration is resolved once per worker life. Passing
730
+ // this object through every dispatch prevents a changing environment from
731
+ // changing recovery policy midway through a claimed order.
732
+ let recoveryPolicy;
733
+ let recoveryConfigurationFailure;
734
+ try {
735
+ recoveryPolicy = active.recoveryPolicy?.();
736
+ }
737
+ catch (error) {
738
+ recoveryConfigurationFailure = error;
739
+ }
740
+ // The live owed set is the only legitimate recovery target. `briefOwes`
741
+ // preserves the legacy packet.outputs fallback for old hub projections.
742
+ const recoveryPaths = (briefOwes(packet) ?? [])
743
+ .map((owed) => owed.path)
744
+ .filter((path) => path !== '');
745
+ const recoveryConfigurationIsTerminal = isHarnessTurnError(recoveryConfigurationFailure);
746
+ const recoveryEnabled = recoveryPaths.length === 1 &&
747
+ (recoveryPolicy !== undefined || recoveryConfigurationIsTerminal);
748
+ // A present-but-invalid adapter setting must stop before *any* provider
749
+ // delivery, including multi-output orders that cannot use bounded recovery.
750
+ // The one-output opt-in path turns that terminal configuration failure into
751
+ // the same durable producer-side hold as other terminal failures below.
752
+ if (recoveryConfigurationFailure !== undefined && !recoveryEnabled) {
753
+ opts.err(`owenloop work agent-run: harness recovery configuration is invalid: ${errMsg(recoveryConfigurationFailure)}`);
754
+ return releaseWith('invalid-harness-recovery-configuration', 'incompatible-harness-policy');
755
+ }
651
756
  // Mandatory final-boundary preflight. Common checks run first, followed by
652
- // the selected adapter's exact capability check.
757
+ // the selected adapter's exact capability check. In an eligible recovery
758
+ // firing this is a terminal policy failure and must hold, not re-offer.
653
759
  const policyIssues = [
654
760
  ...preflightStepPermissions(permissions),
655
761
  ...active.preflight(permissions),
656
762
  ];
657
- if (policyIssues.length > 0) {
763
+ if (policyIssues.length > 0 && recoveryEnabled) {
764
+ opts.err('owenloop work agent-run: recovery harness failure category=permission-policy (details redacted)');
765
+ }
766
+ else {
658
767
  for (const issue of policyIssues) {
659
768
  opts.err(`owenloop work agent-run: harness policy refusal for ${order} on '${resolution.id}'` +
660
769
  `${issue.field !== undefined ? ` (${issue.field})` : ''}: ${issue.message}`);
661
770
  }
771
+ }
772
+ let recoveryPreflightFailure;
773
+ if (policyIssues.length > 0) {
774
+ recoveryPreflightFailure = new HarnessTurnError('permission-policy', true, 'the selected harness refused the configured permission policy');
775
+ }
776
+ if (recoveryPreflightFailure !== undefined && !recoveryEnabled) {
662
777
  return releaseWith('incompatible-harness-policy', 'incompatible-harness-policy');
663
778
  }
664
779
  if (routing.kind === 'unrouted') {
@@ -761,7 +876,9 @@ export function createAgentRunLoop(opts) {
761
876
  client: opts.hub,
762
877
  workflow: opts.workflow,
763
878
  run: opts.run,
764
- onEvent: (e) => opts.out(`owenloop work agent-run: approval ${e.kind} — ${e.text}`),
879
+ onEvent: (e) => opts.out(recoveryEnabled
880
+ ? `owenloop work agent-run: recovery approval ${e.kind} (details redacted)`
881
+ : `owenloop work agent-run: approval ${e.kind} — ${e.text}`),
765
882
  });
766
883
  // A resolved crew-roster candidate wins over the step's authored harness,
767
884
  // model, and effort. The harness was already selected above; these values
@@ -775,6 +892,7 @@ export function createAgentRunLoop(opts) {
775
892
  permissions,
776
893
  approvals,
777
894
  ...(resolvedModel ?? {}),
895
+ ...(recoveryEnabled && recoveryPolicy !== undefined ? { recoveryPolicy } : {}),
778
896
  };
779
897
  /** Built lazily: a cold start after a refused resume needs a FRESH one. */
780
898
  const coldArgs = () => ({
@@ -791,7 +909,401 @@ export function createAgentRunLoop(opts) {
791
909
  permissions,
792
910
  approvals,
793
911
  ...(resolvedModel ?? {}),
912
+ ...(recoveryEnabled && recoveryPolicy !== undefined ? { recoveryPolicy } : {}),
794
913
  });
914
+ if (recoveryEnabled) {
915
+ const recoveryPath = recoveryPaths[0];
916
+ /**
917
+ * The opt-in branch is intentionally self-contained. The ordinary path
918
+ * below is left untouched for adapters/workflows that do not opt in.
919
+ */
920
+ const runRecovery = async () => {
921
+ const exact = opts.latestRunSession?.(workflow, runId, packet.step) ?? null;
922
+ const saved = exact?.recovery;
923
+ recovery =
924
+ saved !== undefined && saved.generation === runId
925
+ ? { ...saved }
926
+ : {
927
+ generation: runId,
928
+ phase: 'primary',
929
+ wakeUsed: false,
930
+ coldRestartUsed: false,
931
+ };
932
+ let resolveRecoveryPersistenceFailure;
933
+ const recoveryPersistenceFailed = new Promise((resolve) => {
934
+ resolveRecoveryPersistenceFailure = resolve;
935
+ });
936
+ let recoveryPersistenceSignalled = false;
937
+ const failRecoveryPersistence = (error) => {
938
+ recoveryPersistenceFailure ??= error;
939
+ if (recoveryPersistenceSignalled)
940
+ return;
941
+ recoveryPersistenceSignalled = true;
942
+ resolveRecoveryPersistenceFailure?.();
943
+ };
944
+ reportRecoveryPersistenceFailure = failRecoveryPersistence;
945
+ if (recoveryPersistenceFailure !== undefined)
946
+ failRecoveryPersistence(recoveryPersistenceFailure);
947
+ const sessionStoreFailed = async () => {
948
+ await teardown();
949
+ return releaseWith('session-store-failed', 'session-store-failed');
950
+ };
951
+ // A saved phase may have dispatched in a prior worker even though this
952
+ // process cannot distinguish that from a crash just after checkpointing.
953
+ // Treat it as post-dispatch so a late hub outcome is still confirmed.
954
+ let providerDispatchStarted = saved !== undefined && saved.generation === runId;
955
+ const setFailure = (failure) => {
956
+ if (!isHarnessTurnError(failure))
957
+ return;
958
+ recovery = {
959
+ ...recovery,
960
+ lastFailure: { category: failure.category, at: opts.now() },
961
+ };
962
+ };
963
+ const checkpoint = (phase, changes = {}) => {
964
+ const nextRecovery = {
965
+ ...recovery,
966
+ ...changes,
967
+ phase,
968
+ phaseStartedAt: opts.now(),
969
+ };
970
+ // A failure describes the phase that just settled. Starting a later
971
+ // provider phase supersedes it so a clean wake/cold turn cannot leave
972
+ // a stale primary timeout in the eventual hold question. The held
973
+ // transition preserves the current phase's failure, when there is one.
974
+ if (phase !== 'held')
975
+ delete nextRecovery.lastFailure;
976
+ recovery = nextRecovery;
977
+ try {
978
+ record('active'); // recovery rows are fsynced by the store.
979
+ return true;
980
+ }
981
+ catch (error) {
982
+ failRecoveryPersistence(error);
983
+ return false;
984
+ }
985
+ };
986
+ const recordRecovery = (status) => {
987
+ try {
988
+ record(status);
989
+ return true;
990
+ }
991
+ catch (error) {
992
+ failRecoveryPersistence(error);
993
+ return false;
994
+ }
995
+ };
996
+ const stopCurrent = async () => {
997
+ if (sessionRef === undefined)
998
+ return;
999
+ const ref = sessionRef;
1000
+ sessionRef = undefined;
1001
+ await stopRecoverySession(ref);
1002
+ };
1003
+ const finishLeaseOutcome = async (outcome) => {
1004
+ if (outcome === 'completed') {
1005
+ if (!recordRecovery('submitted')) {
1006
+ opts.err('owenloop work agent-run: could not persist the submitted recovery diagnostic (hub outcome remains authoritative)');
1007
+ }
1008
+ await stopCurrent();
1009
+ return 'submitted';
1010
+ }
1011
+ if (!recordRecovery('dead')) {
1012
+ opts.err('owenloop work agent-run: could not persist the dead recovery diagnostic (lease outcome remains authoritative)');
1013
+ }
1014
+ const mapped = signalled ? 'killed' : mapLeaseDuringTurn(outcome);
1015
+ await stopCurrent();
1016
+ return mapped;
1017
+ };
1018
+ const finishConfirmedOutcome = async (outcome) => {
1019
+ if (outcome === 'submitted') {
1020
+ lease.stop('submitted', { release: false });
1021
+ await leasePromise;
1022
+ if (!recordRecovery('submitted')) {
1023
+ opts.err('owenloop work agent-run: could not persist the submitted recovery diagnostic (hub outcome remains authoritative)');
1024
+ }
1025
+ await stopCurrent();
1026
+ return 'submitted';
1027
+ }
1028
+ lease.stop('lease-lost', { release: false });
1029
+ await leasePromise;
1030
+ if (!recordRecovery('dead')) {
1031
+ opts.err('owenloop work agent-run: could not persist the dead recovery diagnostic (claim loss remains authoritative)');
1032
+ }
1033
+ await stopCurrent();
1034
+ return 'lease-lost';
1035
+ };
1036
+ const confirmPhase = async () => {
1037
+ if (settledLeaseOutcome !== undefined)
1038
+ return finishLeaseOutcome(settledLeaseOutcome);
1039
+ const confirmed = await Promise.race([
1040
+ confirmOutcome({
1041
+ hub,
1042
+ workflow,
1043
+ run: runId,
1044
+ holder: opts.holder,
1045
+ sleep: opts.sleep,
1046
+ now: opts.now,
1047
+ err: opts.err,
1048
+ intervalMs: opts.confirmIntervalMs ?? DEFAULT_CONFIRM_INTERVAL_MS,
1049
+ graceMs: opts.submitGraceMs ?? DEFAULT_SUBMIT_GRACE_MS,
1050
+ cancelled: () => signalled || leaseSettled,
1051
+ }).then((v) => ({ t: 'confirm', v })),
1052
+ leasePromise.then((o) => ({ t: 'lease', o })),
1053
+ ]);
1054
+ if (confirmed.t === 'lease')
1055
+ return finishLeaseOutcome(confirmed.o);
1056
+ if (confirmed.v !== 'no-submit')
1057
+ return finishConfirmedOutcome(confirmed.v);
1058
+ return 'continue';
1059
+ };
1060
+ const sessionStoreFailedAfterDispatch = async () => {
1061
+ // Give the hub authority before cleanup: a hung provider stop can neither
1062
+ // hide a submit nor delay recognition that the claim is already gone.
1063
+ const confirmed = await confirmPhase();
1064
+ if (confirmed !== 'continue')
1065
+ return confirmed;
1066
+ await stopCurrent();
1067
+ return releaseWith('session-store-failed', 'session-store-failed');
1068
+ };
1069
+ const persistenceFailureOutcome = () => providerDispatchStarted ? sessionStoreFailedAfterDispatch() : sessionStoreFailed();
1070
+ const dispatch = async (phase, changes, perform) => {
1071
+ if (!checkpoint(phase, changes)) {
1072
+ return { outcome: await persistenceFailureOutcome() };
1073
+ }
1074
+ // Give the independently running lease one event-loop turn to publish an
1075
+ // outcome that landed after the prior confirm. There is no await between
1076
+ // this check and perform(), making the provider-dispatch boundary atomic
1077
+ // with respect to this worker.
1078
+ const beforeDispatch = await Promise.race([
1079
+ leasePromise.then((outcome) => ({ t: 'lease', outcome })),
1080
+ opts.sleep(0).then(() => ({ t: 'continue' })),
1081
+ ]);
1082
+ if (beforeDispatch.t === 'lease') {
1083
+ return { outcome: await finishLeaseOutcome(beforeDispatch.outcome) };
1084
+ }
1085
+ if (signalled || leaseSettled) {
1086
+ const outcome = settledLeaseOutcome ?? await leasePromise;
1087
+ return { outcome: await finishLeaseOutcome(outcome) };
1088
+ }
1089
+ if (activePersistenceFailure !== undefined || recoveryPersistenceFailure !== undefined) {
1090
+ return { outcome: await sessionStoreFailedAfterDispatch() };
1091
+ }
1092
+ providerDispatchStarted = true;
1093
+ const turn = perform().then(() => ({ t: 'turn' }), (failure) => ({ t: 'turn', failure }));
1094
+ const raced = await Promise.race([
1095
+ turn,
1096
+ leasePromise.then((o) => ({ t: 'lease', o })),
1097
+ recoveryPersistenceFailed.then(() => ({ t: 'persistence' })),
1098
+ ]);
1099
+ if (raced.t === 'persistence') {
1100
+ return { outcome: await sessionStoreFailedAfterDispatch() };
1101
+ }
1102
+ if (raced.t === 'lease') {
1103
+ return { outcome: await finishLeaseOutcome(raced.o) };
1104
+ }
1105
+ if (activePersistenceFailure !== undefined || recoveryPersistenceFailure !== undefined) {
1106
+ return { outcome: await sessionStoreFailedAfterDispatch() };
1107
+ }
1108
+ const failure = 'failure' in raced ? raced.failure : undefined;
1109
+ if (failure !== undefined) {
1110
+ setFailure(failure);
1111
+ if (isHarnessTurnError(failure)) {
1112
+ opts.err(`owenloop work agent-run: recovery harness failure category=${failure.category}`);
1113
+ }
1114
+ }
1115
+ if (!recordRecovery('turn-ended'))
1116
+ return { outcome: await sessionStoreFailedAfterDispatch() };
1117
+ const confirmed = await confirmPhase();
1118
+ if (confirmed !== 'continue')
1119
+ return { outcome: confirmed };
1120
+ return failure === undefined ? {} : { failure };
1121
+ };
1122
+ const askHeld = async (heldAfter) => {
1123
+ const facts = recovery;
1124
+ const question = `Harness recovery held ${recoveryPath} after ${heldAfter}; a human decision is required.`;
1125
+ const context = JSON.stringify({
1126
+ generation: facts.generation,
1127
+ phase: facts.phase,
1128
+ wakeUsed: facts.wakeUsed,
1129
+ coldRestartUsed: facts.coldRestartUsed,
1130
+ ...(facts.lastFailure !== undefined ? { lastFailure: facts.lastFailure } : {}),
1131
+ ...(facts.lastActivityAt !== undefined ? { lastActivityAt: facts.lastActivityAt } : {}),
1132
+ ...(facts.deadlineAt !== undefined ? { deadlineAt: facts.deadlineAt } : {}),
1133
+ });
1134
+ // A transport error is ambiguous. Check the hub before one safe ask
1135
+ // retry; neither path can construct a fourth provider dispatch.
1136
+ for (let attemptAsk = 0; attemptAsk < 2; attemptAsk += 1) {
1137
+ try {
1138
+ const asked = await hub.ask({ workflow, run: runId, path: recoveryPath, question, context });
1139
+ if (asked.ok === true && asked.closed === true) {
1140
+ if (!recordRecovery('turn-ended')) {
1141
+ opts.err('owenloop work agent-run: could not persist the held recovery diagnostic (accepted ask remains authoritative)');
1142
+ }
1143
+ lease.stop('recovery-held', { release: false });
1144
+ await leasePromise;
1145
+ return 'held';
1146
+ }
1147
+ if (asked.ok !== true) {
1148
+ opts.err('owenloop work agent-run: recovery ask was refused (details redacted)');
1149
+ if (asked.closed === true) {
1150
+ if (!recordRecovery('dead')) {
1151
+ opts.err('owenloop work agent-run: could not persist the dead recovery diagnostic (closed ask refusal remains authoritative)');
1152
+ }
1153
+ lease.stop('recovery-ask-refused', { release: false });
1154
+ await leasePromise;
1155
+ return 'no-submit';
1156
+ }
1157
+ if (!recordRecovery('dead'))
1158
+ return sessionStoreFailed();
1159
+ return releaseWith('recovery-ask-refused', 'no-submit');
1160
+ }
1161
+ // `ask` is contractually closing. A success envelope that does not
1162
+ // close is transport-ambiguous, so use the same authoritative
1163
+ // outcome check/retry path as a thrown request below.
1164
+ throw new Error('recovery ask was accepted without closing the run');
1165
+ }
1166
+ catch {
1167
+ try {
1168
+ const current = await hub.getOrder({ workflow, run: runId, holder: opts.holder });
1169
+ if (current.lease.outcome !== undefined) {
1170
+ if (!recordRecovery('submitted')) {
1171
+ opts.err('owenloop work agent-run: could not persist the submitted recovery diagnostic (hub outcome remains authoritative)');
1172
+ }
1173
+ lease.stop('submitted', { release: false });
1174
+ await leasePromise;
1175
+ return 'submitted';
1176
+ }
1177
+ if (current.lease.claimed === false) {
1178
+ if (!recordRecovery('dead')) {
1179
+ opts.err('owenloop work agent-run: could not persist the dead recovery diagnostic (claim loss remains authoritative)');
1180
+ }
1181
+ lease.stop('lease-lost', { release: false });
1182
+ await leasePromise;
1183
+ return 'lease-lost';
1184
+ }
1185
+ }
1186
+ catch {
1187
+ // The second ask is still the only recoverable transport action.
1188
+ }
1189
+ if (attemptAsk === 1) {
1190
+ opts.err('owenloop work agent-run: recovery ask failed (details redacted)');
1191
+ if (!recordRecovery('dead'))
1192
+ return sessionStoreFailed();
1193
+ return releaseWith('recovery-ask-failed', 'hub-unreachable');
1194
+ }
1195
+ }
1196
+ }
1197
+ return 'hub-unreachable';
1198
+ };
1199
+ const hold = async (failure) => {
1200
+ if (failure !== undefined)
1201
+ setFailure(failure);
1202
+ const heldAfter = recovery.lastFailure?.category ?? (recovery.phase !== 'held'
1203
+ ? recovery.phase
1204
+ : recovery.coldRestartUsed
1205
+ ? 'cold-restart'
1206
+ : recovery.wakeUsed
1207
+ ? 'wake'
1208
+ : 'primary');
1209
+ // Persist the terminal transition before stopping a query or attempting
1210
+ // an ask. A process death can underspend recovery, never repeat it.
1211
+ if (!checkpoint('held'))
1212
+ return persistenceFailureOutcome();
1213
+ await stopCurrent();
1214
+ return askHeld(heldAfter);
1215
+ };
1216
+ const prepareColdRestart = () => checkpoint('cold-restart', { wakeUsed: true, coldRestartUsed: true });
1217
+ if (recoveryConfigurationFailure !== undefined) {
1218
+ return hold(recoveryConfigurationFailure);
1219
+ }
1220
+ if (recoveryPreflightFailure !== undefined)
1221
+ return hold(recoveryPreflightFailure);
1222
+ // A re-dispatch of this concrete run consumes the checkpointed phase;
1223
+ // it never repeats a provider call that was already prepared durably.
1224
+ let next = 'primary';
1225
+ if (saved?.generation === runId) {
1226
+ const terminal = saved.lastFailure !== undefined &&
1227
+ ['authentication', 'permission-policy', 'model-unavailable', 'configuration'].includes(saved.lastFailure.category);
1228
+ next = terminal || saved.phase === 'held' || saved.coldRestartUsed || saved.phase === 'cold-restart'
1229
+ ? 'held'
1230
+ : saved.wakeUsed || saved.phase === 'wake'
1231
+ ? 'cold-restart'
1232
+ : 'wake';
1233
+ if (next === 'wake' && exact !== null && exact.token !== '') {
1234
+ sessionRef = { harness: exact.harness, token: exact.token };
1235
+ createdAt = exact.createdAt;
1236
+ }
1237
+ }
1238
+ if (next === 'primary') {
1239
+ activePersistenceFailure = undefined;
1240
+ const primary = await dispatch('primary', {}, async () => {
1241
+ if (resumable && prev !== null) {
1242
+ sessionRef = { harness: prev.harness, token: prev.token };
1243
+ createdAt = prev.createdAt;
1244
+ await active.deliver(sessionRef, delta.message, deliverArgs, onEvent);
1245
+ markDelivered();
1246
+ return;
1247
+ }
1248
+ const ref = await active.start(coldArgs(), onEvent);
1249
+ sessionRef = ref;
1250
+ markDelivered();
1251
+ });
1252
+ if ('outcome' in primary)
1253
+ return primary.outcome;
1254
+ if (isHarnessTurnError(primary.failure) && primary.failure.terminal) {
1255
+ return hold();
1256
+ }
1257
+ // A primary resume that no longer exists has no live native session to wake.
1258
+ next = isResumeUnavailable(primary.failure) ? 'cold-restart' : 'wake';
1259
+ }
1260
+ if (next === 'wake') {
1261
+ if (sessionRef === undefined && exact?.token !== undefined && exact.token !== '') {
1262
+ sessionRef = { harness: exact.harness, token: exact.token };
1263
+ createdAt = exact.createdAt;
1264
+ }
1265
+ if (sessionRef === undefined) {
1266
+ next = 'cold-restart';
1267
+ }
1268
+ else {
1269
+ const wake = await dispatch('wake', { wakeUsed: true }, async () => {
1270
+ await active.deliver(sessionRef, renderRecoveryWake(recoveryPath, recovery), deliverArgs, onEvent);
1271
+ });
1272
+ if ('outcome' in wake)
1273
+ return wake.outcome;
1274
+ if (isHarnessTurnError(wake.failure) && wake.failure.terminal) {
1275
+ return hold();
1276
+ }
1277
+ next = 'cold-restart';
1278
+ }
1279
+ }
1280
+ if (next === 'cold-restart') {
1281
+ // This includes the ResumeUnavailable skip. Both the skipped wake
1282
+ // and the cold dispatch are durably consumed before any stop await.
1283
+ if (!prepareColdRestart())
1284
+ return persistenceFailureOutcome();
1285
+ await stopCurrent();
1286
+ createdAt = 0;
1287
+ const cold = await dispatch('cold-restart', { wakeUsed: true, coldRestartUsed: true }, async () => {
1288
+ activePersistenceFailure = undefined;
1289
+ const base = renderReplayBrief(renderBrief(step.brief, spec), {
1290
+ packet,
1291
+ ...(prev?.deliveredReasonAt !== undefined ? { deliveredReasonAt: prev.deliveredReasonAt } : {}),
1292
+ });
1293
+ const ref = await active.start({ ...coldArgs(), brief: `${base}\n\n---\n\n${renderColdRecoveryAppendix(recovery)}` }, onEvent);
1294
+ sessionRef = ref;
1295
+ markDelivered();
1296
+ });
1297
+ if ('outcome' in cold)
1298
+ return cold.outcome;
1299
+ if (isHarnessTurnError(cold.failure) && cold.failure.terminal) {
1300
+ return hold();
1301
+ }
1302
+ }
1303
+ return hold();
1304
+ };
1305
+ return runRecovery();
1306
+ }
795
1307
  const path = resumable ? 'resume' : delta.message !== '' ? 'cold replay' : 'cold start';
796
1308
  opts.out(`owenloop work agent-run: hosting ${order} (step '${packet.step}', harness '${resolution.id}', attempt ${attempt}, ${path})`);
797
1309
  /** Cold-start this firing. Shared by the ordinary path and the fallback. */