@opengeni/worker-bundle 0.20.0 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,7 +16,10 @@ import type * as activities from "../activities";
16
16
  import {
17
17
  ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
18
18
  ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
19
+ PRE_CLAIM_FAILURE_MESSAGE,
20
+ PRE_CLAIM_FAILURE_TYPE,
19
21
  type EscapedMcpTimeoutRecoveryDetail,
22
+ type PreClaimFailureDisposition,
20
23
  } from "../activities/types";
21
24
  import {
22
25
  activity,
@@ -89,6 +92,37 @@ export function deferredResultMayContinue(entryWakeups: number, currentWakeups:
89
92
  return currentWakeups !== entryWakeups;
90
93
  }
91
94
 
95
+ /**
96
+ * Bound repeated failures that happen before an attempt row exists. The
97
+ * activity mirrors this deterministic delay into the durable wake outbox, so
98
+ * a live workflow and a replacement workflow observe the same retry floor.
99
+ */
100
+ export function unclaimedAttemptRetryDelayMs(consecutiveFailures: number): number {
101
+ if (!Number.isFinite(consecutiveFailures) || consecutiveFailures <= 0) return 1_000;
102
+ const exponent = Math.min(6, Math.max(0, Math.trunc(consecutiveFailures) - 1));
103
+ return Math.min(60_000, 1_000 * 2 ** exponent);
104
+ }
105
+
106
+ export type SessionWakeCounters = {
107
+ wakeups: number;
108
+ interruptionWakeups: number;
109
+ approvalWakeups: number;
110
+ capacityWakeups: number;
111
+ };
112
+
113
+ /** Any signal that can make the failed admission runnable interrupts backoff. */
114
+ export function unclaimedAttemptWakeChanged(
115
+ baseline: SessionWakeCounters,
116
+ current: SessionWakeCounters,
117
+ ): boolean {
118
+ return (
119
+ current.wakeups !== baseline.wakeups ||
120
+ current.interruptionWakeups !== baseline.interruptionWakeups ||
121
+ current.approvalWakeups !== baseline.approvalWakeups ||
122
+ current.capacityWakeups !== baseline.capacityWakeups
123
+ );
124
+ }
125
+
92
126
  /** Deterministic Temporal timer delay for a persisted structured-input deadline. */
93
127
  export function humanInputDeadlineWaitMs(expiresAt: string, nowMs = Date.now()): number {
94
128
  const deadline = Date.parse(expiresAt);
@@ -165,6 +199,40 @@ export function escapedMcpTimeoutRecoveryDetail(
165
199
  };
166
200
  }
167
201
 
202
+ /** Read only the upgraded turn worker's explicit pre-claim wire contract. */
203
+ export function preClaimFailureDisposition(error: unknown): PreClaimFailureDisposition | undefined {
204
+ if (!(error instanceof ActivityFailure) || error.activityType !== "runAgentTurn") {
205
+ return undefined;
206
+ }
207
+ const cause = error.cause;
208
+ if (
209
+ !(cause instanceof ApplicationFailure) ||
210
+ cause.type !== PRE_CLAIM_FAILURE_TYPE ||
211
+ cause.message !== PRE_CLAIM_FAILURE_MESSAGE
212
+ ) {
213
+ return undefined;
214
+ }
215
+ const detail = cause.details?.[0] as { disposition?: unknown; code?: unknown } | undefined;
216
+ if (
217
+ detail?.code !== "db_deadlock" &&
218
+ detail?.code !== "db_serialization_failure" &&
219
+ detail?.code !== "db_failure" &&
220
+ detail?.code !== "claim_invariant"
221
+ ) {
222
+ return undefined;
223
+ }
224
+ const disposition = detail.disposition;
225
+ if (disposition !== "retryable" && disposition !== "permanent") return undefined;
226
+ if (
227
+ (detail.code === "db_deadlock" || detail.code === "db_serialization_failure") &&
228
+ disposition !== "retryable"
229
+ ) {
230
+ return undefined;
231
+ }
232
+ if (detail.code === "claim_invariant" && disposition !== "permanent") return undefined;
233
+ return disposition;
234
+ }
235
+
168
236
  /**
169
237
  * Classify only the exact turn-fence cancellation protocol used to arbitrate a
170
238
  * database control commit that reaches the activity just before its Temporal
@@ -216,6 +284,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
216
284
  const receiptGatedCancellation = patched("session-attempt-quiescence-v2");
217
285
  const writerSetQuiescenceRecovery = patched("session-attempt-writer-set-quiescence-v1");
218
286
  const staleControlSignalIsOnlyWakeHint = patched("session-control-stale-wake-v1");
287
+ const unclaimedAttemptRecovery = patched("session-unclaimed-attempt-recovery-v1");
219
288
  const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue, receiptGatedCancellation);
220
289
  let approvalWakeups = 0;
221
290
  let interruptionWakeups = 0;
@@ -230,6 +299,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
230
299
  // weeks-long session survivable.
231
300
  let turnsThisRun = 0;
232
301
  let capacityChecksThisRun = 0;
302
+ let unclaimedAttemptFailures = 0;
233
303
 
234
304
  setHandler(userMessage, () => {
235
305
  signalVersion += 1;
@@ -579,6 +649,16 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
579
649
  trigger: activities.RunAgentTurnInput["trigger"],
580
650
  ): Promise<boolean> {
581
651
  const capacityWaitEntryBaseline = { wakeups, capacityWakeups };
652
+ // Capture every admission-relevant signal before activity dispatch. A
653
+ // signal may arrive while runAgentTurn is still failing before claim, or
654
+ // while the failure-control activity settles. Either must interrupt the
655
+ // bounded recovery timer instead of being erased by a later baseline.
656
+ const preDispatchRetryWakeBaseline: SessionWakeCounters = {
657
+ wakeups,
658
+ interruptionWakeups,
659
+ approvalWakeups,
660
+ capacityWakeups,
661
+ };
582
662
  const attemptId = uuid4();
583
663
  let interruptionBaseline = interruptionWakeups;
584
664
 
@@ -763,16 +843,127 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
763
843
  // truth when the bounded redispatch ceiling was exceeded.
764
844
  return false;
765
845
  }
766
- await activity.failSessionAttempt({
767
- accountId,
768
- workspaceId,
769
- sessionId,
770
- attemptId,
771
- error: workflowFailureMessage(outcome.error),
772
- });
773
- return false;
846
+ const postDispatchRetryWakeBaseline: SessionWakeCounters = {
847
+ wakeups,
848
+ interruptionWakeups,
849
+ approvalWakeups,
850
+ capacityWakeups,
851
+ };
852
+ if (!unclaimedAttemptRecovery) {
853
+ // Replay compatibility: histories recorded before v2 scheduled this
854
+ // exact argument shape and treated the activity as void. Preserve the
855
+ // completed-history tail during replay, but keep a still-open legacy
856
+ // history alive after its activity completes. The nested patch marker
857
+ // is recorded only when an old history reaches this previously absent
858
+ // tail on a live workflow task; already-completed histories replay the
859
+ // original close without emitting a new timer command.
860
+ const legacyFailure: activities.FailSessionAttemptResult | undefined =
861
+ await activity.failSessionAttempt({
862
+ accountId,
863
+ workspaceId,
864
+ sessionId,
865
+ attemptId,
866
+ error: workflowFailureMessage(outcome.error),
867
+ });
868
+ if (!patched("session-legacy-unclaimed-attempt-tail-v1")) {
869
+ return false;
870
+ }
871
+ if (legacyFailure?.action === "failed" || legacyFailure?.action === "terminal") {
872
+ return false;
873
+ }
874
+ const retryDelayMs = unclaimedAttemptRetryDelayMs(unclaimedAttemptFailures + 1);
875
+ // Histories that already recorded the legacy tail used a post-control
876
+ // two-signal baseline. Preserve that exact condition on replay; an old
877
+ // open history reaching this tail for the first time can adopt the
878
+ // pre-dispatch four-signal contract.
879
+ const upgradedLegacySignalBaseline = patched("session-legacy-preclaim-signal-baseline-v1");
880
+ const retryWakeBaseline = upgradedLegacySignalBaseline
881
+ ? preDispatchRetryWakeBaseline
882
+ : {
883
+ wakeups,
884
+ interruptionWakeups,
885
+ approvalWakeups,
886
+ capacityWakeups,
887
+ };
888
+ unclaimedAttemptFailures += 1;
889
+ await condition(() => {
890
+ const current = {
891
+ wakeups,
892
+ interruptionWakeups,
893
+ approvalWakeups,
894
+ capacityWakeups,
895
+ };
896
+ return upgradedLegacySignalBaseline
897
+ ? unclaimedAttemptWakeChanged(retryWakeBaseline, current)
898
+ : current.interruptionWakeups !== retryWakeBaseline.interruptionWakeups ||
899
+ current.wakeups !== retryWakeBaseline.wakeups;
900
+ }, retryDelayMs);
901
+ return true;
902
+ }
903
+ const retryDelayMs = unclaimedAttemptRetryDelayMs(unclaimedAttemptFailures + 1);
904
+ const admissionFailureDisposition = preClaimFailureDisposition(outcome.error);
905
+ // Keep this marker immediately adjacent to the changed command. A
906
+ // history that already recorded the v2 activity replays the old shape;
907
+ // an open history that has never reached this branch can record v3.
908
+ const classifiedPreClaimFailure = patched("session-preclaim-failure-classification-v1");
909
+ const retryWakeBaseline = classifiedPreClaimFailure
910
+ ? preDispatchRetryWakeBaseline
911
+ : postDispatchRetryWakeBaseline;
912
+ const failure: activities.FailSessionAttemptResult | undefined =
913
+ await activity.failSessionAttempt(
914
+ classifiedPreClaimFailure
915
+ ? {
916
+ accountId,
917
+ workspaceId,
918
+ sessionId,
919
+ attemptId,
920
+ workflowId: workflowInfo().workflowId,
921
+ retryDelayMs,
922
+ ...(admissionFailureDisposition
923
+ ? { preClaimFailureDisposition: admissionFailureDisposition }
924
+ : {}),
925
+ trigger,
926
+ error: workflowFailureMessage(outcome.error),
927
+ }
928
+ : {
929
+ // Replay the exact v2 command shape. Adding optional fields to
930
+ // a Temporal activity argument still changes command history.
931
+ accountId,
932
+ workspaceId,
933
+ sessionId,
934
+ attemptId,
935
+ workflowId: workflowInfo().workflowId,
936
+ retryDelayMs,
937
+ error: workflowFailureMessage(outcome.error),
938
+ },
939
+ );
940
+ // During a rolling deploy an upgraded workflow worker can schedule this
941
+ // activity on a legacy control worker whose wire result was void. Treat
942
+ // that unknown commit outcome like an unclaimed attempt: wait, then
943
+ // re-read durable state. A legacy worker that settled the failure leaves
944
+ // an idle turn; one that no-op'd before claim leaves recoverable work.
945
+ // Neither path replays model or tool side effects speculatively.
946
+ if (!failure || failure.action === "unclaimed") {
947
+ unclaimedAttemptFailures += 1;
948
+ await condition(() => {
949
+ const current = {
950
+ wakeups,
951
+ interruptionWakeups,
952
+ approvalWakeups,
953
+ capacityWakeups,
954
+ };
955
+ return classifiedPreClaimFailure
956
+ ? unclaimedAttemptWakeChanged(retryWakeBaseline, current)
957
+ : current.interruptionWakeups !== retryWakeBaseline.interruptionWakeups ||
958
+ current.wakeups !== retryWakeBaseline.wakeups;
959
+ }, retryDelayMs);
960
+ return true;
961
+ }
962
+ unclaimedAttemptFailures = 0;
963
+ return failure.action !== "failed" && failure.action !== "terminal";
774
964
  }
775
965
 
966
+ unclaimedAttemptFailures = 0;
776
967
  if (outcome.result.status === "unclaimed") {
777
968
  return true;
778
969
  }