@opengeni/worker-bundle 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +24 -1
- package/dist/activities/run-input.d.ts +8 -0
- package/dist/activities/session-state.d.ts +5 -3
- package/dist/activities/types.d.ts +27 -0
- package/dist/{activities-control-XURE4VKN.js → activities-control-SMKF2OBE.js} +54 -7
- package/dist/{activities-control-XURE4VKN.js.map → activities-control-SMKF2OBE.js.map} +1 -1
- package/dist/activities-control.d.ts +2 -2
- package/dist/{activities-turn-6EBXGGLS.js → activities-turn-IDS3WEHA.js} +1471 -975
- package/dist/activities-turn-IDS3WEHA.js.map +1 -0
- package/dist/activities.d.ts +3 -3
- package/dist/{chunk-TYNPCG4U.js → chunk-N5TJIFCE.js} +69 -1
- package/dist/chunk-N5TJIFCE.js.map +1 -0
- package/dist/{chunk-5GRJIP5H.js → chunk-P6Z2SLAA.js} +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -2
- package/dist/observability-metrics.d.ts +37 -0
- package/dist/workflow-bundle.js +174 -4
- package/dist/workflows/session.d.ts +17 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +1675 -1175
- package/src/activities/run-input.ts +137 -45
- package/src/activities/session-state.ts +69 -5
- package/src/activities/types.ts +27 -0
- package/src/activities.ts +2 -0
- package/src/observability-metrics.ts +184 -3
- package/src/workflows/session.ts +199 -8
- package/dist/activities-turn-6EBXGGLS.js.map +0 -1
- package/dist/chunk-TYNPCG4U.js.map +0 -1
- /package/dist/{chunk-5GRJIP5H.js.map → chunk-P6Z2SLAA.js.map} +0 -0
package/src/workflows/session.ts
CHANGED
|
@@ -16,7 +16,10 @@ import type * as activities from "../activities";
|
|
|
16
16
|
import {
|
|
17
17
|
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
18
18
|
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
19
|
+
PRE_CLAIM_FAILURE_MESSAGE,
|
|
20
|
+
PRE_CLAIM_FAILURE_TYPE,
|
|
19
21
|
type EscapedMcpTimeoutRecoveryDetail,
|
|
22
|
+
type PreClaimFailureDisposition,
|
|
20
23
|
} from "../activities/types";
|
|
21
24
|
import {
|
|
22
25
|
activity,
|
|
@@ -89,6 +92,37 @@ export function deferredResultMayContinue(entryWakeups: number, currentWakeups:
|
|
|
89
92
|
return currentWakeups !== entryWakeups;
|
|
90
93
|
}
|
|
91
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Bound repeated failures that happen before an attempt row exists. The
|
|
97
|
+
* activity mirrors this deterministic delay into the durable wake outbox, so
|
|
98
|
+
* a live workflow and a replacement workflow observe the same retry floor.
|
|
99
|
+
*/
|
|
100
|
+
export function unclaimedAttemptRetryDelayMs(consecutiveFailures: number): number {
|
|
101
|
+
if (!Number.isFinite(consecutiveFailures) || consecutiveFailures <= 0) return 1_000;
|
|
102
|
+
const exponent = Math.min(6, Math.max(0, Math.trunc(consecutiveFailures) - 1));
|
|
103
|
+
return Math.min(60_000, 1_000 * 2 ** exponent);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export type SessionWakeCounters = {
|
|
107
|
+
wakeups: number;
|
|
108
|
+
interruptionWakeups: number;
|
|
109
|
+
approvalWakeups: number;
|
|
110
|
+
capacityWakeups: number;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
/** Any signal that can make the failed admission runnable interrupts backoff. */
|
|
114
|
+
export function unclaimedAttemptWakeChanged(
|
|
115
|
+
baseline: SessionWakeCounters,
|
|
116
|
+
current: SessionWakeCounters,
|
|
117
|
+
): boolean {
|
|
118
|
+
return (
|
|
119
|
+
current.wakeups !== baseline.wakeups ||
|
|
120
|
+
current.interruptionWakeups !== baseline.interruptionWakeups ||
|
|
121
|
+
current.approvalWakeups !== baseline.approvalWakeups ||
|
|
122
|
+
current.capacityWakeups !== baseline.capacityWakeups
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
|
|
92
126
|
/** Deterministic Temporal timer delay for a persisted structured-input deadline. */
|
|
93
127
|
export function humanInputDeadlineWaitMs(expiresAt: string, nowMs = Date.now()): number {
|
|
94
128
|
const deadline = Date.parse(expiresAt);
|
|
@@ -165,6 +199,40 @@ export function escapedMcpTimeoutRecoveryDetail(
|
|
|
165
199
|
};
|
|
166
200
|
}
|
|
167
201
|
|
|
202
|
+
/** Read only the upgraded turn worker's explicit pre-claim wire contract. */
|
|
203
|
+
export function preClaimFailureDisposition(error: unknown): PreClaimFailureDisposition | undefined {
|
|
204
|
+
if (!(error instanceof ActivityFailure) || error.activityType !== "runAgentTurn") {
|
|
205
|
+
return undefined;
|
|
206
|
+
}
|
|
207
|
+
const cause = error.cause;
|
|
208
|
+
if (
|
|
209
|
+
!(cause instanceof ApplicationFailure) ||
|
|
210
|
+
cause.type !== PRE_CLAIM_FAILURE_TYPE ||
|
|
211
|
+
cause.message !== PRE_CLAIM_FAILURE_MESSAGE
|
|
212
|
+
) {
|
|
213
|
+
return undefined;
|
|
214
|
+
}
|
|
215
|
+
const detail = cause.details?.[0] as { disposition?: unknown; code?: unknown } | undefined;
|
|
216
|
+
if (
|
|
217
|
+
detail?.code !== "db_deadlock" &&
|
|
218
|
+
detail?.code !== "db_serialization_failure" &&
|
|
219
|
+
detail?.code !== "db_failure" &&
|
|
220
|
+
detail?.code !== "claim_invariant"
|
|
221
|
+
) {
|
|
222
|
+
return undefined;
|
|
223
|
+
}
|
|
224
|
+
const disposition = detail.disposition;
|
|
225
|
+
if (disposition !== "retryable" && disposition !== "permanent") return undefined;
|
|
226
|
+
if (
|
|
227
|
+
(detail.code === "db_deadlock" || detail.code === "db_serialization_failure") &&
|
|
228
|
+
disposition !== "retryable"
|
|
229
|
+
) {
|
|
230
|
+
return undefined;
|
|
231
|
+
}
|
|
232
|
+
if (detail.code === "claim_invariant" && disposition !== "permanent") return undefined;
|
|
233
|
+
return disposition;
|
|
234
|
+
}
|
|
235
|
+
|
|
168
236
|
/**
|
|
169
237
|
* Classify only the exact turn-fence cancellation protocol used to arbitrate a
|
|
170
238
|
* database control commit that reaches the activity just before its Temporal
|
|
@@ -216,6 +284,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
216
284
|
const receiptGatedCancellation = patched("session-attempt-quiescence-v2");
|
|
217
285
|
const writerSetQuiescenceRecovery = patched("session-attempt-writer-set-quiescence-v1");
|
|
218
286
|
const staleControlSignalIsOnlyWakeHint = patched("session-control-stale-wake-v1");
|
|
287
|
+
const unclaimedAttemptRecovery = patched("session-unclaimed-attempt-recovery-v1");
|
|
219
288
|
const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue, receiptGatedCancellation);
|
|
220
289
|
let approvalWakeups = 0;
|
|
221
290
|
let interruptionWakeups = 0;
|
|
@@ -230,6 +299,7 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
230
299
|
// weeks-long session survivable.
|
|
231
300
|
let turnsThisRun = 0;
|
|
232
301
|
let capacityChecksThisRun = 0;
|
|
302
|
+
let unclaimedAttemptFailures = 0;
|
|
233
303
|
|
|
234
304
|
setHandler(userMessage, () => {
|
|
235
305
|
signalVersion += 1;
|
|
@@ -579,6 +649,16 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
579
649
|
trigger: activities.RunAgentTurnInput["trigger"],
|
|
580
650
|
): Promise<boolean> {
|
|
581
651
|
const capacityWaitEntryBaseline = { wakeups, capacityWakeups };
|
|
652
|
+
// Capture every admission-relevant signal before activity dispatch. A
|
|
653
|
+
// signal may arrive while runAgentTurn is still failing before claim, or
|
|
654
|
+
// while the failure-control activity settles. Either must interrupt the
|
|
655
|
+
// bounded recovery timer instead of being erased by a later baseline.
|
|
656
|
+
const preDispatchRetryWakeBaseline: SessionWakeCounters = {
|
|
657
|
+
wakeups,
|
|
658
|
+
interruptionWakeups,
|
|
659
|
+
approvalWakeups,
|
|
660
|
+
capacityWakeups,
|
|
661
|
+
};
|
|
582
662
|
const attemptId = uuid4();
|
|
583
663
|
let interruptionBaseline = interruptionWakeups;
|
|
584
664
|
|
|
@@ -763,16 +843,127 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
763
843
|
// truth when the bounded redispatch ceiling was exceeded.
|
|
764
844
|
return false;
|
|
765
845
|
}
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
846
|
+
const postDispatchRetryWakeBaseline: SessionWakeCounters = {
|
|
847
|
+
wakeups,
|
|
848
|
+
interruptionWakeups,
|
|
849
|
+
approvalWakeups,
|
|
850
|
+
capacityWakeups,
|
|
851
|
+
};
|
|
852
|
+
if (!unclaimedAttemptRecovery) {
|
|
853
|
+
// Replay compatibility: histories recorded before v2 scheduled this
|
|
854
|
+
// exact argument shape and treated the activity as void. Preserve the
|
|
855
|
+
// completed-history tail during replay, but keep a still-open legacy
|
|
856
|
+
// history alive after its activity completes. The nested patch marker
|
|
857
|
+
// is recorded only when an old history reaches this previously absent
|
|
858
|
+
// tail on a live workflow task; already-completed histories replay the
|
|
859
|
+
// original close without emitting a new timer command.
|
|
860
|
+
const legacyFailure: activities.FailSessionAttemptResult | undefined =
|
|
861
|
+
await activity.failSessionAttempt({
|
|
862
|
+
accountId,
|
|
863
|
+
workspaceId,
|
|
864
|
+
sessionId,
|
|
865
|
+
attemptId,
|
|
866
|
+
error: workflowFailureMessage(outcome.error),
|
|
867
|
+
});
|
|
868
|
+
if (!patched("session-legacy-unclaimed-attempt-tail-v1")) {
|
|
869
|
+
return false;
|
|
870
|
+
}
|
|
871
|
+
if (legacyFailure?.action === "failed" || legacyFailure?.action === "terminal") {
|
|
872
|
+
return false;
|
|
873
|
+
}
|
|
874
|
+
const retryDelayMs = unclaimedAttemptRetryDelayMs(unclaimedAttemptFailures + 1);
|
|
875
|
+
// Histories that already recorded the legacy tail used a post-control
|
|
876
|
+
// two-signal baseline. Preserve that exact condition on replay; an old
|
|
877
|
+
// open history reaching this tail for the first time can adopt the
|
|
878
|
+
// pre-dispatch four-signal contract.
|
|
879
|
+
const upgradedLegacySignalBaseline = patched("session-legacy-preclaim-signal-baseline-v1");
|
|
880
|
+
const retryWakeBaseline = upgradedLegacySignalBaseline
|
|
881
|
+
? preDispatchRetryWakeBaseline
|
|
882
|
+
: {
|
|
883
|
+
wakeups,
|
|
884
|
+
interruptionWakeups,
|
|
885
|
+
approvalWakeups,
|
|
886
|
+
capacityWakeups,
|
|
887
|
+
};
|
|
888
|
+
unclaimedAttemptFailures += 1;
|
|
889
|
+
await condition(() => {
|
|
890
|
+
const current = {
|
|
891
|
+
wakeups,
|
|
892
|
+
interruptionWakeups,
|
|
893
|
+
approvalWakeups,
|
|
894
|
+
capacityWakeups,
|
|
895
|
+
};
|
|
896
|
+
return upgradedLegacySignalBaseline
|
|
897
|
+
? unclaimedAttemptWakeChanged(retryWakeBaseline, current)
|
|
898
|
+
: current.interruptionWakeups !== retryWakeBaseline.interruptionWakeups ||
|
|
899
|
+
current.wakeups !== retryWakeBaseline.wakeups;
|
|
900
|
+
}, retryDelayMs);
|
|
901
|
+
return true;
|
|
902
|
+
}
|
|
903
|
+
const retryDelayMs = unclaimedAttemptRetryDelayMs(unclaimedAttemptFailures + 1);
|
|
904
|
+
const admissionFailureDisposition = preClaimFailureDisposition(outcome.error);
|
|
905
|
+
// Keep this marker immediately adjacent to the changed command. A
|
|
906
|
+
// history that already recorded the v2 activity replays the old shape;
|
|
907
|
+
// an open history that has never reached this branch can record v3.
|
|
908
|
+
const classifiedPreClaimFailure = patched("session-preclaim-failure-classification-v1");
|
|
909
|
+
const retryWakeBaseline = classifiedPreClaimFailure
|
|
910
|
+
? preDispatchRetryWakeBaseline
|
|
911
|
+
: postDispatchRetryWakeBaseline;
|
|
912
|
+
const failure: activities.FailSessionAttemptResult | undefined =
|
|
913
|
+
await activity.failSessionAttempt(
|
|
914
|
+
classifiedPreClaimFailure
|
|
915
|
+
? {
|
|
916
|
+
accountId,
|
|
917
|
+
workspaceId,
|
|
918
|
+
sessionId,
|
|
919
|
+
attemptId,
|
|
920
|
+
workflowId: workflowInfo().workflowId,
|
|
921
|
+
retryDelayMs,
|
|
922
|
+
...(admissionFailureDisposition
|
|
923
|
+
? { preClaimFailureDisposition: admissionFailureDisposition }
|
|
924
|
+
: {}),
|
|
925
|
+
trigger,
|
|
926
|
+
error: workflowFailureMessage(outcome.error),
|
|
927
|
+
}
|
|
928
|
+
: {
|
|
929
|
+
// Replay the exact v2 command shape. Adding optional fields to
|
|
930
|
+
// a Temporal activity argument still changes command history.
|
|
931
|
+
accountId,
|
|
932
|
+
workspaceId,
|
|
933
|
+
sessionId,
|
|
934
|
+
attemptId,
|
|
935
|
+
workflowId: workflowInfo().workflowId,
|
|
936
|
+
retryDelayMs,
|
|
937
|
+
error: workflowFailureMessage(outcome.error),
|
|
938
|
+
},
|
|
939
|
+
);
|
|
940
|
+
// During a rolling deploy an upgraded workflow worker can schedule this
|
|
941
|
+
// activity on a legacy control worker whose wire result was void. Treat
|
|
942
|
+
// that unknown commit outcome like an unclaimed attempt: wait, then
|
|
943
|
+
// re-read durable state. A legacy worker that settled the failure leaves
|
|
944
|
+
// an idle turn; one that no-op'd before claim leaves recoverable work.
|
|
945
|
+
// Neither path replays model or tool side effects speculatively.
|
|
946
|
+
if (!failure || failure.action === "unclaimed") {
|
|
947
|
+
unclaimedAttemptFailures += 1;
|
|
948
|
+
await condition(() => {
|
|
949
|
+
const current = {
|
|
950
|
+
wakeups,
|
|
951
|
+
interruptionWakeups,
|
|
952
|
+
approvalWakeups,
|
|
953
|
+
capacityWakeups,
|
|
954
|
+
};
|
|
955
|
+
return classifiedPreClaimFailure
|
|
956
|
+
? unclaimedAttemptWakeChanged(retryWakeBaseline, current)
|
|
957
|
+
: current.interruptionWakeups !== retryWakeBaseline.interruptionWakeups ||
|
|
958
|
+
current.wakeups !== retryWakeBaseline.wakeups;
|
|
959
|
+
}, retryDelayMs);
|
|
960
|
+
return true;
|
|
961
|
+
}
|
|
962
|
+
unclaimedAttemptFailures = 0;
|
|
963
|
+
return failure.action !== "failed" && failure.action !== "terminal";
|
|
774
964
|
}
|
|
775
965
|
|
|
966
|
+
unclaimedAttemptFailures = 0;
|
|
776
967
|
if (outcome.result.status === "unclaimed") {
|
|
777
968
|
return true;
|
|
778
969
|
}
|