@opengeni/worker-bundle 0.22.4-canary.2 → 0.22.7-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/errors.d.ts +20 -0
- package/dist/activities/agent-turn/governance-model.d.ts +9 -0
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +14 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +8 -0
- package/dist/activities/generated-images.d.ts +3 -1
- package/dist/activities/image-generation-references.d.ts +39 -7
- package/dist/activities/sandbox-lease.d.ts +12 -0
- package/dist/activities/types.d.ts +24 -0
- package/dist/{activities-control-4XVBA3Q5.js → activities-control-VZIIV7RX.js} +77 -12
- package/dist/activities-control-VZIIV7RX.js.map +1 -0
- package/dist/{activities-turn-J3CR27E6.js → activities-turn-5RGQDBFV.js} +253 -51
- package/dist/activities-turn-5RGQDBFV.js.map +1 -0
- package/dist/{chunk-FA35XBBC.js → chunk-NAVSAZ56.js} +1 -1
- package/dist/chunk-NAVSAZ56.js.map +1 -0
- package/dist/{chunk-XWQPS6GG.js → chunk-O4HDX5SC.js} +59 -10
- package/dist/{chunk-XWQPS6GG.js.map → chunk-O4HDX5SC.js.map} +1 -1
- package/dist/index.js +2 -2
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/workflow-bundle.js +62 -9
- package/dist/workflows/session.d.ts +4 -1
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +29 -18
- package/src/activities/agent-turn/claim.ts +6 -6
- package/src/activities/agent-turn/errors.ts +104 -21
- package/src/activities/agent-turn/failure-settlement.ts +40 -0
- package/src/activities/agent-turn/governance-model.ts +31 -4
- package/src/activities/agent-turn/history.ts +7 -3
- package/src/activities/agent-turn/image-reference-sandbox.ts +35 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +13 -0
- package/src/activities/agent-turn/run.ts +7 -0
- package/src/activities/agent-turn/stream-attempt.ts +25 -2
- package/src/activities/generated-images.ts +5 -2
- package/src/activities/image-generation-references.ts +184 -37
- package/src/activities/sandbox-lease.ts +48 -9
- package/src/activities/session-state.ts +69 -1
- package/src/activities/types.ts +26 -0
- package/src/observability-metrics.ts +2 -0
- package/src/workflows/session.ts +95 -21
- package/dist/activities-control-4XVBA3Q5.js.map +0 -1
- package/dist/activities-turn-J3CR27E6.js.map +0 -1
- package/dist/chunk-FA35XBBC.js.map +0 -1
|
@@ -6,6 +6,11 @@ import { type MandatoryHistoryPersistenceStage } from "./quiescence.js";
|
|
|
6
6
|
export declare const PROVIDER_BACKPRESSURE_DELAY_MS = 60000;
|
|
7
7
|
export declare const PROVIDER_CONNECTIVITY_BACKOFF_MS: readonly [2000, 5000, 15000, 30000, 60000];
|
|
8
8
|
export declare const MAX_AUTOMATIC_PROVIDER_RECOVERIES: 5;
|
|
9
|
+
export declare const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
|
|
10
|
+
export declare class PostCompactionContinuationEmptyError extends Error {
|
|
11
|
+
readonly code = "post_compaction_continuation_empty";
|
|
12
|
+
constructor();
|
|
13
|
+
}
|
|
9
14
|
export type ProviderRecoveryResult = {
|
|
10
15
|
status: "recovering";
|
|
11
16
|
continueDelayMs: number;
|
|
@@ -67,6 +72,21 @@ export declare function escapedMcpTimeoutRecoveryFailure(input: {
|
|
|
67
72
|
* settlement.
|
|
68
73
|
*/
|
|
69
74
|
export declare function preClaimAdmissionFailure(error: unknown): ApplicationFailure;
|
|
75
|
+
/**
|
|
76
|
+
* Carry one exact claimed-but-not-started attempt into the workflow's DB-only
|
|
77
|
+
* recovery lane. Permanent database/state failures remain terminal; only the
|
|
78
|
+
* same operational outage classes that are safe before claim are admitted.
|
|
79
|
+
*/
|
|
80
|
+
export declare function postClaimDatabaseRecoveryFailure(input: {
|
|
81
|
+
error: unknown;
|
|
82
|
+
turnId: string;
|
|
83
|
+
triggerEventId: string;
|
|
84
|
+
executionGeneration: number;
|
|
85
|
+
providerRecovery?: {
|
|
86
|
+
failureCode: string;
|
|
87
|
+
providerRecoveryCount: number;
|
|
88
|
+
};
|
|
89
|
+
}): ApplicationFailure | null;
|
|
70
90
|
/**
|
|
71
91
|
* Resolve which Codex account a turn runs on (multi-account P1): session-pin >
|
|
72
92
|
* workspace-active. No rotation in P1. The selected id must still be in the
|
|
@@ -63,4 +63,13 @@ export type GovernanceModelOutcome = {
|
|
|
63
63
|
} | {
|
|
64
64
|
ok: GovernanceModelOk;
|
|
65
65
|
};
|
|
66
|
+
type PreferenceSnapshotTurn = Pick<ClaimTurnOk["turn"], "initiatingHumanSubjectId" | "initiator">;
|
|
67
|
+
export declare function preferenceSnapshotHumanSubjectId(turn: PreferenceSnapshotTurn): string | null;
|
|
68
|
+
/**
|
|
69
|
+
* Preferences are scoped partly to the frozen initiating human. A service-only
|
|
70
|
+
* turn has no such scope, so avoid calling the human-bound snapshot capability
|
|
71
|
+
* merely to catch its expected authority rejection.
|
|
72
|
+
*/
|
|
73
|
+
export declare function preferenceSnapshotForTurn<T>(turn: PreferenceSnapshotTurn, load: () => Promise<T>): Promise<T | null>;
|
|
66
74
|
export declare function prepareGovernanceAndModel(deps: GovernanceModelDeps): Promise<GovernanceModelOutcome>;
|
|
75
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { ChannelASession } from "@opengeni/runtime/sandbox";
|
|
2
|
+
import type { SandboxRuntimeState } from "./turn-context.js";
|
|
3
|
+
type ImageReferenceSandboxState = Pick<SandboxRuntimeState, "resolvedSandbox" | "lazyOwnedSandbox" | "turnSandboxProvisioner">;
|
|
4
|
+
/**
|
|
5
|
+
* Join the turn's canonical lazy sandbox boundary before reading a
|
|
6
|
+
* model-supplied /workspace image reference. Durable File and artifact
|
|
7
|
+
* references never call this helper because their bytes come from object
|
|
8
|
+
* storage instead.
|
|
9
|
+
*/
|
|
10
|
+
export declare function resolveImageReferenceSandboxSession(sandboxState: ImageReferenceSandboxState, sdkOwnedSandboxSession: unknown): Promise<{
|
|
11
|
+
session: ChannelASession;
|
|
12
|
+
leaseEpoch: number;
|
|
13
|
+
}>;
|
|
14
|
+
export {};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { TurnHistorySink } from "./history-sink.js";
|
|
2
|
+
/**
|
|
3
|
+
* Persist the SDK's complete prior model/tool history before another provider
|
|
4
|
+
* request can start. The first request has no prior model/tool rows to append.
|
|
5
|
+
* Follow-up requests run only after the preceding complete call/result batch is
|
|
6
|
+
* replay-safe.
|
|
7
|
+
*/
|
|
8
|
+
export declare function checkpointHistoryBeforeProviderDispatch(historySink: Pick<TurnHistorySink, "reconcileConversationTruth">): Promise<void>;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type GeneratedImageReceipt } from "@opengeni/contracts";
|
|
2
2
|
import { type Database, type GeneratedImageArtifact, type GeneratedImageSourceStrategy } from "@opengeni/db";
|
|
3
3
|
import { type ObjectStorage } from "@opengeni/storage";
|
|
4
|
+
import { ScreenshotValidationError } from "./retained-screenshots.js";
|
|
4
5
|
export type GeneratedImageMediaType = "image/png" | "image/jpeg" | "image/webp";
|
|
5
6
|
export type ValidatedGeneratedImage = {
|
|
6
7
|
bytes: Uint8Array;
|
|
@@ -23,7 +24,8 @@ export type RetainedGeneratedImage = {
|
|
|
23
24
|
artifact: GeneratedImageArtifact;
|
|
24
25
|
};
|
|
25
26
|
export declare class GeneratedImageValidationError extends Error {
|
|
26
|
-
|
|
27
|
+
readonly reason?: ScreenshotValidationError["reason"] | undefined;
|
|
28
|
+
constructor(message: string, reason?: ScreenshotValidationError["reason"] | undefined);
|
|
27
29
|
}
|
|
28
30
|
/** Extract the completed Responses hosted-tool payload before JSON serialization. */
|
|
29
31
|
export declare function generatedImageFromSdkEvent(event: unknown): GeneratedImageOutput | null;
|
|
@@ -11,12 +11,24 @@ export type ResolvedImageGenerationReference = Readonly<{
|
|
|
11
11
|
sha256: string;
|
|
12
12
|
}>;
|
|
13
13
|
export type SandboxImageReferenceReader = (path: string, maxBytes: number) => Promise<Uint8Array>;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
14
|
+
export type ImageGenerationReferenceErrorCode = "invalid_reference" | "unsupported_reference_media" | "reference_too_large" | "reference_not_ready" | "reference_unavailable" | "reference_integrity_mismatch";
|
|
15
|
+
/** A deterministic input rejection that occurs before any image provider request. */
|
|
16
|
+
export declare class ImageGenerationReferenceError extends Error {
|
|
17
|
+
readonly code: ImageGenerationReferenceErrorCode;
|
|
18
|
+
constructor(code: ImageGenerationReferenceErrorCode, message: string, options?: ErrorOptions);
|
|
19
|
+
}
|
|
20
|
+
export type ImageGenerationReferenceRejectedResult = Readonly<{
|
|
21
|
+
isError: true;
|
|
22
|
+
status: "rejected";
|
|
23
|
+
code: ImageGenerationReferenceErrorCode;
|
|
24
|
+
message: string;
|
|
25
|
+
operationCreated: false;
|
|
26
|
+
content: readonly [{
|
|
27
|
+
readonly type: "text";
|
|
28
|
+
readonly text: string;
|
|
29
|
+
}];
|
|
30
|
+
}>;
|
|
31
|
+
export type ResolveImageGenerationReferencesInput = Readonly<{
|
|
20
32
|
db: Database;
|
|
21
33
|
objectStorage: ObjectStorage;
|
|
22
34
|
accountId: string;
|
|
@@ -24,4 +36,24 @@ export declare function resolveImageGenerationReferences(input: {
|
|
|
24
36
|
subjectId: string | null;
|
|
25
37
|
references: readonly ImageGenerationReference[];
|
|
26
38
|
readSandboxFile?: SandboxImageReferenceReader;
|
|
27
|
-
}
|
|
39
|
+
}>;
|
|
40
|
+
export type ImageGenerationReferenceResolution = Readonly<{
|
|
41
|
+
status: "resolved";
|
|
42
|
+
references: ResolvedImageGenerationReference[];
|
|
43
|
+
}> | Readonly<{
|
|
44
|
+
status: "rejected";
|
|
45
|
+
result: ImageGenerationReferenceRejectedResult;
|
|
46
|
+
}>;
|
|
47
|
+
/**
|
|
48
|
+
* Carry deterministic pre-provider input failures on the tool output. Unknown
|
|
49
|
+
* database and object-storage failures still throw, and provider execution
|
|
50
|
+
* remains outside this boundary so an outcome-unknown operation can never be
|
|
51
|
+
* reported rejected.
|
|
52
|
+
*/
|
|
53
|
+
export declare function resolveImageGenerationReferencesForTool(input: ResolveImageGenerationReferencesInput): Promise<ImageGenerationReferenceResolution>;
|
|
54
|
+
/**
|
|
55
|
+
* Resolve ordered model-facing references without accepting arbitrary URLs.
|
|
56
|
+
* Stored files remain workspace-RLS scoped; sandbox reads stay rooted beneath
|
|
57
|
+
* /workspace. Provider adapters receive only validated immutable bytes.
|
|
58
|
+
*/
|
|
59
|
+
export declare function resolveImageGenerationReferences(input: ResolveImageGenerationReferencesInput): Promise<ResolvedImageGenerationReference[]>;
|
|
@@ -4,6 +4,7 @@ import { type ConnectedMachineBackgroundCommandClaim, type ConnectedMachineBackg
|
|
|
4
4
|
import { createSandboxClientForBackend, inspectModalSandboxLifecycle, resolveModalCheckpointProviderBindingForSession, terminateModalSandboxById, type ModalOrphanSweepTermination, type ModalCheckpointProviderBinding, type ProviderWorkspaceCapturePolicy, type WorkspaceArchiveDescriptor, type ControlRpc } from "@opengeni/runtime/sandbox";
|
|
5
5
|
import { type SandboxDrainActivityInput, type SandboxLeaseSweepMaintenanceInput } from "../sandbox-reaper-contract.js";
|
|
6
6
|
import type { ControlActivityServices as ActivityServices } from "./types.js";
|
|
7
|
+
import { type RetainedProcessReconciliationOutcome } from "../observability-metrics.js";
|
|
7
8
|
import { type OpenSandboxKubernetesInventory } from "../opensandbox-kubernetes-inventory.js";
|
|
8
9
|
export { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
9
10
|
export type ReapSandboxLeasesResult = {
|
|
@@ -99,6 +100,8 @@ export type DrainableProviderProbeFn = (settings: ActivityServices["settings"],
|
|
|
99
100
|
export declare const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
100
101
|
export declare const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS: number;
|
|
101
102
|
export declare const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5000;
|
|
103
|
+
export declare const RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
|
|
104
|
+
export declare const RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS: number;
|
|
102
105
|
export declare const CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
|
|
103
106
|
export declare const CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS: number;
|
|
104
107
|
export declare class SandboxProviderCaptureTimeoutError extends Error {
|
|
@@ -140,6 +143,15 @@ export declare function createSandboxLeaseActivities(services: () => Promise<Act
|
|
|
140
143
|
};
|
|
141
144
|
export declare function probeConnectedMachineBackgroundCommand(claim: ConnectedMachineBackgroundCommandClaim, controlRpc: ControlRpc, controlTimeoutMs: number, observedAt?: Date): Promise<ConnectedMachineBackgroundCommandProof | null>;
|
|
142
145
|
export declare function connectedCommandProofFromStatus(status: OpStatus, observedAt: Date): ConnectedMachineBackgroundCommandProof | null;
|
|
146
|
+
type RetainedProcessDeferralOutcome = Extract<RetainedProcessProbeResult, {
|
|
147
|
+
status: "deferred";
|
|
148
|
+
}>["reason"] | "settlement_failed";
|
|
149
|
+
export declare function retainedProcessReconciliationDeferral(settings: Pick<ActivityServices["settings"], "sandboxLeaseReaperPeriodMs">, process: Pick<SandboxRetainedProcess, "reconcileAttempts">, outcome: RetainedProcessDeferralOutcome): {
|
|
150
|
+
durableOutcome: string;
|
|
151
|
+
metricOutcome: RetainedProcessReconciliationOutcome;
|
|
152
|
+
retryAfterMs: number;
|
|
153
|
+
quarantined: boolean;
|
|
154
|
+
};
|
|
143
155
|
export declare function probeRetainedProcessAtProvider(settings: ActivityServices["settings"], lease: LeaseSnapshot, process: SandboxRetainedProcess, mode?: "observe" | "cancel"): Promise<RetainedProcessProbeResult>;
|
|
144
156
|
/** Classify only exact SDK control banners. Arbitrary output, malformed text,
|
|
145
157
|
* and a running marker are observations to retry; none is physical exit proof. */
|
|
@@ -180,6 +180,14 @@ export type FailSessionAttemptInput = {
|
|
|
180
180
|
* admission transaction; omission is a rolling-deploy/legacy unknown and
|
|
181
181
|
* deliberately keeps the recoverable wake behavior. */
|
|
182
182
|
preClaimFailureDisposition?: PreClaimFailureDisposition;
|
|
183
|
+
/** Added in v4. Retains the safe classified DB code so an ambiguously
|
|
184
|
+
* committed claim discovered by the control lane can recover the exact
|
|
185
|
+
* attempt instead of terminally failing it. */
|
|
186
|
+
preClaimFailure?: PreClaimFailureDetail;
|
|
187
|
+
/** Added in v4. A claimed attempt that lost operational database access
|
|
188
|
+
* before turn-start completion carries its exact immutable turn identity so
|
|
189
|
+
* the DB-only control lane can recover it instead of terminally failing it. */
|
|
190
|
+
postClaimDatabaseRecovery?: PostClaimDatabaseRecoveryDetail;
|
|
183
191
|
/** The workflow admission trigger is required to re-evaluate the same
|
|
184
192
|
* durable admission branch before terminally settling a permanent failure. */
|
|
185
193
|
trigger?: RunAgentTurnInput["trigger"];
|
|
@@ -187,6 +195,8 @@ export type FailSessionAttemptInput = {
|
|
|
187
195
|
};
|
|
188
196
|
export type FailSessionAttemptResult = {
|
|
189
197
|
action: "failed";
|
|
198
|
+
} | {
|
|
199
|
+
action: "recovering";
|
|
190
200
|
} | {
|
|
191
201
|
action: "unclaimed";
|
|
192
202
|
} | {
|
|
@@ -201,6 +211,20 @@ export type PreClaimFailureDetail = {
|
|
|
201
211
|
};
|
|
202
212
|
export declare const PRE_CLAIM_FAILURE_TYPE = "OpenGeniPreClaimFailure";
|
|
203
213
|
export declare const PRE_CLAIM_FAILURE_MESSAGE = "Agent turn admission failed before attempt claim.";
|
|
214
|
+
export type PostClaimDatabaseRecoveryDetail = {
|
|
215
|
+
turnId: string;
|
|
216
|
+
triggerEventId: string;
|
|
217
|
+
executionGeneration: number;
|
|
218
|
+
code: "db_deadlock" | "db_serialization_failure" | "db_failure";
|
|
219
|
+
/** Present when the database outage interrupted a retryable provider
|
|
220
|
+
* recovery checkpoint. The control lane must advance this exact durable
|
|
221
|
+
* count instead of reopening the same recovery generation. */
|
|
222
|
+
providerRecoveryCount?: number;
|
|
223
|
+
/** Safe classified provider cause paired with providerRecoveryCount. */
|
|
224
|
+
providerFailureCode?: string;
|
|
225
|
+
};
|
|
226
|
+
export declare const POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE = "OpenGeniPostClaimDatabaseRecovery";
|
|
227
|
+
export declare const POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE = "Agent turn database recovery required after attempt claim.";
|
|
204
228
|
export type RecoverDispatchInput = {
|
|
205
229
|
accountId: string;
|
|
206
230
|
workspaceId: string;
|
|
@@ -32,7 +32,7 @@ import {
|
|
|
32
32
|
scheduledUserMessagePayload,
|
|
33
33
|
withFirstPartyTools,
|
|
34
34
|
workflowIdForSession
|
|
35
|
-
} from "./chunk-
|
|
35
|
+
} from "./chunk-O4HDX5SC.js";
|
|
36
36
|
import {
|
|
37
37
|
inspectOpenSandboxKubernetesInventory,
|
|
38
38
|
recordCreditBalanceGauges,
|
|
@@ -51,7 +51,7 @@ import {
|
|
|
51
51
|
recordSandboxRotationBacklogGauges,
|
|
52
52
|
recordTurnsQueuedGauge,
|
|
53
53
|
runtimeMetricsHooksForObservability
|
|
54
|
-
} from "./chunk-
|
|
54
|
+
} from "./chunk-NAVSAZ56.js";
|
|
55
55
|
|
|
56
56
|
// src/activities/browser-state-artifact-reaper.ts
|
|
57
57
|
import {
|
|
@@ -5224,6 +5224,8 @@ import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni
|
|
|
5224
5224
|
var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
5225
5225
|
var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
|
|
5226
5226
|
var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
|
|
5227
|
+
var RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
|
|
5228
|
+
var RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 6e4;
|
|
5227
5229
|
var CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
|
|
5228
5230
|
var CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
|
|
5229
5231
|
var SandboxProviderCaptureTimeoutError = class extends Error {
|
|
@@ -6121,11 +6123,7 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
6121
6123
|
}
|
|
6122
6124
|
}
|
|
6123
6125
|
async function deferRetainedProcessClaim(db, settings, observability, process, expected, claimId, outcome) {
|
|
6124
|
-
const
|
|
6125
|
-
const retryAfterMs = Math.min(
|
|
6126
|
-
5 * 6e4,
|
|
6127
|
-
Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
|
|
6128
|
-
);
|
|
6126
|
+
const deferral = retainedProcessReconciliationDeferral(settings, process, outcome);
|
|
6129
6127
|
try {
|
|
6130
6128
|
await deferRetainedProcessReconciliation(db, {
|
|
6131
6129
|
accountId: process.accountId,
|
|
@@ -6134,10 +6132,10 @@ async function deferRetainedProcessClaim(db, settings, observability, process, e
|
|
|
6134
6132
|
processId: process.id,
|
|
6135
6133
|
expected,
|
|
6136
6134
|
claimId,
|
|
6137
|
-
outcome,
|
|
6138
|
-
retryAfterMs
|
|
6135
|
+
outcome: deferral.durableOutcome,
|
|
6136
|
+
retryAfterMs: deferral.retryAfterMs
|
|
6139
6137
|
});
|
|
6140
|
-
recordRetainedProcessReconciliation(observability,
|
|
6138
|
+
recordRetainedProcessReconciliation(observability, deferral.metricOutcome);
|
|
6141
6139
|
} catch (error) {
|
|
6142
6140
|
recordRetainedProcessReconciliation(observability, "defer_failed");
|
|
6143
6141
|
observability.warn("sandbox reaper: retained-process defer failed", {
|
|
@@ -6146,6 +6144,28 @@ async function deferRetainedProcessClaim(db, settings, observability, process, e
|
|
|
6146
6144
|
});
|
|
6147
6145
|
}
|
|
6148
6146
|
}
|
|
6147
|
+
function retainedProcessReconciliationDeferral(settings, process, outcome) {
|
|
6148
|
+
const bindingQuarantine = process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS && (outcome === "provider_binding_missing" || outcome === "provider_binding_mismatch");
|
|
6149
|
+
if (bindingQuarantine) {
|
|
6150
|
+
return {
|
|
6151
|
+
durableOutcome: `quarantined_${outcome}`,
|
|
6152
|
+
metricOutcome: outcome === "provider_binding_missing" ? "quarantined_binding_missing" : "quarantined_binding_mismatch",
|
|
6153
|
+
retryAfterMs: RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS,
|
|
6154
|
+
quarantined: true
|
|
6155
|
+
};
|
|
6156
|
+
}
|
|
6157
|
+
const exponent = Math.min(4, Math.max(0, process.reconcileAttempts - 1));
|
|
6158
|
+
const retryAfterMs = Math.min(
|
|
6159
|
+
5 * 6e4,
|
|
6160
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
|
|
6161
|
+
);
|
|
6162
|
+
return {
|
|
6163
|
+
durableOutcome: outcome,
|
|
6164
|
+
metricOutcome: outcome,
|
|
6165
|
+
retryAfterMs,
|
|
6166
|
+
quarantined: false
|
|
6167
|
+
};
|
|
6168
|
+
}
|
|
6149
6169
|
var RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
|
|
6150
6170
|
async function withRetainedProcessProbeTimeout(promise) {
|
|
6151
6171
|
let timer;
|
|
@@ -9404,7 +9424,8 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9404
9424
|
temporalWorkflowId: workflowId
|
|
9405
9425
|
});
|
|
9406
9426
|
if (attempt) return { action: "stale" };
|
|
9407
|
-
|
|
9427
|
+
const preClaimFailureDisposition = input.preClaimFailure?.disposition ?? input.preClaimFailureDisposition;
|
|
9428
|
+
if (preClaimFailureDisposition === "permanent" && input.trigger) {
|
|
9408
9429
|
const failed = await failSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
|
|
9409
9430
|
accountId: input.accountId,
|
|
9410
9431
|
sessionId: input.sessionId,
|
|
@@ -9437,6 +9458,50 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9437
9458
|
});
|
|
9438
9459
|
return { action: "unclaimed" };
|
|
9439
9460
|
}
|
|
9461
|
+
const postClaimRecovery = input.postClaimDatabaseRecovery;
|
|
9462
|
+
const postClaimIdentityMatches = Boolean(
|
|
9463
|
+
postClaimRecovery && turn.id === postClaimRecovery.turnId && turn.triggerEventId === postClaimRecovery.triggerEventId && turn.executionGeneration === postClaimRecovery.executionGeneration
|
|
9464
|
+
);
|
|
9465
|
+
const providerRecoveryCount = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryCount : void 0;
|
|
9466
|
+
const providerFailureCode = postClaimIdentityMatches ? postClaimRecovery?.providerFailureCode : void 0;
|
|
9467
|
+
const hasProviderRecoveryCount = providerRecoveryCount !== void 0;
|
|
9468
|
+
const hasProviderFailureCode = providerFailureCode !== void 0;
|
|
9469
|
+
if (hasProviderRecoveryCount !== hasProviderFailureCode) {
|
|
9470
|
+
return { action: "stale" };
|
|
9471
|
+
}
|
|
9472
|
+
if (providerRecoveryCount !== void 0 && (!Number.isSafeInteger(providerRecoveryCount) || providerRecoveryCount <= 0 || providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 || providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || typeof providerFailureCode !== "string" || !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))) {
|
|
9473
|
+
return { action: "stale" };
|
|
9474
|
+
}
|
|
9475
|
+
const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : null;
|
|
9476
|
+
if (recoveredClaimCode) {
|
|
9477
|
+
const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
|
|
9478
|
+
sessionId: input.sessionId,
|
|
9479
|
+
turnId: turn.id,
|
|
9480
|
+
triggerEventId: turn.triggerEventId,
|
|
9481
|
+
attemptId: input.attemptId,
|
|
9482
|
+
reason: providerFailureCode ?? "claimed_attempt_database_failure",
|
|
9483
|
+
...providerRecoveryCount !== void 0 ? { providerRecoveryCount } : {},
|
|
9484
|
+
detail: {
|
|
9485
|
+
code: providerFailureCode ?? recoveredClaimCode,
|
|
9486
|
+
retryable: true,
|
|
9487
|
+
...providerRecoveryCount !== void 0 ? {
|
|
9488
|
+
databaseFailureCode: recoveredClaimCode,
|
|
9489
|
+
providerRecoveryCount
|
|
9490
|
+
} : {},
|
|
9491
|
+
recoverySource: "workflow_activity_failure"
|
|
9492
|
+
},
|
|
9493
|
+
fromStatuses: ["running"]
|
|
9494
|
+
});
|
|
9495
|
+
if (recovery.action !== "recovering") return { action: "stale" };
|
|
9496
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
9497
|
+
await refreshQueuedTurnsGauge(
|
|
9498
|
+
db,
|
|
9499
|
+
observability,
|
|
9500
|
+
countQueuedTurnsFn,
|
|
9501
|
+
recordTurnsQueuedGaugeFn
|
|
9502
|
+
);
|
|
9503
|
+
return { action: "recovering" };
|
|
9504
|
+
}
|
|
9440
9505
|
const trigger = await getSessionEventFn(db, input.workspaceId, turn.triggerEventId);
|
|
9441
9506
|
const result = await applySessionTurnSettlementFn(db, input.workspaceId, {
|
|
9442
9507
|
sessionId: input.sessionId,
|
|
@@ -9760,4 +9825,4 @@ export {
|
|
|
9760
9825
|
createControlActivities,
|
|
9761
9826
|
createControlActivitiesFromServices
|
|
9762
9827
|
};
|
|
9763
|
-
//# sourceMappingURL=activities-control-
|
|
9828
|
+
//# sourceMappingURL=activities-control-VZIIV7RX.js.map
|