@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +17 -4
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync.d.ts +4 -0
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +9 -10
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12701
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +147 -49
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync.ts +178 -37
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +102 -142
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-BJSDRSIP.js.map +0 -1
- package/dist/activities-turn-J5YO7XXX.js +0 -17243
- package/dist/activities-turn-J5YO7XXX.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -0,0 +1,754 @@
|
|
|
1
|
+
import {
|
|
2
|
+
commitSessionAttemptQuiescence,
|
|
3
|
+
recordCodexAccountUsageWithWakeTargets,
|
|
4
|
+
releaseCodexCredentialLease,
|
|
5
|
+
releaseXaiCredentialLease,
|
|
6
|
+
updateXaiQuotaMetadata,
|
|
7
|
+
type SessionAttemptQuiescenceCommit,
|
|
8
|
+
} from "@opengeni/db";
|
|
9
|
+
import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
|
|
10
|
+
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
11
|
+
import { clearRunCredentialsForAttempt } from "@opengeni/runtime";
|
|
12
|
+
import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
|
|
13
|
+
import type { Settings } from "@opengeni/config";
|
|
14
|
+
import { signalCodexCapacityWakeTargets } from "../codex-capacity";
|
|
15
|
+
import type { currentActivityContext } from "../streaming";
|
|
16
|
+
import type { CodemodeTokenRenewalController } from "../codemode-token-renewal";
|
|
17
|
+
import type { RunCredentialRenewalController } from "../run-credential-renewal";
|
|
18
|
+
import type {
|
|
19
|
+
RunAgentTurnInput,
|
|
20
|
+
SessionAttemptQuiescenceProof,
|
|
21
|
+
TurnActivityServices as ActivityServices,
|
|
22
|
+
} from "../types";
|
|
23
|
+
import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "../workspace-capture";
|
|
24
|
+
import { type ChannelASession } from "@opengeni/runtime/sandbox";
|
|
25
|
+
import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
|
|
26
|
+
import { makeMachineOpObserver, turnLifecycleMetricsFor } from "../../observability-metrics";
|
|
27
|
+
import {
|
|
28
|
+
maybePersistWarmWorkspaceSnapshot,
|
|
29
|
+
waitForWarmSnapshot,
|
|
30
|
+
type ResumedTurnSandbox,
|
|
31
|
+
} from "../../sandbox-resume";
|
|
32
|
+
import { createTurnCredentialLeases } from "./credential-leases";
|
|
33
|
+
import { safeErrorDiagnostic, safeErrorForTelemetry } from "./errors";
|
|
34
|
+
import {
|
|
35
|
+
assertPhysicalToolQuiescenceForCancellation,
|
|
36
|
+
assertSessionAttemptQuiescenceRecoveryDurable,
|
|
37
|
+
clearAttemptCredentialsWithSettledFence,
|
|
38
|
+
drainAttemptOwnedSandboxWriters,
|
|
39
|
+
persistOrSignalSessionAttemptQuiescence,
|
|
40
|
+
releaseTurnSandboxAfterWriterDrain,
|
|
41
|
+
shouldRunTurnEndWorkspacePersistence,
|
|
42
|
+
turnFinalizerCancellationSignal,
|
|
43
|
+
waitForTurnFinalizerStep,
|
|
44
|
+
} from "./quiescence";
|
|
45
|
+
import type {
|
|
46
|
+
AttemptIdentityState,
|
|
47
|
+
EventingState,
|
|
48
|
+
ProviderTurnState,
|
|
49
|
+
RecordingState,
|
|
50
|
+
RenewalState,
|
|
51
|
+
SandboxRuntimeState,
|
|
52
|
+
TurnControlState,
|
|
53
|
+
WorkspaceRefState,
|
|
54
|
+
} from "./turn-context";
|
|
55
|
+
|
|
56
|
+
export type TurnFinalizationDeps = {
|
|
57
|
+
input: RunAgentTurnInput;
|
|
58
|
+
settings: Settings;
|
|
59
|
+
db: ActivityServices["db"];
|
|
60
|
+
bus: ActivityServices["bus"];
|
|
61
|
+
objectStorage: ActivityServices["objectStorage"];
|
|
62
|
+
observability: ActivityServices["observability"];
|
|
63
|
+
wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
|
|
64
|
+
signalSessionAttemptQuiesced: ActivityServices["signalSessionAttemptQuiesced"];
|
|
65
|
+
signalCodexCapacityWorkflow: ActivityServices["signalCodexCapacityWorkflow"];
|
|
66
|
+
cancellationSignal: AbortSignal | undefined;
|
|
67
|
+
sandboxResumeController: AbortController;
|
|
68
|
+
activityContext: ReturnType<typeof currentActivityContext>;
|
|
69
|
+
activityStarted: number;
|
|
70
|
+
activitySpan: ReturnType<ActivityServices["observability"]["startSpan"]>;
|
|
71
|
+
dispatchId: string;
|
|
72
|
+
noteCancellationRequested: () => void;
|
|
73
|
+
machineOpObserver: ReturnType<typeof makeMachineOpObserver>;
|
|
74
|
+
leases: ReturnType<typeof createTurnCredentialLeases>;
|
|
75
|
+
control: TurnControlState;
|
|
76
|
+
attempt: AttemptIdentityState;
|
|
77
|
+
sandboxState: SandboxRuntimeState;
|
|
78
|
+
renewals: RenewalState;
|
|
79
|
+
recordingState: RecordingState;
|
|
80
|
+
eventing: EventingState;
|
|
81
|
+
providerTurn: ProviderTurnState;
|
|
82
|
+
workspaceRefs: WorkspaceRefState;
|
|
83
|
+
runWorkspaceMutationForSandbox: <T>(
|
|
84
|
+
sandbox: ResumedTurnSandbox,
|
|
85
|
+
operation: string,
|
|
86
|
+
mutation: () => Promise<T>,
|
|
87
|
+
) => Promise<T>;
|
|
88
|
+
requireResolvedSandboxForMutation: (message: string) => ResumedTurnSandbox;
|
|
89
|
+
stopLeaseHeartbeat: () => void;
|
|
90
|
+
abandonActiveRecording: (reason: string, disposition?: "failed" | "discard") => Promise<void>;
|
|
91
|
+
turnCompletionMemoryCollector: ReturnType<typeof createModelCheckpointMemoryCollector>;
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<void> {
|
|
95
|
+
const {
|
|
96
|
+
input,
|
|
97
|
+
settings,
|
|
98
|
+
db,
|
|
99
|
+
bus,
|
|
100
|
+
objectStorage,
|
|
101
|
+
observability,
|
|
102
|
+
wakeSessionWorkflow,
|
|
103
|
+
signalSessionAttemptQuiesced,
|
|
104
|
+
signalCodexCapacityWorkflow,
|
|
105
|
+
cancellationSignal,
|
|
106
|
+
sandboxResumeController,
|
|
107
|
+
activityContext,
|
|
108
|
+
activityStarted,
|
|
109
|
+
activitySpan,
|
|
110
|
+
dispatchId,
|
|
111
|
+
noteCancellationRequested,
|
|
112
|
+
machineOpObserver,
|
|
113
|
+
leases,
|
|
114
|
+
control,
|
|
115
|
+
attempt,
|
|
116
|
+
sandboxState,
|
|
117
|
+
renewals,
|
|
118
|
+
recordingState,
|
|
119
|
+
eventing,
|
|
120
|
+
providerTurn,
|
|
121
|
+
workspaceRefs,
|
|
122
|
+
runWorkspaceMutationForSandbox,
|
|
123
|
+
requireResolvedSandboxForMutation,
|
|
124
|
+
stopLeaseHeartbeat,
|
|
125
|
+
abandonActiveRecording,
|
|
126
|
+
turnCompletionMemoryCollector,
|
|
127
|
+
} = deps;
|
|
128
|
+
// This is the logical ownership boundary. Abort before any fallible
|
|
129
|
+
// housekeeping so a still-pending provider establish releases its private
|
|
130
|
+
// holder/timer even when Temporal itself never delivered cancellation.
|
|
131
|
+
if (sandboxState.resolvedSandbox === null && !sandboxResumeController.signal.aborted) {
|
|
132
|
+
sandboxResumeController.abort(
|
|
133
|
+
cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
const finalizationStarted = performance.now();
|
|
137
|
+
let finalizationError: unknown;
|
|
138
|
+
let physicalToolQuiescenceConfirmed = !control.acknowledgeQuiescence;
|
|
139
|
+
let quiescenceReceiptOrProofDurable = !control.acknowledgeQuiescence;
|
|
140
|
+
const finalizerSignal = turnFinalizerCancellationSignal(
|
|
141
|
+
cancellationSignal,
|
|
142
|
+
control.activityStatus,
|
|
143
|
+
);
|
|
144
|
+
try {
|
|
145
|
+
const toolCancellationFence = eventing.toolCancellationFenceRef.current;
|
|
146
|
+
// Every renewal controller is an attempt-owned sandbox writer. Capture
|
|
147
|
+
// and close all of them before the tool/run-writer drain and before the
|
|
148
|
+
// quiescence receipt; none may start another admitted write afterward.
|
|
149
|
+
renewals.gitCredentialRenewalClosed = true;
|
|
150
|
+
const gitRenewalsToStop = renewals.gitCredentialRenewals;
|
|
151
|
+
renewals.gitCredentialRenewals = [];
|
|
152
|
+
renewals.codemodeTokenRenewalClosed = true;
|
|
153
|
+
const codemodeRenewalToStop =
|
|
154
|
+
renewals.codemodeTokenRenewal as CodemodeTokenRenewalController | null;
|
|
155
|
+
renewals.codemodeTokenRenewal = null;
|
|
156
|
+
renewals.runCredentialRenewalClosed = true;
|
|
157
|
+
const runRenewalToStop = renewals.runCredentialRenewal as RunCredentialRenewalController | null;
|
|
158
|
+
renewals.runCredentialRenewal = null;
|
|
159
|
+
|
|
160
|
+
// Capture the attempt-qualified credential target before draining the
|
|
161
|
+
// controllers that can still write a newer generation.
|
|
162
|
+
const credentialSessionToClear = renewals.runCredentialSession;
|
|
163
|
+
renewals.runCredentialSession = null;
|
|
164
|
+
await drainAttemptOwnedSandboxWriters({
|
|
165
|
+
// Normal turn completion owns the same process boundary as
|
|
166
|
+
// Pause/Steer: yielded provider shells must be terminated, polled,
|
|
167
|
+
// and durably settled before workspace capture. Only receipt
|
|
168
|
+
// publication remains conditional on acknowledgeQuiescence.
|
|
169
|
+
toolCancellationFence,
|
|
170
|
+
cancellationReason: cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
|
|
171
|
+
gitCredentialRenewals: gitRenewalsToStop,
|
|
172
|
+
codemodeTokenRenewal: codemodeRenewalToStop,
|
|
173
|
+
runCredentialRenewal: runRenewalToStop,
|
|
174
|
+
});
|
|
175
|
+
// Attempt-qualified credential deletion is also a real workspace write,
|
|
176
|
+
// but it must not be submitted behind the very command being cancelled.
|
|
177
|
+
// Drain tool processes and credential renewals first, then delete only this
|
|
178
|
+
// attempt's generation before publishing physical quiescence. A failure
|
|
179
|
+
// deliberately keeps the receipt closed.
|
|
180
|
+
if (credentialSessionToClear) {
|
|
181
|
+
const clearAttemptCredentials = async (): Promise<void> =>
|
|
182
|
+
await clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
183
|
+
sessionId: input.sessionId,
|
|
184
|
+
attemptId: input.attemptId,
|
|
185
|
+
executionGeneration: attempt.executionGeneration,
|
|
186
|
+
});
|
|
187
|
+
await clearAttemptCredentialsWithSettledFence({
|
|
188
|
+
activityStatus: control.activityStatus,
|
|
189
|
+
runWorkspaceFencedClear: async () =>
|
|
190
|
+
await runWorkspaceMutationForSandbox(
|
|
191
|
+
requireResolvedSandboxForMutation(
|
|
192
|
+
"Run credential cleanup has no exact sandbox lease target",
|
|
193
|
+
),
|
|
194
|
+
"runCredentialAttemptClear",
|
|
195
|
+
clearAttemptCredentials,
|
|
196
|
+
),
|
|
197
|
+
clearExactAttempt: clearAttemptCredentials,
|
|
198
|
+
onSettledAttemptFence: () => {
|
|
199
|
+
// Terminal settlement closes the active attempt before this finally
|
|
200
|
+
// runs, so the ordinary workspace admission correctly rejects it.
|
|
201
|
+
// The attempt-qualified delete is nevertheless successor-safe: it
|
|
202
|
+
// removes only this attempt/generation and clears the pointer only
|
|
203
|
+
// when it still names that exact generation. Finish that cleanup
|
|
204
|
+
// directly so a successful turn can continue into workspace capture,
|
|
205
|
+
// tool teardown, and lease release.
|
|
206
|
+
observability.info("retrying exact run credential cleanup after turn settlement", {
|
|
207
|
+
"opengeni.session_id": input.sessionId,
|
|
208
|
+
"opengeni.turn_id": attempt.turnId ?? "",
|
|
209
|
+
"opengeni.attempt_id": input.attemptId,
|
|
210
|
+
});
|
|
211
|
+
},
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
sandboxState.attemptWritersDrained = true;
|
|
215
|
+
if (control.acknowledgeQuiescence) {
|
|
216
|
+
// A cancellation before sandbox-backed capabilities exist still has
|
|
217
|
+
// no tool controller to drain. Sandbox agent construction fails closed
|
|
218
|
+
// when a backend exists but no controller was installed. Renewal
|
|
219
|
+
// writers, when present, were drained above in either case.
|
|
220
|
+
physicalToolQuiescenceConfirmed = true;
|
|
221
|
+
}
|
|
222
|
+
if (control.acknowledgeQuiescence && physicalToolQuiescenceConfirmed) {
|
|
223
|
+
// This receipt is part of the hard cancellation boundary, not
|
|
224
|
+
// housekeeping. Persist it immediately after the sandbox/tool fence
|
|
225
|
+
// and before lease, cache, recording, or provider cleanup. Its
|
|
226
|
+
// transaction also enqueues the exact workflow wake that will admit
|
|
227
|
+
// the replacement; Temporal activity terminalization does neither.
|
|
228
|
+
const proof: SessionAttemptQuiescenceProof = {
|
|
229
|
+
accountId: input.accountId,
|
|
230
|
+
workspaceId: input.workspaceId,
|
|
231
|
+
sessionId: input.sessionId,
|
|
232
|
+
attemptId: input.attemptId,
|
|
233
|
+
workflowId: input.workflowId,
|
|
234
|
+
workflowRunId: input.workflowRunId,
|
|
235
|
+
activityId: dispatchId,
|
|
236
|
+
};
|
|
237
|
+
const recoveryMode = await persistOrSignalSessionAttemptQuiescence({
|
|
238
|
+
proof,
|
|
239
|
+
persistReceipt: async () =>
|
|
240
|
+
await commitSessionAttemptQuiescence(db, {
|
|
241
|
+
accountId: input.accountId,
|
|
242
|
+
workspaceId: input.workspaceId,
|
|
243
|
+
sessionId: input.sessionId,
|
|
244
|
+
attemptId: input.attemptId,
|
|
245
|
+
temporalWorkflowId: input.workflowId,
|
|
246
|
+
temporalWorkflowRunId: input.workflowRunId,
|
|
247
|
+
temporalActivityId: dispatchId,
|
|
248
|
+
allowUninterrupted: true,
|
|
249
|
+
}),
|
|
250
|
+
...(wakeSessionWorkflow
|
|
251
|
+
? {
|
|
252
|
+
deliverWorkflowWake: async (
|
|
253
|
+
wake: NonNullable<SessionAttemptQuiescenceCommit["workflowWake"]>,
|
|
254
|
+
) =>
|
|
255
|
+
await wakeSessionWorkflow({
|
|
256
|
+
accountId: wake.accountId,
|
|
257
|
+
workspaceId: wake.workspaceId,
|
|
258
|
+
sessionId: wake.sessionId,
|
|
259
|
+
workflowId: wake.temporalWorkflowId,
|
|
260
|
+
wakeRevision: wake.wakeRevision,
|
|
261
|
+
interruptionRequested: wake.interruptionRequested,
|
|
262
|
+
}),
|
|
263
|
+
}
|
|
264
|
+
: {}),
|
|
265
|
+
publishEvents: async (events) => {
|
|
266
|
+
await waitForTurnFinalizerStep(
|
|
267
|
+
publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events),
|
|
268
|
+
finalizerSignal,
|
|
269
|
+
);
|
|
270
|
+
},
|
|
271
|
+
signalProof: signalSessionAttemptQuiesced,
|
|
272
|
+
heartbeat: (deliveryAttempt, retryMs) => {
|
|
273
|
+
activityContext?.heartbeat({
|
|
274
|
+
phase: "quiescence-proof-delivery",
|
|
275
|
+
sessionId: input.sessionId,
|
|
276
|
+
attemptId: input.attemptId,
|
|
277
|
+
deliveryAttempt,
|
|
278
|
+
retryMs,
|
|
279
|
+
at: new Date().toISOString(),
|
|
280
|
+
});
|
|
281
|
+
},
|
|
282
|
+
onReceiptFailure: (error) => {
|
|
283
|
+
console.error(
|
|
284
|
+
"agent turn quiescence receipt exhausted; signalling proof",
|
|
285
|
+
safeErrorDiagnostic(error),
|
|
286
|
+
);
|
|
287
|
+
},
|
|
288
|
+
onWakeFailure: (error) => {
|
|
289
|
+
console.error(
|
|
290
|
+
"agent turn quiescence immediate workflow wake failed; outbox repair retained",
|
|
291
|
+
safeErrorDiagnostic(error),
|
|
292
|
+
);
|
|
293
|
+
},
|
|
294
|
+
onPublishFailure: (error) => {
|
|
295
|
+
console.error("agent turn quiescence event fanout failed", safeErrorDiagnostic(error));
|
|
296
|
+
},
|
|
297
|
+
onSignalFailure: (error, deliveryAttempt, retryMs) => {
|
|
298
|
+
console.error("agent turn quiescence proof signal failed; retrying", {
|
|
299
|
+
error: safeErrorDiagnostic(error),
|
|
300
|
+
attempt: deliveryAttempt,
|
|
301
|
+
retryMs,
|
|
302
|
+
});
|
|
303
|
+
},
|
|
304
|
+
});
|
|
305
|
+
quiescenceReceiptOrProofDurable = true;
|
|
306
|
+
if (recoveryMode === "signal") {
|
|
307
|
+
observability.info("agent turn quiescence proof handed to workflow recovery", {
|
|
308
|
+
"opengeni.session_id": input.sessionId,
|
|
309
|
+
"opengeni.attempt_id": input.attemptId,
|
|
310
|
+
"opengeni.workflow_run_id": input.workflowRunId,
|
|
311
|
+
"opengeni.activity_id": dispatchId,
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
// Drain the buffered Connected Machine op events (infra failures + healed
|
|
316
|
+
// recoveries) to durable session events — awaited, best-effort, never blocking
|
|
317
|
+
// the turn. Sync observer → buffer → single awaited append here (no unawaited
|
|
318
|
+
// DB write inside the activity). Scoped to this turn; skipped if no turnId
|
|
319
|
+
// (the op ran under a turn, so on the normal path turnId is set).
|
|
320
|
+
const machineOpEvents = machineOpObserver.drainEvents();
|
|
321
|
+
if (machineOpEvents.length > 0 && attempt.turnId && attempt.executionGeneration > 0) {
|
|
322
|
+
await waitForTurnFinalizerStep(
|
|
323
|
+
appendAndPublishTurnEventsFenced(
|
|
324
|
+
db,
|
|
325
|
+
bus,
|
|
326
|
+
input.workspaceId,
|
|
327
|
+
input.sessionId,
|
|
328
|
+
attempt.turnId,
|
|
329
|
+
attempt.executionGeneration,
|
|
330
|
+
input.attemptId,
|
|
331
|
+
machineOpEvents.map((event) => ({
|
|
332
|
+
...event,
|
|
333
|
+
turnId: attempt.turnId ?? null,
|
|
334
|
+
})),
|
|
335
|
+
).catch(() => undefined),
|
|
336
|
+
finalizerSignal,
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
// Multi-account P4: flush the serving account's free per-turn caches ONCE,
|
|
340
|
+
// best-effort (same discipline as today's usage write). Both writers skip
|
|
341
|
+
// version/updatedAt, so neither can race the token-refresh CAS.
|
|
342
|
+
if (providerTurn.effectiveCodexCredentialId) {
|
|
343
|
+
// Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
|
|
344
|
+
// full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
|
|
345
|
+
// is byte-identical to the /wham/usage write — no partial-window clobber.
|
|
346
|
+
if (providerTurn.latestCodexUsage) {
|
|
347
|
+
const usageMutation = await waitForTurnFinalizerStep(
|
|
348
|
+
recordCodexAccountUsageWithWakeTargets(
|
|
349
|
+
db,
|
|
350
|
+
input.workspaceId,
|
|
351
|
+
providerTurn.effectiveCodexCredentialId,
|
|
352
|
+
providerTurn.latestCodexUsage,
|
|
353
|
+
).catch(() => null),
|
|
354
|
+
finalizerSignal,
|
|
355
|
+
);
|
|
356
|
+
if (usageMutation) {
|
|
357
|
+
await waitForTurnFinalizerStep(
|
|
358
|
+
signalCodexCapacityWakeTargets(
|
|
359
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
360
|
+
usageMutation.wakeTargets,
|
|
361
|
+
),
|
|
362
|
+
finalizerSignal,
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
leases.codex.stopHeartbeat();
|
|
368
|
+
if (
|
|
369
|
+
leases.codex.held &&
|
|
370
|
+
attempt.turnId &&
|
|
371
|
+
leases.codex.holderId &&
|
|
372
|
+
leases.codex.generation !== null
|
|
373
|
+
) {
|
|
374
|
+
await waitForTurnFinalizerStep(
|
|
375
|
+
releaseCodexCredentialLease(
|
|
376
|
+
db,
|
|
377
|
+
input.accountId,
|
|
378
|
+
input.workspaceId,
|
|
379
|
+
attempt.turnId,
|
|
380
|
+
leases.codex.holderId,
|
|
381
|
+
leases.codex.generation,
|
|
382
|
+
).catch(() => undefined),
|
|
383
|
+
finalizerSignal,
|
|
384
|
+
);
|
|
385
|
+
leases.codex.held = false;
|
|
386
|
+
}
|
|
387
|
+
if (
|
|
388
|
+
providerTurn.effectiveXaiCredentialId &&
|
|
389
|
+
leases.xai.subjectId &&
|
|
390
|
+
providerTurn.xaiRequestContext &&
|
|
391
|
+
!providerTurn.xaiCredentialQuarantined
|
|
392
|
+
) {
|
|
393
|
+
const quota = await waitForTurnFinalizerStep(
|
|
394
|
+
fetchXaiSubscriptionQuota({ context: providerTurn.xaiRequestContext }).catch(() => null),
|
|
395
|
+
finalizerSignal,
|
|
396
|
+
);
|
|
397
|
+
if (quota) {
|
|
398
|
+
await waitForTurnFinalizerStep(
|
|
399
|
+
updateXaiQuotaMetadata(db, {
|
|
400
|
+
workspaceId: input.workspaceId,
|
|
401
|
+
subjectId: leases.xai.subjectId,
|
|
402
|
+
credentialId: providerTurn.effectiveXaiCredentialId,
|
|
403
|
+
quotaUsedPercent: quota.usedPercent,
|
|
404
|
+
quotaResetAt: quota.period?.end ?? null,
|
|
405
|
+
quotaCheckedAt: quota.checkedAt,
|
|
406
|
+
exhaustedUntil:
|
|
407
|
+
quota.usedPercent !== null && quota.usedPercent >= 100
|
|
408
|
+
? (quota.period?.end ?? null)
|
|
409
|
+
: null,
|
|
410
|
+
}).catch(() => undefined),
|
|
411
|
+
finalizerSignal,
|
|
412
|
+
);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
leases.xai.stopHeartbeat();
|
|
416
|
+
if (
|
|
417
|
+
leases.xai.held &&
|
|
418
|
+
attempt.turnId &&
|
|
419
|
+
leases.xai.subjectId &&
|
|
420
|
+
leases.xai.holderId &&
|
|
421
|
+
leases.xai.generation !== null
|
|
422
|
+
) {
|
|
423
|
+
await waitForTurnFinalizerStep(
|
|
424
|
+
releaseXaiCredentialLease(db, {
|
|
425
|
+
workspaceId: input.workspaceId,
|
|
426
|
+
subjectId: leases.xai.subjectId,
|
|
427
|
+
turnId: attempt.turnId,
|
|
428
|
+
holderId: leases.xai.holderId,
|
|
429
|
+
generation: leases.xai.generation,
|
|
430
|
+
}).catch(() => undefined),
|
|
431
|
+
finalizerSignal,
|
|
432
|
+
);
|
|
433
|
+
leases.xai.held = false;
|
|
434
|
+
}
|
|
435
|
+
// Workbench v2 turn-end workspace capture — runs FIRST in
|
|
436
|
+
// the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
|
|
437
|
+
// tool ran before this finally, so /workspace is already final; capture is
|
|
438
|
+
// FS-equivalent to the already-settled recording preparation and the warm
|
|
439
|
+
// snapshot (neither mutates workspace files). Running it here — BEFORE
|
|
440
|
+
// preparedTools.close() (which tears down tools / computer-use / the display
|
|
441
|
+
// stack and is what starts the Modal box exiting a few seconds later) —
|
|
442
|
+
// gives capture the full live-box margin instead of racing the teardown
|
|
443
|
+
// tail, which was dropping 100% of captures on real Modal desktop boxes
|
|
444
|
+
// ("request cancelled due to container exiting", 0 rows). External module:
|
|
445
|
+
// self-capped at 120s, best-effort (never throws past its boundary),
|
|
446
|
+
// epoch-fenced, and it NEVER closes the box. The emitted
|
|
447
|
+
// workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
|
|
448
|
+
// content).
|
|
449
|
+
if (process.env.OPENGENI_TEST_SCENARIO === "sandbox") {
|
|
450
|
+
console.log(
|
|
451
|
+
`[sandbox-e2e] capture preflight ownership=${settings.sandboxOwnershipEnabled} enabled=${settings.workspaceCaptureEnabled} resolved=${Boolean(sandboxState.resolvedSandbox)} session=${Boolean(sandboxState.setupBoxSession)} group=${Boolean(sandboxState.sandboxGroupId)} storage=${Boolean(objectStorage)}`,
|
|
452
|
+
);
|
|
453
|
+
}
|
|
454
|
+
const runTurnEndPersistence = shouldRunTurnEndWorkspacePersistence({
|
|
455
|
+
activityStatus: control.activityStatus,
|
|
456
|
+
cancellationRequested: finalizerSignal?.aborted === true,
|
|
457
|
+
});
|
|
458
|
+
if (
|
|
459
|
+
runTurnEndPersistence &&
|
|
460
|
+
attempt.turnId &&
|
|
461
|
+
sandboxState.resolvedSandbox &&
|
|
462
|
+
sandboxState.setupBoxSession &&
|
|
463
|
+
sandboxState.sandboxGroupId
|
|
464
|
+
) {
|
|
465
|
+
// Block new periodic snapshots, then drain any one already in flight.
|
|
466
|
+
// Keep the lease heartbeat itself running: capture may legitimately
|
|
467
|
+
// exceed the 90s holder TTL, and the reaper must remain unable to drain
|
|
468
|
+
// the exact box while this holder still reads it.
|
|
469
|
+
sandboxState.turnEndCaptureInProgress = true;
|
|
470
|
+
if (sandboxState.snapshotInFlight) {
|
|
471
|
+
await waitForWarmSnapshot(
|
|
472
|
+
sandboxState.snapshotInFlight,
|
|
473
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
474
|
+
finalizerSignal,
|
|
475
|
+
);
|
|
476
|
+
}
|
|
477
|
+
const captureEstablished = sandboxState.resolvedSandbox.established;
|
|
478
|
+
await captureWorkspaceRevision({
|
|
479
|
+
db,
|
|
480
|
+
objectStorage,
|
|
481
|
+
settings,
|
|
482
|
+
publish: async (events) => {
|
|
483
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
|
|
484
|
+
},
|
|
485
|
+
session: sandboxState.setupBoxSession as ChannelASession,
|
|
486
|
+
openReadSession: async () =>
|
|
487
|
+
await openFreshWorkspaceCaptureSession({
|
|
488
|
+
backendId: captureEstablished.backendId,
|
|
489
|
+
client: captureEstablished.client,
|
|
490
|
+
session: sandboxState.setupBoxSession as ChannelASession,
|
|
491
|
+
expectedInstanceId: captureEstablished.instanceId,
|
|
492
|
+
}),
|
|
493
|
+
leaseEpoch: sandboxState.resolvedSandbox.leaseEpoch,
|
|
494
|
+
sandboxGroupId: sandboxState.sandboxGroupId,
|
|
495
|
+
accountId: input.accountId,
|
|
496
|
+
workspaceId: input.workspaceId,
|
|
497
|
+
sessionId: input.sessionId,
|
|
498
|
+
turnId: attempt.turnId,
|
|
499
|
+
attemptId: input.attemptId,
|
|
500
|
+
observability,
|
|
501
|
+
...(finalizerSignal ? { signal: finalizerSignal } : {}),
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
eventing.toolPreparationClosing = true;
|
|
505
|
+
if (eventing.toolPreparationReady) {
|
|
506
|
+
await waitForTurnFinalizerStep(
|
|
507
|
+
eventing.toolPreparationReady.catch(() => undefined),
|
|
508
|
+
finalizerSignal,
|
|
509
|
+
);
|
|
510
|
+
}
|
|
511
|
+
if (eventing.codemodeDispatcher) {
|
|
512
|
+
await waitForTurnFinalizerStep(
|
|
513
|
+
eventing.codemodeDispatcher.close().catch(() => undefined),
|
|
514
|
+
finalizerSignal,
|
|
515
|
+
);
|
|
516
|
+
eventing.codemodeDispatcher = null;
|
|
517
|
+
}
|
|
518
|
+
if (eventing.preparedTools) {
|
|
519
|
+
await waitForTurnFinalizerStep(
|
|
520
|
+
eventing.preparedTools.close().catch(() => undefined),
|
|
521
|
+
finalizerSignal,
|
|
522
|
+
);
|
|
523
|
+
}
|
|
524
|
+
if (eventing.heartbeatTimer) {
|
|
525
|
+
clearInterval(eventing.heartbeatTimer);
|
|
526
|
+
}
|
|
527
|
+
if (sandboxState.turnSandboxProvisioner?.hasStarted()) {
|
|
528
|
+
await waitForTurnFinalizerStep(
|
|
529
|
+
sandboxState.turnSandboxProvisioner.waitForSettled(30_000),
|
|
530
|
+
finalizerSignal,
|
|
531
|
+
);
|
|
532
|
+
} else if (sandboxState.prefetchedManagedBox) {
|
|
533
|
+
// Create finished (or was aborted) before get()/setup. Join so a
|
|
534
|
+
// successful prefetch cannot leak a holder after this attempt ends.
|
|
535
|
+
await waitForTurnFinalizerStep(
|
|
536
|
+
sandboxState.prefetchedManagedBox.then(
|
|
537
|
+
(box) => {
|
|
538
|
+
sandboxState.prefetchedManagedBoxResult = box;
|
|
539
|
+
},
|
|
540
|
+
() => undefined,
|
|
541
|
+
),
|
|
542
|
+
finalizerSignal,
|
|
543
|
+
);
|
|
544
|
+
}
|
|
545
|
+
// P1.2: stop the lease-TTL refresh, release the turn holder (idempotent
|
|
546
|
+
// delete-my-row; refcount-- and warm->draining if it hit 0 with no turns),
|
|
547
|
+
// and DROP the in-memory handle. Release NEVER stops the box — the reaper
|
|
548
|
+
// (P1.3) issues the provider stop() past the drain grace at refcount 0; the
|
|
549
|
+
// box rides the provider idle-timeout in the meantime. Best-effort: a
|
|
550
|
+
// release failure must never mask the turn's real outcome.
|
|
551
|
+
if (sandboxState.leaseHeartbeatTimer) {
|
|
552
|
+
stopLeaseHeartbeat();
|
|
553
|
+
}
|
|
554
|
+
if (sandboxState.rotationInFlight) {
|
|
555
|
+
await waitForTurnFinalizerStep(sandboxState.rotationInFlight, finalizerSignal).catch(
|
|
556
|
+
() => undefined,
|
|
557
|
+
);
|
|
558
|
+
}
|
|
559
|
+
// A recording normally closes inside the attempt-fenced turn settlement.
|
|
560
|
+
// Reaching finally with one still active means settlement threw, never ran,
|
|
561
|
+
// or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
|
|
562
|
+
// failed; publish no event and leave the artifact recoverable on the box.
|
|
563
|
+
await waitForTurnFinalizerStep(
|
|
564
|
+
abandonActiveRecording(
|
|
565
|
+
"activity ended without recording settlement",
|
|
566
|
+
recordingState.didComputerUse ? "failed" : "discard",
|
|
567
|
+
),
|
|
568
|
+
finalizerSignal,
|
|
569
|
+
);
|
|
570
|
+
if (sandboxState.resolvedSandbox) {
|
|
571
|
+
// TURN-END mid-session snapshot (sandbox-file-persistence): fold the
|
|
572
|
+
// turn's finished /workspace onto the lease before releasing the holder,
|
|
573
|
+
// so the work this turn just produced survives any unclean box death in
|
|
574
|
+
// the idle window ahead. Throttled by the same interval as the heartbeat
|
|
575
|
+
// tick (a short turn right after a snapshot skips — bounded-loss contract
|
|
576
|
+
// is the interval, not per-turn). Best-effort and time-capped by the
|
|
577
|
+
// helper's own failure discipline; never delays release on failure.
|
|
578
|
+
const settledTurnId = attempt.turnId;
|
|
579
|
+
if (
|
|
580
|
+
runTurnEndPersistence &&
|
|
581
|
+
sandboxState.setupBoxSession &&
|
|
582
|
+
sandboxState.sandboxGroupId &&
|
|
583
|
+
settledTurnId
|
|
584
|
+
) {
|
|
585
|
+
// Single-flight vs the heartbeat capture: the timer is already cleared
|
|
586
|
+
// above, but a capture it launched may still be in flight — and that
|
|
587
|
+
// capture predates the turn's final writes. Wait for it, but only up
|
|
588
|
+
// to the snapshot timeout: release must never depend on an unbounded
|
|
589
|
+
// provider capture.
|
|
590
|
+
if (sandboxState.snapshotInFlight) {
|
|
591
|
+
await waitForWarmSnapshot(
|
|
592
|
+
sandboxState.snapshotInFlight,
|
|
593
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
594
|
+
finalizerSignal,
|
|
595
|
+
);
|
|
596
|
+
}
|
|
597
|
+
const persisted = await maybePersistWarmWorkspaceSnapshot(
|
|
598
|
+
{ db, settings, objectStorage },
|
|
599
|
+
{
|
|
600
|
+
accountId: input.accountId,
|
|
601
|
+
workspaceId: input.workspaceId,
|
|
602
|
+
sessionId: input.sessionId,
|
|
603
|
+
turnId: settledTurnId,
|
|
604
|
+
attemptId: input.attemptId,
|
|
605
|
+
sandboxGroupId: sandboxState.sandboxGroupId,
|
|
606
|
+
},
|
|
607
|
+
sandboxState.setupBoxSession,
|
|
608
|
+
sandboxState.resolvedSandbox.leaseEpoch,
|
|
609
|
+
finalizerSignal,
|
|
610
|
+
);
|
|
611
|
+
if (persisted && eventing.publish) {
|
|
612
|
+
await eventing
|
|
613
|
+
.publish([
|
|
614
|
+
{
|
|
615
|
+
type: "sandbox.box.snapshot",
|
|
616
|
+
payload: { trigger: "turn-end" },
|
|
617
|
+
},
|
|
618
|
+
])
|
|
619
|
+
.catch(() => undefined);
|
|
620
|
+
}
|
|
621
|
+
// NB workspace capture no longer runs here — it moved to
|
|
622
|
+
// the TOP of this finally (before preparedTools.close) so it completes
|
|
623
|
+
// while the box is still solidly alive, instead of racing the turn-end
|
|
624
|
+
// teardown that was killing 100% of captures on real Modal desktop boxes.
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
} catch (error) {
|
|
628
|
+
finalizationError ??= error;
|
|
629
|
+
console.error(
|
|
630
|
+
"agent turn finalization failed (turn outcome unaffected)",
|
|
631
|
+
safeErrorDiagnostic(error),
|
|
632
|
+
);
|
|
633
|
+
} finally {
|
|
634
|
+
// The writer drain is the only authority that licenses settling an
|
|
635
|
+
// abandoned turn admission. Keep this second-stage release outside all
|
|
636
|
+
// later housekeeping so an event/cache/capture/recording failure cannot
|
|
637
|
+
// strand a null-outcome admission forever. Do not race it against
|
|
638
|
+
// Temporal cancellation: the eager listener may already have dropped
|
|
639
|
+
// the holder, and this idempotent proof-bearing pass still must run.
|
|
640
|
+
stopLeaseHeartbeat();
|
|
641
|
+
const sandboxReleaseTargets = new Set(sandboxState.lateSandboxesAwaitingWriterDrain);
|
|
642
|
+
sandboxState.lateSandboxesAwaitingWriterDrain.clear();
|
|
643
|
+
if (sandboxState.resolvedSandbox) {
|
|
644
|
+
sandboxReleaseTargets.add(sandboxState.resolvedSandbox);
|
|
645
|
+
sandboxState.resolvedSandbox = null;
|
|
646
|
+
} else if (sandboxState.prefetchedManagedBoxResult) {
|
|
647
|
+
sandboxReleaseTargets.add(sandboxState.prefetchedManagedBoxResult);
|
|
648
|
+
}
|
|
649
|
+
sandboxState.prefetchedManagedBoxResult = null;
|
|
650
|
+
if (sandboxState.attemptWritersDrained) {
|
|
651
|
+
for (const sandboxToRelease of sandboxReleaseTargets) {
|
|
652
|
+
try {
|
|
653
|
+
await releaseTurnSandboxAfterWriterDrain(sandboxToRelease);
|
|
654
|
+
} catch (releaseError) {
|
|
655
|
+
finalizationError ??= releaseError;
|
|
656
|
+
console.error(
|
|
657
|
+
"sandbox lease quiesced release failed (turn outcome unaffected)",
|
|
658
|
+
safeErrorDiagnostic(releaseError),
|
|
659
|
+
);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
} else {
|
|
663
|
+
// No proof exists. Ensure every late result still receives the eager
|
|
664
|
+
// leak-prevention release while preserving its admissions as a
|
|
665
|
+
// fail-closed archive-capture fence.
|
|
666
|
+
for (const sandboxToRelease of sandboxReleaseTargets) {
|
|
667
|
+
await sandboxToRelease.release().catch(() => undefined);
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
// Close provisioning after consuming the currently known targets. A
|
|
671
|
+
// provider result that races in later is routed through releaseLateSandbox;
|
|
672
|
+
// targets already released above synchronously removed their listener.
|
|
673
|
+
if (!sandboxResumeController.signal.aborted) {
|
|
674
|
+
sandboxResumeController.abort(
|
|
675
|
+
cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
|
|
676
|
+
);
|
|
677
|
+
}
|
|
678
|
+
cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
|
|
679
|
+
const completedAt = performance.now();
|
|
680
|
+
const durationSeconds = (completedAt - activityStarted) / 1000;
|
|
681
|
+
const finalizationDurationSeconds = (completedAt - finalizationStarted) / 1000;
|
|
682
|
+
observability.observeHistogram({
|
|
683
|
+
name: "opengeni_turn_finalization_duration_seconds",
|
|
684
|
+
help: "Agent turn finalization duration, including workspace housekeeping and lease release.",
|
|
685
|
+
labels: {
|
|
686
|
+
cancellation_requested: String(control.cancellationRequestedAt !== null),
|
|
687
|
+
},
|
|
688
|
+
value: finalizationDurationSeconds,
|
|
689
|
+
});
|
|
690
|
+
if (control.cancellationRequestedAt !== null) {
|
|
691
|
+
const physicalCancellationDurationSeconds =
|
|
692
|
+
(completedAt - control.cancellationRequestedAt) / 1000;
|
|
693
|
+
observability.observeHistogram({
|
|
694
|
+
name: "opengeni_turn_physical_cancellation_duration_seconds",
|
|
695
|
+
help: "Time from Temporal cancellation delivery until the activity physically stops.",
|
|
696
|
+
value: physicalCancellationDurationSeconds,
|
|
697
|
+
});
|
|
698
|
+
observability.info("agent turn physical cancellation completed", {
|
|
699
|
+
durationMs: Math.round(physicalCancellationDurationSeconds * 1000),
|
|
700
|
+
...(sandboxState.sandboxGroupId
|
|
701
|
+
? {
|
|
702
|
+
sandboxLeaseKey: sandboxLeaseTelemetryKey(
|
|
703
|
+
input.workspaceId,
|
|
704
|
+
sandboxState.sandboxGroupId,
|
|
705
|
+
),
|
|
706
|
+
}
|
|
707
|
+
: {}),
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
observability.recordWorkerActivity({
|
|
711
|
+
activity: "runAgentTurn",
|
|
712
|
+
status: finalizationError ? "cleanup_failed" : control.activityStatus,
|
|
713
|
+
durationSeconds,
|
|
714
|
+
});
|
|
715
|
+
if (attempt.turnId && control.activityStatus !== "unknown") {
|
|
716
|
+
turnLifecycleMetricsFor(observability).finish(
|
|
717
|
+
attempt.turnId,
|
|
718
|
+
control.turnMetricOutcome,
|
|
719
|
+
durationSeconds,
|
|
720
|
+
);
|
|
721
|
+
}
|
|
722
|
+
activitySpan.end({
|
|
723
|
+
attributes: {
|
|
724
|
+
"opengeni.turn_id": attempt.turnId ?? "",
|
|
725
|
+
"opengeni.status": control.activityStatus,
|
|
726
|
+
"opengeni.variable_set_id": workspaceRefs.variableSetId,
|
|
727
|
+
"opengeni.rig_id": workspaceRefs.rigId,
|
|
728
|
+
"opengeni.rig_version_id": workspaceRefs.rigVersionId,
|
|
729
|
+
"opengeni.codex_credential_id": providerTurn.effectiveCodexCredentialId ?? "",
|
|
730
|
+
"opengeni.duration_ms": Math.round(durationSeconds * 1000),
|
|
731
|
+
"opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1000),
|
|
732
|
+
},
|
|
733
|
+
error:
|
|
734
|
+
finalizationError || control.activityError
|
|
735
|
+
? safeErrorForTelemetry(finalizationError ?? control.activityError)
|
|
736
|
+
: undefined,
|
|
737
|
+
});
|
|
738
|
+
// This timer runs only after the activity promise and its full turn
|
|
739
|
+
// stack unwind. Checkpoint collection alone cannot reclaim objects that
|
|
740
|
+
// remain strongly reachable until this terminal boundary.
|
|
741
|
+
turnCompletionMemoryCollector.schedule(observability);
|
|
742
|
+
assertPhysicalToolQuiescenceForCancellation({
|
|
743
|
+
acknowledgeQuiescence: control.acknowledgeQuiescence,
|
|
744
|
+
physicalToolQuiescenceConfirmed,
|
|
745
|
+
failure: finalizationError,
|
|
746
|
+
});
|
|
747
|
+
assertSessionAttemptQuiescenceRecoveryDurable({
|
|
748
|
+
acknowledgeQuiescence: control.acknowledgeQuiescence,
|
|
749
|
+
physicalToolQuiescenceConfirmed,
|
|
750
|
+
receiptOrProofDurable: quiescenceReceiptOrProofDurable,
|
|
751
|
+
failure: finalizationError,
|
|
752
|
+
});
|
|
753
|
+
}
|
|
754
|
+
}
|