@opengeni/worker-bundle 0.11.1 → 0.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +22 -6
- package/dist/index.js +892 -317
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +15 -12
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +444 -146
- package/src/activities/capabilities.ts +19 -37
- package/src/activities/codex-capacity.ts +1 -1
- package/src/activities/redaction.ts +9 -56
- package/src/activities/sandbox-lease.ts +411 -0
- package/src/activities/types.ts +6 -6
- package/src/http.ts +14 -2
- package/src/index.ts +24 -2
- package/src/observability-metrics.ts +137 -0
- package/src/sandbox-resume.ts +33 -114
- package/src/sandbox-routing.ts +95 -30
- package/src/workflows/session.ts +14 -11
package/dist/index.js
CHANGED
|
@@ -11,7 +11,11 @@ import {
|
|
|
11
11
|
startupRetryOptions,
|
|
12
12
|
temporalConnectionOptions
|
|
13
13
|
} from "@opengeni/config";
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
assertRuntimeDatabasePosture as assertRuntimeDatabasePosture2,
|
|
16
|
+
createDb as createDb2,
|
|
17
|
+
markSessionWorkflowWakeDelivered
|
|
18
|
+
} from "@opengeni/db";
|
|
15
19
|
import { createNatsEventBus as createNatsEventBus2 } from "@opengeni/events";
|
|
16
20
|
import {
|
|
17
21
|
createObservability as createObservability2,
|
|
@@ -137,6 +141,7 @@ import {
|
|
|
137
141
|
SUMMARY_BUFFER_TOKENS as SUMMARY_BUFFER_TOKENS2,
|
|
138
142
|
runOwnedSandboxSetup,
|
|
139
143
|
RoutingMutationOutcomeUnknownError,
|
|
144
|
+
WorkspaceArchiveIntegrityError as WorkspaceArchiveIntegrityError2,
|
|
140
145
|
swapTargetEstablishability,
|
|
141
146
|
deleteRecordingArtifacts as deleteRecordingArtifacts2,
|
|
142
147
|
stopRecording as stopRecordingOnBox2
|
|
@@ -181,22 +186,14 @@ import { CancelledFailure as CancelledFailure2 } from "@temporalio/activity";
|
|
|
181
186
|
// src/activities/capabilities.ts
|
|
182
187
|
import {
|
|
183
188
|
environmentsEncryptionKeyBytes,
|
|
184
|
-
|
|
189
|
+
withCodexCatalogProvider
|
|
185
190
|
} from "@opengeni/config";
|
|
186
191
|
import { settingsWithMcpCapabilityServers } from "@opengeni/core";
|
|
187
192
|
import {
|
|
188
193
|
CODEX_APPS_MCP_SERVER_ID,
|
|
189
194
|
CODEX_APPS_MCP_SERVER_NAME,
|
|
190
195
|
CODEX_APPS_MCP_URL,
|
|
191
|
-
CODEX_APPS_STARTUP_TIMEOUT_MS
|
|
192
|
-
CODEX_FALLBACK_MODEL_SLUGS,
|
|
193
|
-
CODEX_MODEL_CONTEXT_WINDOW_TOKENS,
|
|
194
|
-
CODEX_MODEL_EFFECTIVE_CONTEXT_WINDOW_TOKENS,
|
|
195
|
-
CODEX_MODEL_AUTO_COMPACT_TOKEN_LIMIT,
|
|
196
|
-
CODEX_MODEL_TOOL_OUTPUT_TRUNCATION_TOKENS,
|
|
197
|
-
CODEX_MODEL_ID_PREFIX,
|
|
198
|
-
CODEX_PROVIDER_BASE_URL,
|
|
199
|
-
CODEX_PROVIDER_ID
|
|
196
|
+
CODEX_APPS_STARTUP_TIMEOUT_MS
|
|
200
197
|
} from "@opengeni/codex";
|
|
201
198
|
import {
|
|
202
199
|
listEnabledMcpCapabilityServers,
|
|
@@ -208,7 +205,7 @@ async function settingsWithEnabledCapabilityMcpServers(db, workspaceId, settings
|
|
|
208
205
|
const enabled = await listEnabledMcpCapabilityServers(db, workspaceId);
|
|
209
206
|
return settingsWithMcpCapabilityServers(settings, enabled);
|
|
210
207
|
}
|
|
211
|
-
async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, settings) {
|
|
208
|
+
async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, settings, options) {
|
|
212
209
|
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
213
210
|
if (!encryptionKey) {
|
|
214
211
|
const metadata = await listSessionMcpServerMetadata(db, workspaceId, sessionId);
|
|
@@ -221,10 +218,15 @@ async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, a
|
|
|
221
218
|
);
|
|
222
219
|
}
|
|
223
220
|
}
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
221
|
+
const servers = await listSessionMcpServersForRun(
|
|
222
|
+
db,
|
|
223
|
+
workspaceId,
|
|
224
|
+
sessionId,
|
|
225
|
+
attemptId,
|
|
226
|
+
encryptionKey ?? null
|
|
227
227
|
);
|
|
228
|
+
options?.onResolvedServers?.(servers);
|
|
229
|
+
return settingsWithSessionMcpServers(settings, servers);
|
|
228
230
|
}
|
|
229
231
|
function settingsWithSessionMcpServers(settings, servers) {
|
|
230
232
|
if (servers.length === 0) {
|
|
@@ -255,6 +257,9 @@ async function settingsWithCodexCredential(db, workspaceId, settings, activeOver
|
|
|
255
257
|
return settings;
|
|
256
258
|
}
|
|
257
259
|
const withProvider = withCodexProvider(settings);
|
|
260
|
+
if (!settings.codexConnectedAppsEnabled) {
|
|
261
|
+
return withProvider;
|
|
262
|
+
}
|
|
258
263
|
return withCodexAppsMcpServer(withProvider);
|
|
259
264
|
}
|
|
260
265
|
function withCodexAppsMcpServer(settings) {
|
|
@@ -279,30 +284,7 @@ function withCodexAppsMcpServer(settings) {
|
|
|
279
284
|
};
|
|
280
285
|
}
|
|
281
286
|
function withCodexProvider(settings) {
|
|
282
|
-
|
|
283
|
-
if (providers.some((provider) => provider.id === CODEX_PROVIDER_ID)) {
|
|
284
|
-
return settings;
|
|
285
|
-
}
|
|
286
|
-
const codexProvider = {
|
|
287
|
-
kind: "codex-subscription",
|
|
288
|
-
id: CODEX_PROVIDER_ID,
|
|
289
|
-
label: "Codex (ChatGPT subscription)",
|
|
290
|
-
api: "responses",
|
|
291
|
-
baseUrl: CODEX_PROVIDER_BASE_URL,
|
|
292
|
-
models: CODEX_FALLBACK_MODEL_SLUGS.map((slug) => ({
|
|
293
|
-
id: `${CODEX_MODEL_ID_PREFIX}${slug}`,
|
|
294
|
-
upstreamModelId: slug,
|
|
295
|
-
label: slug,
|
|
296
|
-
reasoningEffort: true,
|
|
297
|
-
// These three values are the live Codex CLI catalog policy, kept distinct
|
|
298
|
-
// so proactive compaction and the effective input guard match Codex core.
|
|
299
|
-
contextWindowTokens: CODEX_MODEL_CONTEXT_WINDOW_TOKENS,
|
|
300
|
-
effectiveContextWindowTokens: CODEX_MODEL_EFFECTIVE_CONTEXT_WINDOW_TOKENS,
|
|
301
|
-
autoCompactTokenLimit: CODEX_MODEL_AUTO_COMPACT_TOKEN_LIMIT,
|
|
302
|
-
toolOutputTruncationTokens: CODEX_MODEL_TOOL_OUTPUT_TRUNCATION_TOKENS
|
|
303
|
-
}))
|
|
304
|
-
};
|
|
305
|
-
return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
|
|
287
|
+
return withCodexCatalogProvider(settings);
|
|
306
288
|
}
|
|
307
289
|
|
|
308
290
|
// src/activities/codex-rotation.ts
|
|
@@ -1009,7 +991,7 @@ function createCodexCapacityActivities(services) {
|
|
|
1009
991
|
}
|
|
1010
992
|
}
|
|
1011
993
|
if (result.action === "resumed") {
|
|
1012
|
-
return { action: "resumed"
|
|
994
|
+
return { action: "resumed" };
|
|
1013
995
|
}
|
|
1014
996
|
if (result.action === "waiting") {
|
|
1015
997
|
return {
|
|
@@ -1029,7 +1011,7 @@ function createCodexCapacityActivities(services) {
|
|
|
1029
1011
|
import {
|
|
1030
1012
|
buildModelResolver,
|
|
1031
1013
|
CODEX_CLIENT_VERSION,
|
|
1032
|
-
CODEX_FALLBACK_MODEL_SLUGS
|
|
1014
|
+
CODEX_FALLBACK_MODEL_SLUGS,
|
|
1033
1015
|
CodexReloginRequired,
|
|
1034
1016
|
classifyCodexResponseTimeoutError,
|
|
1035
1017
|
classifyCodexUsageLimitError,
|
|
@@ -1063,7 +1045,11 @@ function workflowIdForSession(sessionId) {
|
|
|
1063
1045
|
}
|
|
1064
1046
|
|
|
1065
1047
|
// src/activities/agent-turn.ts
|
|
1066
|
-
import {
|
|
1048
|
+
import {
|
|
1049
|
+
enabledCapabilityMcpToolRefs,
|
|
1050
|
+
resolveSessionToolPolicy,
|
|
1051
|
+
sessionToolPolicyAllowsDefaultNativeTools
|
|
1052
|
+
} from "@opengeni/core";
|
|
1067
1053
|
|
|
1068
1054
|
// src/activities/context-compaction.ts
|
|
1069
1055
|
import {
|
|
@@ -2675,44 +2661,14 @@ function childCompletionSummary(child, goal, terminalStatus) {
|
|
|
2675
2661
|
}
|
|
2676
2662
|
|
|
2677
2663
|
// src/activities/redaction.ts
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
let out = payload;
|
|
2687
|
-
for (const secret of usable) {
|
|
2688
|
-
if (out.includes(secret.value)) {
|
|
2689
|
-
out = out.split(secret.value).join(`[redacted:${secret.name}]`);
|
|
2690
|
-
}
|
|
2691
|
-
}
|
|
2692
|
-
return out;
|
|
2693
|
-
}
|
|
2694
|
-
if (Array.isArray(payload)) {
|
|
2695
|
-
return payload.map(redact);
|
|
2696
|
-
}
|
|
2697
|
-
if (isPlainObject(payload)) {
|
|
2698
|
-
return Object.fromEntries(
|
|
2699
|
-
Object.entries(payload).map(([key, value]) => [key, redact(value)])
|
|
2700
|
-
);
|
|
2701
|
-
}
|
|
2702
|
-
return payload;
|
|
2703
|
-
};
|
|
2704
|
-
return redact;
|
|
2705
|
-
}
|
|
2706
|
-
function identityRedactor(payload) {
|
|
2707
|
-
return payload;
|
|
2708
|
-
}
|
|
2709
|
-
function isPlainObject(value) {
|
|
2710
|
-
if (!value || typeof value !== "object") {
|
|
2711
|
-
return false;
|
|
2712
|
-
}
|
|
2713
|
-
const proto = Object.getPrototypeOf(value);
|
|
2714
|
-
return proto === Object.prototype || proto === null;
|
|
2715
|
-
}
|
|
2664
|
+
import {
|
|
2665
|
+
createSecretRedactor,
|
|
2666
|
+
identityRedactor,
|
|
2667
|
+
isCredentialHeaderName,
|
|
2668
|
+
redactSensitiveData,
|
|
2669
|
+
redactSensitiveText,
|
|
2670
|
+
redactSerializedJson
|
|
2671
|
+
} from "@opengeni/contracts";
|
|
2716
2672
|
|
|
2717
2673
|
// src/activities/run-input.ts
|
|
2718
2674
|
import { createHash } from "crypto";
|
|
@@ -3162,6 +3118,7 @@ async function nextStreamEvent(iterator, context) {
|
|
|
3162
3118
|
}
|
|
3163
3119
|
|
|
3164
3120
|
// src/sandbox-resume.ts
|
|
3121
|
+
import { redactSensitiveText as redactSensitiveText2 } from "@opengeni/contracts";
|
|
3165
3122
|
import {
|
|
3166
3123
|
acquireLease,
|
|
3167
3124
|
beginSandboxRematerialization,
|
|
@@ -3198,6 +3155,9 @@ function sandboxLeaseHolderIdForAttempt(attemptId) {
|
|
|
3198
3155
|
}
|
|
3199
3156
|
return `turn-attempt:${normalized}`;
|
|
3200
3157
|
}
|
|
3158
|
+
function isRetryableDegradedRestore(restore) {
|
|
3159
|
+
return restore.status === "degraded" && restore.retryable === true;
|
|
3160
|
+
}
|
|
3201
3161
|
var SandboxWarmingTimeoutError = class extends Error {
|
|
3202
3162
|
constructor(backend, timeoutMs) {
|
|
3203
3163
|
super(
|
|
@@ -3237,6 +3197,16 @@ var SnapshotTimeoutError = class extends Error {
|
|
|
3237
3197
|
this.name = "SnapshotTimeoutError";
|
|
3238
3198
|
}
|
|
3239
3199
|
};
|
|
3200
|
+
function safeSnapshotError(error) {
|
|
3201
|
+
const rawName = error instanceof Error ? error.name : "Error";
|
|
3202
|
+
return {
|
|
3203
|
+
name: /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error",
|
|
3204
|
+
message: redactSensitiveText2(error instanceof Error ? error.message : String(error)).slice(
|
|
3205
|
+
0,
|
|
3206
|
+
2048
|
|
3207
|
+
)
|
|
3208
|
+
};
|
|
3209
|
+
}
|
|
3240
3210
|
async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
|
|
3241
3211
|
if (signal?.aborted) return false;
|
|
3242
3212
|
let timeout;
|
|
@@ -3262,7 +3232,10 @@ async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
|
|
|
3262
3232
|
} catch (error) {
|
|
3263
3233
|
if (signal?.aborted) return false;
|
|
3264
3234
|
if (error instanceof SnapshotTimeoutError) {
|
|
3265
|
-
console.error(
|
|
3235
|
+
console.error(
|
|
3236
|
+
"mid-session workspace snapshot wait timed out (turn unaffected)",
|
|
3237
|
+
safeSnapshotError(error)
|
|
3238
|
+
);
|
|
3266
3239
|
return false;
|
|
3267
3240
|
}
|
|
3268
3241
|
return true;
|
|
@@ -3438,7 +3411,10 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
|
|
|
3438
3411
|
await deletePriorPersistedSnapshot(persistable, priorArchiveForGc);
|
|
3439
3412
|
return true;
|
|
3440
3413
|
} catch (error) {
|
|
3441
|
-
console.error(
|
|
3414
|
+
console.error(
|
|
3415
|
+
"mid-session workspace snapshot failed (turn unaffected)",
|
|
3416
|
+
safeSnapshotError(error)
|
|
3417
|
+
);
|
|
3442
3418
|
return false;
|
|
3443
3419
|
}
|
|
3444
3420
|
}
|
|
@@ -3759,70 +3735,6 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
3759
3735
|
throw warmingError;
|
|
3760
3736
|
}
|
|
3761
3737
|
}
|
|
3762
|
-
async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
3763
|
-
const { db, settings } = services;
|
|
3764
|
-
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
3765
|
-
let released = false;
|
|
3766
|
-
const release = async () => {
|
|
3767
|
-
if (released) {
|
|
3768
|
-
return;
|
|
3769
|
-
}
|
|
3770
|
-
released = true;
|
|
3771
|
-
await releaseLeaseHolder(db, {
|
|
3772
|
-
accountId: ids.accountId,
|
|
3773
|
-
workspaceId: ids.workspaceId,
|
|
3774
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
3775
|
-
kind,
|
|
3776
|
-
holderId,
|
|
3777
|
-
idleGraceMs: settings.sandboxIdleGraceMs
|
|
3778
|
-
});
|
|
3779
|
-
};
|
|
3780
|
-
const acquired = await acquireLease(db, {
|
|
3781
|
-
accountId: ids.accountId,
|
|
3782
|
-
workspaceId: ids.workspaceId,
|
|
3783
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
3784
|
-
kind,
|
|
3785
|
-
holderId,
|
|
3786
|
-
subjectId: ids.sessionId,
|
|
3787
|
-
backend: "selfhosted",
|
|
3788
|
-
os: "linux",
|
|
3789
|
-
leaseTtlMs,
|
|
3790
|
-
warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs
|
|
3791
|
-
});
|
|
3792
|
-
if (acquired.role === "blocked") {
|
|
3793
|
-
await release();
|
|
3794
|
-
throw new SandboxLeaseRecoveryBlockedError(
|
|
3795
|
-
ids.sandboxGroupId,
|
|
3796
|
-
acquired.lease.leaseEpoch,
|
|
3797
|
-
acquired.code,
|
|
3798
|
-
acquired.lease.recovery
|
|
3799
|
-
);
|
|
3800
|
-
}
|
|
3801
|
-
if (acquired.role === "fenced") {
|
|
3802
|
-
await release();
|
|
3803
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
3804
|
-
}
|
|
3805
|
-
if (acquired.role === "spawner") {
|
|
3806
|
-
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
3807
|
-
const committed = await commitWarmingToWarm(db, {
|
|
3808
|
-
accountId: ids.accountId,
|
|
3809
|
-
workspaceId: ids.workspaceId,
|
|
3810
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
3811
|
-
expectedEpoch,
|
|
3812
|
-
// No provider box id; stamp a clearly non-box marker for diagnostics.
|
|
3813
|
-
instanceId: `selfhosted:${ids.sessionId}`,
|
|
3814
|
-
resumeBackendId: "selfhosted",
|
|
3815
|
-
resumeState: null,
|
|
3816
|
-
leaseTtlMs
|
|
3817
|
-
});
|
|
3818
|
-
if (!committed.committed || !committed.lease) {
|
|
3819
|
-
await release();
|
|
3820
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
3821
|
-
}
|
|
3822
|
-
return { leaseEpoch: committed.lease.leaseEpoch, release };
|
|
3823
|
-
}
|
|
3824
|
-
return { leaseEpoch: acquired.lease.leaseEpoch, release };
|
|
3825
|
-
}
|
|
3826
3738
|
async function waitForWarm(services, ids) {
|
|
3827
3739
|
const { db, settings } = services;
|
|
3828
3740
|
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
@@ -3839,6 +3751,9 @@ async function waitForWarm(services, ids) {
|
|
|
3839
3751
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
3840
3752
|
}
|
|
3841
3753
|
if (lease.liveness === "cold") {
|
|
3754
|
+
if (isRetryableDegradedRestore(lease.recovery.restore)) {
|
|
3755
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
3756
|
+
}
|
|
3842
3757
|
if (lease.recovery.restore.status === "degraded" || lease.recovery.restore.status === "unrecoverable") {
|
|
3843
3758
|
throw new SandboxLeaseRecoveryBlockedError(
|
|
3844
3759
|
ids.sandboxGroupId,
|
|
@@ -3857,14 +3772,17 @@ async function waitForWarm(services, ids) {
|
|
|
3857
3772
|
import {
|
|
3858
3773
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
3859
3774
|
advanceWorkspaceGeneration,
|
|
3775
|
+
getRetainedProcess,
|
|
3860
3776
|
retainWorkspaceMutationProcess,
|
|
3777
|
+
retainedProcessSettlementIdentity,
|
|
3861
3778
|
settleRetainedProcess,
|
|
3862
3779
|
verifyRetainedProcessMutationSettlement,
|
|
3863
3780
|
verifyWorkspaceMutationSettlement,
|
|
3864
3781
|
getSandbox,
|
|
3865
3782
|
markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
|
|
3866
3783
|
readLease as readLease2,
|
|
3867
|
-
readActiveSandbox
|
|
3784
|
+
readActiveSandbox,
|
|
3785
|
+
SandboxRetainedProcessPromotionFencedError
|
|
3868
3786
|
} from "@opengeni/db";
|
|
3869
3787
|
import {
|
|
3870
3788
|
buildSelfhostedBackendSession,
|
|
@@ -3900,7 +3818,10 @@ function selfhostedTimeoutsFromSettings(settings) {
|
|
|
3900
3818
|
}
|
|
3901
3819
|
function selfhostedResolverTimeouts(settings) {
|
|
3902
3820
|
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
3903
|
-
return {
|
|
3821
|
+
return {
|
|
3822
|
+
selfhostedTimeoutMs: timeoutMs,
|
|
3823
|
+
selfhostedExecTimeoutMs: execTimeoutMs
|
|
3824
|
+
};
|
|
3904
3825
|
}
|
|
3905
3826
|
function homeRouteRecoveryDisposition(lease) {
|
|
3906
3827
|
if (!lease) return "unrecoverable";
|
|
@@ -3945,7 +3866,10 @@ async function resolveCurrentHomeBackend(services, ids, established) {
|
|
|
3945
3866
|
throw homeRouteRecoveryError(lease, fallbackEpoch);
|
|
3946
3867
|
}
|
|
3947
3868
|
if (lease.instanceId === established.instanceId) {
|
|
3948
|
-
services.onHomeSandboxRebound?.({
|
|
3869
|
+
services.onHomeSandboxRebound?.({
|
|
3870
|
+
established,
|
|
3871
|
+
leaseEpoch: lease.leaseEpoch
|
|
3872
|
+
});
|
|
3949
3873
|
return {
|
|
3950
3874
|
session: established.session,
|
|
3951
3875
|
sandboxId: null,
|
|
@@ -3999,7 +3923,10 @@ async function resolveCurrentHomeBackend(services, ids, established) {
|
|
|
3999
3923
|
if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
|
|
4000
3924
|
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
4001
3925
|
}
|
|
4002
|
-
services.onHomeSandboxRebound?.({
|
|
3926
|
+
services.onHomeSandboxRebound?.({
|
|
3927
|
+
established: rebound,
|
|
3928
|
+
leaseEpoch: lease.leaseEpoch
|
|
3929
|
+
});
|
|
4003
3930
|
Object.assign(established, rebound);
|
|
4004
3931
|
return {
|
|
4005
3932
|
session: rebound.session,
|
|
@@ -4040,29 +3967,46 @@ function afterPersistableHomeMutation(services, ids, home) {
|
|
|
4040
3967
|
}
|
|
4041
3968
|
const exactAdmission = admission;
|
|
4042
3969
|
if (outcome === "resolved" && retainedProcess) {
|
|
4043
|
-
|
|
4044
|
-
|
|
4045
|
-
|
|
4046
|
-
|
|
4047
|
-
|
|
4048
|
-
|
|
4049
|
-
|
|
4050
|
-
|
|
4051
|
-
|
|
4052
|
-
|
|
4053
|
-
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
|
|
3970
|
+
try {
|
|
3971
|
+
await retainWorkspaceMutationProcess(services.db, {
|
|
3972
|
+
accountId: fence.accountId,
|
|
3973
|
+
workspaceId: ids.workspaceId,
|
|
3974
|
+
sessionId: ids.sessionId,
|
|
3975
|
+
processId: retainedProcess.id,
|
|
3976
|
+
providerSessionId: retainedProcess.providerSessionId,
|
|
3977
|
+
admissionId: exactAdmission.id,
|
|
3978
|
+
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
3979
|
+
operation: op,
|
|
3980
|
+
owner: {
|
|
3981
|
+
kind: "turn",
|
|
3982
|
+
turnId: fence.turnId,
|
|
3983
|
+
executionGeneration: fence.executionGeneration,
|
|
3984
|
+
attemptId: fence.attemptId,
|
|
3985
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
3986
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
3987
|
+
expectedEpoch: backend.leaseEpoch,
|
|
3988
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
3989
|
+
routeKind: exactAdmission.routeKind,
|
|
3990
|
+
routeTargetId: exactAdmission.routeTargetId,
|
|
3991
|
+
routeEpoch: exactAdmission.routeEpoch
|
|
3992
|
+
}
|
|
3993
|
+
});
|
|
3994
|
+
} catch (error) {
|
|
3995
|
+
if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
|
|
3996
|
+
const durable = error.process;
|
|
3997
|
+
if (durable.id !== retainedProcess.id || durable.providerSessionId !== retainedProcess.providerSessionId || durable.sessionId !== ids.sessionId || durable.sandboxGroupId !== home.sandboxGroupId || durable.parentAdmissionId !== exactAdmission.id || durable.leaseEpoch !== backend.leaseEpoch || durable.providerInstanceId !== backend.providerInstanceId || durable.routeKind !== exactAdmission.routeKind || durable.routeTargetId !== exactAdmission.routeTargetId || durable.routeEpoch !== exactAdmission.routeEpoch || durable.state !== "active") {
|
|
3998
|
+
throw new Error("Durable retained-process promotion returned a mismatched identity", {
|
|
3999
|
+
cause: error
|
|
4000
|
+
});
|
|
4064
4001
|
}
|
|
4065
|
-
|
|
4002
|
+
return {
|
|
4003
|
+
status: "retained_process_durable_output_rejected",
|
|
4004
|
+
retainedProcess: {
|
|
4005
|
+
id: durable.id,
|
|
4006
|
+
providerSessionId: durable.providerSessionId
|
|
4007
|
+
}
|
|
4008
|
+
};
|
|
4009
|
+
}
|
|
4066
4010
|
return;
|
|
4067
4011
|
}
|
|
4068
4012
|
await verifyWorkspaceMutationSettlement(services.db, {
|
|
@@ -4114,12 +4058,21 @@ function afterRetainedProcessMutation(services, ids) {
|
|
|
4114
4058
|
function settleRetainedProcessForTurn(services, ids) {
|
|
4115
4059
|
const fence = ids.workspaceMutationFence;
|
|
4116
4060
|
if (!fence) return void 0;
|
|
4117
|
-
return async ({ process: process2, proof }) => {
|
|
4061
|
+
return async ({ backend, process: process2, proof }) => {
|
|
4062
|
+
const durable = await getRetainedProcess(services.db, {
|
|
4063
|
+
workspaceId: ids.workspaceId,
|
|
4064
|
+
sessionId: ids.sessionId,
|
|
4065
|
+
processId: process2.id
|
|
4066
|
+
});
|
|
4067
|
+
if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
|
|
4068
|
+
throw new Error("Retained-process settlement lost its exact durable backend identity");
|
|
4069
|
+
}
|
|
4118
4070
|
await settleRetainedProcess(services.db, {
|
|
4119
4071
|
accountId: fence.accountId,
|
|
4120
4072
|
workspaceId: ids.workspaceId,
|
|
4121
4073
|
sessionId: ids.sessionId,
|
|
4122
4074
|
processId: process2.id,
|
|
4075
|
+
expected: retainedProcessSettlementIdentity(durable),
|
|
4123
4076
|
outcome: proof.outcome,
|
|
4124
4077
|
exitCode: proof.exitCode,
|
|
4125
4078
|
reason: proof.reason,
|
|
@@ -4449,6 +4402,7 @@ import {
|
|
|
4449
4402
|
var turnTrackers = /* @__PURE__ */ new WeakMap();
|
|
4450
4403
|
var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
|
|
4451
4404
|
var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
|
|
4405
|
+
var retainedProcessTerminalBacklogTotals = /* @__PURE__ */ new WeakMap();
|
|
4452
4406
|
var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
|
|
4453
4407
|
function observabilityEventLogger(observability) {
|
|
4454
4408
|
return {
|
|
@@ -4734,6 +4688,103 @@ function recordSandboxOrphansTerminated(observability, count) {
|
|
|
4734
4688
|
amount: count
|
|
4735
4689
|
});
|
|
4736
4690
|
}
|
|
4691
|
+
var RETAINED_PROCESS_OWNER_STATES = [
|
|
4692
|
+
"direct",
|
|
4693
|
+
"queued",
|
|
4694
|
+
"running",
|
|
4695
|
+
"requires_action",
|
|
4696
|
+
"recovering",
|
|
4697
|
+
"waiting_capacity",
|
|
4698
|
+
"completed",
|
|
4699
|
+
"failed",
|
|
4700
|
+
"cancelled",
|
|
4701
|
+
"superseded",
|
|
4702
|
+
"withdrawn_for_edit",
|
|
4703
|
+
"missing",
|
|
4704
|
+
"unknown"
|
|
4705
|
+
];
|
|
4706
|
+
var RETAINED_PROCESS_OWNER_STATE_SET = new Set(RETAINED_PROCESS_OWNER_STATES);
|
|
4707
|
+
function recordRetainedProcessInventoryGauges(observability, counts) {
|
|
4708
|
+
const normalized = /* @__PURE__ */ new Map();
|
|
4709
|
+
for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {
|
|
4710
|
+
normalized.set(ownerState, { active: 0, terminal: 0 });
|
|
4711
|
+
}
|
|
4712
|
+
for (const count of counts) {
|
|
4713
|
+
const ownerState = RETAINED_PROCESS_OWNER_STATE_SET.has(count.ownerState) ? count.ownerState : "unknown";
|
|
4714
|
+
const current = normalized.get(ownerState);
|
|
4715
|
+
current.active += Math.max(0, count.activeCount);
|
|
4716
|
+
current.terminal += Math.max(0, count.terminalOwnerCount);
|
|
4717
|
+
}
|
|
4718
|
+
let terminalTotal = 0;
|
|
4719
|
+
for (const [ownerState, count] of normalized) {
|
|
4720
|
+
terminalTotal += count.terminal;
|
|
4721
|
+
observability.setGauge({
|
|
4722
|
+
name: "opengeni_retained_processes_active",
|
|
4723
|
+
help: "Current active retained provider processes by durable owner state.",
|
|
4724
|
+
labels: { owner_state: ownerState },
|
|
4725
|
+
value: count.active
|
|
4726
|
+
});
|
|
4727
|
+
observability.setGauge({
|
|
4728
|
+
name: "opengeni_retained_processes_terminal_owner_backlog",
|
|
4729
|
+
help: "Current active retained processes whose exact owner attempt is terminal.",
|
|
4730
|
+
labels: { owner_state: ownerState },
|
|
4731
|
+
value: count.terminal
|
|
4732
|
+
});
|
|
4733
|
+
}
|
|
4734
|
+
const previous = retainedProcessTerminalBacklogTotals.get(observability);
|
|
4735
|
+
if (previous !== void 0 && terminalTotal > previous) {
|
|
4736
|
+
observability.incrementCounter({
|
|
4737
|
+
name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
|
|
4738
|
+
help: "Positive growth in terminal-owner retained-process backlog between app samples.",
|
|
4739
|
+
amount: terminalTotal - previous
|
|
4740
|
+
});
|
|
4741
|
+
}
|
|
4742
|
+
retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
|
|
4743
|
+
}
|
|
4744
|
+
var EXPIRED_DRAINING_BACKENDS = [
|
|
4745
|
+
"none",
|
|
4746
|
+
"local",
|
|
4747
|
+
"docker",
|
|
4748
|
+
"modal",
|
|
4749
|
+
"e2b",
|
|
4750
|
+
"daytona",
|
|
4751
|
+
"blaxel",
|
|
4752
|
+
"runloop",
|
|
4753
|
+
"selfhosted",
|
|
4754
|
+
"unknown"
|
|
4755
|
+
];
|
|
4756
|
+
var EXPIRED_DRAINING_BACKEND_SET = new Set(EXPIRED_DRAINING_BACKENDS);
|
|
4757
|
+
var EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
|
|
4758
|
+
function recordExpiredDrainingSandboxLeaseGauges(observability, counts) {
|
|
4759
|
+
const normalized = /* @__PURE__ */ new Map();
|
|
4760
|
+
for (const backend of EXPIRED_DRAINING_BACKENDS) {
|
|
4761
|
+
for (const ageBucket of EXPIRED_DRAINING_AGE_BUCKETS) {
|
|
4762
|
+
normalized.set(`${backend}:${ageBucket}`, 0);
|
|
4763
|
+
}
|
|
4764
|
+
}
|
|
4765
|
+
for (const count of counts) {
|
|
4766
|
+
const backend = EXPIRED_DRAINING_BACKEND_SET.has(count.backend) ? count.backend : "unknown";
|
|
4767
|
+
if (!EXPIRED_DRAINING_AGE_BUCKETS.includes(count.ageBucket)) continue;
|
|
4768
|
+
const key = `${backend}:${count.ageBucket}`;
|
|
4769
|
+
normalized.set(key, (normalized.get(key) ?? 0) + Math.max(0, count.count));
|
|
4770
|
+
}
|
|
4771
|
+
for (const [key, value] of normalized) {
|
|
4772
|
+
const [backend, ageBucket] = key.split(":");
|
|
4773
|
+
observability.setGauge({
|
|
4774
|
+
name: "opengeni_sandbox_leases_expired_draining",
|
|
4775
|
+
help: "Current expired draining sandbox leases by backend and fixed age bucket.",
|
|
4776
|
+
labels: { backend, age_bucket: ageBucket },
|
|
4777
|
+
value
|
|
4778
|
+
});
|
|
4779
|
+
}
|
|
4780
|
+
}
|
|
4781
|
+
function recordRetainedProcessReconciliation(observability, outcome) {
|
|
4782
|
+
observability.incrementCounter({
|
|
4783
|
+
name: "opengeni_retained_process_reconciliation_total",
|
|
4784
|
+
help: "Bounded retained-process reconciliation observations by fixed app outcome.",
|
|
4785
|
+
labels: { outcome }
|
|
4786
|
+
});
|
|
4787
|
+
}
|
|
4737
4788
|
function recordCreditMicros(observability, kind, amountMicros) {
|
|
4738
4789
|
if (!observability || amountMicros <= 0) {
|
|
4739
4790
|
return;
|
|
@@ -5952,6 +6003,56 @@ function turnFinalizerCancellationSignal(temporalSignal, activityStatus) {
|
|
|
5952
6003
|
function compactionFailureReason(reason) {
|
|
5953
6004
|
return reason.startsWith("compaction summarization failed:") ? reason : `compaction summarization failed: ${reason}`;
|
|
5954
6005
|
}
|
|
6006
|
+
function safeErrorDiagnostic(error, redactText = redactSensitiveText) {
|
|
6007
|
+
const rawName = error instanceof Error ? error.name : "Error";
|
|
6008
|
+
const name = /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error";
|
|
6009
|
+
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
6010
|
+
const diagnostic = {
|
|
6011
|
+
name,
|
|
6012
|
+
message: redactText(rawMessage).slice(0, 2048)
|
|
6013
|
+
};
|
|
6014
|
+
if (error && typeof error === "object") {
|
|
6015
|
+
const status = Number(
|
|
6016
|
+
error.status ?? error.statusCode
|
|
6017
|
+
);
|
|
6018
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
6019
|
+
diagnostic.status = status;
|
|
6020
|
+
}
|
|
6021
|
+
const rawCode = error.code;
|
|
6022
|
+
if (typeof rawCode === "string" || typeof rawCode === "number") {
|
|
6023
|
+
const code = redactText(String(rawCode)).slice(0, 80);
|
|
6024
|
+
if (/^[A-Za-z0-9_.:-]+$/.test(code)) diagnostic.code = code;
|
|
6025
|
+
}
|
|
6026
|
+
}
|
|
6027
|
+
return diagnostic;
|
|
6028
|
+
}
|
|
6029
|
+
function safeErrorForTelemetry(error, redactText) {
|
|
6030
|
+
const diagnostic = safeErrorDiagnostic(error, redactText);
|
|
6031
|
+
const safe = new Error(diagnostic.message);
|
|
6032
|
+
safe.name = diagnostic.name;
|
|
6033
|
+
return safe;
|
|
6034
|
+
}
|
|
6035
|
+
function headerSecretRedactions(prefix, headers, credentialHeaderNames) {
|
|
6036
|
+
const discovered = [];
|
|
6037
|
+
const explicitCredentialHeaders = credentialHeaderNames ? new Set([...credentialHeaderNames].map((name) => name.toLowerCase())) : null;
|
|
6038
|
+
for (const [headerName, value] of Object.entries(headers ?? {})) {
|
|
6039
|
+
if (!explicitCredentialHeaders?.has(headerName.toLowerCase()) && !isCredentialHeaderName(headerName)) {
|
|
6040
|
+
continue;
|
|
6041
|
+
}
|
|
6042
|
+
const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
|
|
6043
|
+
discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
|
|
6044
|
+
if (/^(?:proxy-)?authorization$/i.test(headerName)) {
|
|
6045
|
+
const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
|
|
6046
|
+
if (credential) {
|
|
6047
|
+
discovered.push({
|
|
6048
|
+
name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
|
|
6049
|
+
value: credential
|
|
6050
|
+
});
|
|
6051
|
+
}
|
|
6052
|
+
}
|
|
6053
|
+
}
|
|
6054
|
+
return discovered;
|
|
6055
|
+
}
|
|
5955
6056
|
function compactionFailureReasonFromError(error) {
|
|
5956
6057
|
if (error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError) {
|
|
5957
6058
|
return compactionFailureReason(error.message);
|
|
@@ -6264,10 +6365,8 @@ function applyCodexCacheTelemetry(observability, provider, normalizedUsage) {
|
|
|
6264
6365
|
promptTokens: normalizedUsage.telemetry.inputTokens
|
|
6265
6366
|
});
|
|
6266
6367
|
}
|
|
6267
|
-
function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens) {
|
|
6268
|
-
const sanitized = sanitizeHistoryItemsForModel2(rawHistory, toolOutputTruncationTokens).filter(
|
|
6269
|
-
(item) => !isEphemeralInternalContext(item)
|
|
6270
|
-
);
|
|
6368
|
+
function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens, redactValue = (value) => value) {
|
|
6369
|
+
const sanitized = sanitizeHistoryItemsForModel2(rawHistory, toolOutputTruncationTokens).filter((item) => !isEphemeralInternalContext(item)).map((item) => redactValue(item));
|
|
6271
6370
|
if (sanitized.length <= persistedHistoryCount) {
|
|
6272
6371
|
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
6273
6372
|
}
|
|
@@ -6281,6 +6380,12 @@ function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = p
|
|
|
6281
6380
|
nextPosition: nextPosition + rows.length
|
|
6282
6381
|
};
|
|
6283
6382
|
}
|
|
6383
|
+
function secretRedactionModelInputFilter(redactValue) {
|
|
6384
|
+
return ({ modelData }) => ({
|
|
6385
|
+
...modelData,
|
|
6386
|
+
input: redactValue(modelData.input)
|
|
6387
|
+
});
|
|
6388
|
+
}
|
|
6284
6389
|
function isModelOrToolProgressHistoryItem(item) {
|
|
6285
6390
|
if (item.type === "message") {
|
|
6286
6391
|
return item.role === "assistant";
|
|
@@ -6312,11 +6417,18 @@ async function resolveActiveSandboxBackend(routingOn, loadPointer, loadSandboxKi
|
|
|
6312
6417
|
} catch (error) {
|
|
6313
6418
|
console.error(
|
|
6314
6419
|
"active sandbox backend resolution failed (turn proceeds on home backend)",
|
|
6315
|
-
error
|
|
6420
|
+
safeErrorDiagnostic(error)
|
|
6316
6421
|
);
|
|
6317
6422
|
return void 0;
|
|
6318
6423
|
}
|
|
6319
6424
|
}
|
|
6425
|
+
function managedSandboxOwnershipForTurn(machinePrimary, attemptId, sandboxGroupId) {
|
|
6426
|
+
if (machinePrimary) return null;
|
|
6427
|
+
return {
|
|
6428
|
+
holderId: sandboxLeaseHolderIdForAttempt(attemptId),
|
|
6429
|
+
sandboxGroupId
|
|
6430
|
+
};
|
|
6431
|
+
}
|
|
6320
6432
|
function pointerReconcileReason(record) {
|
|
6321
6433
|
if (!record) {
|
|
6322
6434
|
return "stale_pointer";
|
|
@@ -6406,6 +6518,10 @@ function computerToolModeForTurn(resolvedModel) {
|
|
|
6406
6518
|
}
|
|
6407
6519
|
return "hosted";
|
|
6408
6520
|
}
|
|
6521
|
+
function hostedWebSearchForTurn(resolvedModel, deploymentWebSearchEnabled, defaultNativeToolsAllowed) {
|
|
6522
|
+
if (!defaultNativeToolsAllowed) return false;
|
|
6523
|
+
return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
|
|
6524
|
+
}
|
|
6409
6525
|
function modelAcceptsTypedAttachmentContentForTurn(resolvedModel) {
|
|
6410
6526
|
return resolvedModel === null || resolvedModel.provider.api === "responses";
|
|
6411
6527
|
}
|
|
@@ -6464,6 +6580,9 @@ function isLazySandboxProvisionRetryable(error) {
|
|
|
6464
6580
|
if (error instanceof SandboxImageConflictError) {
|
|
6465
6581
|
return false;
|
|
6466
6582
|
}
|
|
6583
|
+
if (error instanceof WorkspaceArchiveIntegrityError2) {
|
|
6584
|
+
return error.retryable;
|
|
6585
|
+
}
|
|
6467
6586
|
if (error instanceof SandboxLeaseSupersededError2 || error instanceof SandboxWarmingTimeoutError) {
|
|
6468
6587
|
return true;
|
|
6469
6588
|
}
|
|
@@ -6854,6 +6973,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
6854
6973
|
await current?.flush().catch(() => void 0);
|
|
6855
6974
|
};
|
|
6856
6975
|
let preparedTools = null;
|
|
6976
|
+
let defaultNativeToolsAllowed = false;
|
|
6857
6977
|
const toolCancellationFenceRef = {
|
|
6858
6978
|
current: null
|
|
6859
6979
|
};
|
|
@@ -6890,7 +7010,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
6890
7010
|
payload: { sandboxId: lostSandbox.instanceId }
|
|
6891
7011
|
}
|
|
6892
7012
|
]).catch((publishError) => {
|
|
6893
|
-
console.error("sandbox box lost event publish failed", publishError);
|
|
7013
|
+
console.error("sandbox box lost event publish failed", safeErrorDiagnostic(publishError));
|
|
6894
7014
|
});
|
|
6895
7015
|
};
|
|
6896
7016
|
const startLeaseHeartbeat = (sandbox, warmBackend) => {
|
|
@@ -7007,7 +7127,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7007
7127
|
}
|
|
7008
7128
|
console.error(
|
|
7009
7129
|
"computer-use recording start failed (action outcome unaffected)",
|
|
7010
|
-
recordingError
|
|
7130
|
+
safeErrorDiagnostic(recordingError)
|
|
7011
7131
|
);
|
|
7012
7132
|
}
|
|
7013
7133
|
})();
|
|
@@ -7027,7 +7147,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
7027
7147
|
rawHistory,
|
|
7028
7148
|
persistedHistoryCount,
|
|
7029
7149
|
nextHistoryPosition,
|
|
7030
|
-
modelRunSettings.modelToolOutputTruncationTokens
|
|
7150
|
+
modelRunSettings.modelToolOutputTruncationTokens,
|
|
7151
|
+
(value) => redact(value)
|
|
7031
7152
|
);
|
|
7032
7153
|
const hasModelOrToolProgress = rows.some(
|
|
7033
7154
|
(row) => isModelOrToolProgressHistoryItem(row.item)
|
|
@@ -7070,11 +7191,14 @@ function createRunAgentTurnActivity(services) {
|
|
|
7070
7191
|
});
|
|
7071
7192
|
}
|
|
7072
7193
|
} catch (persistError) {
|
|
7073
|
-
console.error(
|
|
7194
|
+
console.error(
|
|
7195
|
+
"session history dual-write failed (run unaffected)",
|
|
7196
|
+
safeErrorDiagnostic(persistError, (value) => String(redact(value)))
|
|
7197
|
+
);
|
|
7074
7198
|
if (options.requireDurable) throw persistError;
|
|
7075
7199
|
}
|
|
7076
7200
|
};
|
|
7077
|
-
let redact =
|
|
7201
|
+
let redact = createSecretRedactor([]);
|
|
7078
7202
|
const secretRedactions = /* @__PURE__ */ new Map();
|
|
7079
7203
|
const publishedRunCredentialNotices = /* @__PURE__ */ new Set();
|
|
7080
7204
|
const registerSecretRedactions = (secrets) => {
|
|
@@ -7271,6 +7395,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7271
7395
|
producerId,
|
|
7272
7396
|
producerSeq: ++producerSeq
|
|
7273
7397
|
}));
|
|
7398
|
+
const redactedRunState = inputSettlement.runState ? redact(inputSettlement.runState) : void 0;
|
|
7274
7399
|
const result = await applySessionTurnSettlement(db, input.workspaceId, {
|
|
7275
7400
|
sessionId: input.sessionId,
|
|
7276
7401
|
turnId,
|
|
@@ -7280,7 +7405,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7280
7405
|
sessionStatus: inputSettlement.sessionStatus,
|
|
7281
7406
|
activeTurnId: inputSettlement.activeTurnId,
|
|
7282
7407
|
events: inputs,
|
|
7283
|
-
...
|
|
7408
|
+
...redactedRunState ? { runState: redactedRunState } : {},
|
|
7284
7409
|
...recordingMutation ? { recording: recordingMutation } : {},
|
|
7285
7410
|
...compactionRequestFailure ? { compactionRequestFailure } : {}
|
|
7286
7411
|
});
|
|
@@ -7634,43 +7759,42 @@ function createRunAgentTurnActivity(services) {
|
|
|
7634
7759
|
const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(
|
|
7635
7760
|
() => null
|
|
7636
7761
|
);
|
|
7637
|
-
|
|
7638
|
-
|
|
7639
|
-
|
|
7640
|
-
|
|
7641
|
-
|
|
7642
|
-
|
|
7643
|
-
|
|
7644
|
-
|
|
7645
|
-
|
|
7646
|
-
|
|
7647
|
-
|
|
7648
|
-
|
|
7649
|
-
|
|
7650
|
-
|
|
7651
|
-
|
|
7652
|
-
|
|
7762
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
7763
|
+
const armed = await armCodexCapacityWait(db, {
|
|
7764
|
+
accountId: input.accountId,
|
|
7765
|
+
workspaceId: input.workspaceId,
|
|
7766
|
+
sessionId: input.sessionId,
|
|
7767
|
+
turnId,
|
|
7768
|
+
attemptId: input.attemptId,
|
|
7769
|
+
workflowId: input.workflowId,
|
|
7770
|
+
goalId: activeGoal?.id ?? null,
|
|
7771
|
+
goalVersion: activeGoal?.version ?? null,
|
|
7772
|
+
earliestResetAt: null,
|
|
7773
|
+
resetKind: "bounded_refresh",
|
|
7774
|
+
failurePayload: {
|
|
7775
|
+
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
7776
|
+
code: "codex_allocator_disabled",
|
|
7777
|
+
detail: "waiting for a credential to be re-enabled, reconnected, or added"
|
|
7778
|
+
}
|
|
7779
|
+
});
|
|
7780
|
+
if (armed.action === "waiting") {
|
|
7781
|
+
await publishDurableSessionEvents(
|
|
7782
|
+
bus,
|
|
7783
|
+
input.workspaceId,
|
|
7784
|
+
input.sessionId,
|
|
7785
|
+
armed.events
|
|
7786
|
+
);
|
|
7787
|
+
turnMetricOutcome = "recovering";
|
|
7788
|
+
activityStatus = "waiting_capacity";
|
|
7789
|
+
return claimedResult({
|
|
7790
|
+
status: "waiting_capacity",
|
|
7791
|
+
capacityWait: {
|
|
7792
|
+
waiterId: armed.waiter.id,
|
|
7793
|
+
generation: armed.waiter.generation,
|
|
7794
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
7795
|
+
wakeRevision: armed.waiter.wakeRevision
|
|
7653
7796
|
}
|
|
7654
7797
|
});
|
|
7655
|
-
if (armed.action === "waiting") {
|
|
7656
|
-
await publishDurableSessionEvents(
|
|
7657
|
-
bus,
|
|
7658
|
-
input.workspaceId,
|
|
7659
|
-
input.sessionId,
|
|
7660
|
-
armed.events
|
|
7661
|
-
);
|
|
7662
|
-
turnMetricOutcome = "failed";
|
|
7663
|
-
activityStatus = "idle";
|
|
7664
|
-
return claimedResult({
|
|
7665
|
-
status: "idle",
|
|
7666
|
-
capacityWait: {
|
|
7667
|
-
waiterId: armed.waiter.id,
|
|
7668
|
-
generation: armed.waiter.generation,
|
|
7669
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
7670
|
-
wakeRevision: armed.waiter.wakeRevision
|
|
7671
|
-
}
|
|
7672
|
-
});
|
|
7673
|
-
}
|
|
7674
7798
|
}
|
|
7675
7799
|
if (!await settle({
|
|
7676
7800
|
events: [
|
|
@@ -7736,43 +7860,41 @@ function createRunAgentTurnActivity(services) {
|
|
|
7736
7860
|
"all connected Codex subscriptions are rate-limited",
|
|
7737
7861
|
{ allAccounts: true }
|
|
7738
7862
|
);
|
|
7739
|
-
|
|
7740
|
-
|
|
7741
|
-
|
|
7742
|
-
|
|
7863
|
+
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
7864
|
+
leased.accounts,
|
|
7865
|
+
/* @__PURE__ */ new Date()
|
|
7866
|
+
);
|
|
7867
|
+
const armed = await armCodexCapacityWait(db, {
|
|
7868
|
+
accountId: input.accountId,
|
|
7869
|
+
workspaceId: input.workspaceId,
|
|
7870
|
+
sessionId: input.sessionId,
|
|
7871
|
+
turnId,
|
|
7872
|
+
attemptId: input.attemptId,
|
|
7873
|
+
workflowId: input.workflowId,
|
|
7874
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
7875
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
7876
|
+
earliestResetAt: authoritativeResetAt,
|
|
7877
|
+
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
7878
|
+
failurePayload
|
|
7879
|
+
});
|
|
7880
|
+
if (armed.action === "waiting") {
|
|
7881
|
+
await publishDurableSessionEvents(
|
|
7882
|
+
bus,
|
|
7883
|
+
input.workspaceId,
|
|
7884
|
+
input.sessionId,
|
|
7885
|
+
armed.events
|
|
7743
7886
|
);
|
|
7744
|
-
|
|
7745
|
-
|
|
7746
|
-
|
|
7747
|
-
|
|
7748
|
-
|
|
7749
|
-
|
|
7750
|
-
|
|
7751
|
-
|
|
7752
|
-
|
|
7753
|
-
|
|
7754
|
-
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
7755
|
-
failurePayload
|
|
7887
|
+
turnMetricOutcome = "recovering";
|
|
7888
|
+
activityStatus = "waiting_capacity";
|
|
7889
|
+
return claimedResult({
|
|
7890
|
+
status: "waiting_capacity",
|
|
7891
|
+
capacityWait: {
|
|
7892
|
+
waiterId: armed.waiter.id,
|
|
7893
|
+
generation: armed.waiter.generation,
|
|
7894
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
7895
|
+
wakeRevision: armed.waiter.wakeRevision
|
|
7896
|
+
}
|
|
7756
7897
|
});
|
|
7757
|
-
if (armed.action === "waiting") {
|
|
7758
|
-
await publishDurableSessionEvents(
|
|
7759
|
-
bus,
|
|
7760
|
-
input.workspaceId,
|
|
7761
|
-
input.sessionId,
|
|
7762
|
-
armed.events
|
|
7763
|
-
);
|
|
7764
|
-
turnMetricOutcome = "failed";
|
|
7765
|
-
activityStatus = "idle";
|
|
7766
|
-
return claimedResult({
|
|
7767
|
-
status: "idle",
|
|
7768
|
-
capacityWait: {
|
|
7769
|
-
waiterId: armed.waiter.id,
|
|
7770
|
-
generation: armed.waiter.generation,
|
|
7771
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
7772
|
-
wakeRevision: armed.waiter.wakeRevision
|
|
7773
|
-
}
|
|
7774
|
-
});
|
|
7775
|
-
}
|
|
7776
7898
|
}
|
|
7777
7899
|
if (!await settle({
|
|
7778
7900
|
events: [
|
|
@@ -7881,17 +8003,57 @@ function createRunAgentTurnActivity(services) {
|
|
|
7881
8003
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
7882
8004
|
sandboxBackend: turn.sandboxBackend
|
|
7883
8005
|
};
|
|
8006
|
+
const sessionMcpHeaderNames = /* @__PURE__ */ new Map();
|
|
7884
8007
|
const runSettings = await settingsWithSessionMcpServersForRun(
|
|
7885
8008
|
db,
|
|
7886
8009
|
input.workspaceId,
|
|
7887
8010
|
input.sessionId,
|
|
7888
8011
|
input.attemptId,
|
|
7889
|
-
baseRunSettings
|
|
8012
|
+
baseRunSettings,
|
|
8013
|
+
{
|
|
8014
|
+
onResolvedServers: (servers) => {
|
|
8015
|
+
for (const server of servers) {
|
|
8016
|
+
sessionMcpHeaderNames.set(server.id, server.headerNames);
|
|
8017
|
+
}
|
|
8018
|
+
}
|
|
8019
|
+
}
|
|
7890
8020
|
);
|
|
8021
|
+
registerSecretRedactions([
|
|
8022
|
+
...runSettings.accessKey ? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }] : [],
|
|
8023
|
+
...runSettings.mcpServers.flatMap((server) => {
|
|
8024
|
+
return headerSecretRedactions(
|
|
8025
|
+
`MCP_${server.id.toUpperCase()}_STATIC`,
|
|
8026
|
+
server.headers,
|
|
8027
|
+
sessionMcpHeaderNames.get(server.id)
|
|
8028
|
+
);
|
|
8029
|
+
})
|
|
8030
|
+
]);
|
|
7891
8031
|
const resolvedModel = runtime.resolveTurnModel(
|
|
7892
8032
|
capabilitySettings,
|
|
7893
8033
|
turnExecutionPolicy.productModelId
|
|
7894
8034
|
);
|
|
8035
|
+
const selectedProvider = resolvedModel?.provider;
|
|
8036
|
+
const publicProviderHeaders = new Set(
|
|
8037
|
+
selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? []
|
|
8038
|
+
);
|
|
8039
|
+
const publicProviderQueries = new Set(
|
|
8040
|
+
selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? []
|
|
8041
|
+
);
|
|
8042
|
+
const fallbackProviderApiKey = capabilitySettings.openaiProvider === "azure" ? capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken : capabilitySettings.openaiApiKey;
|
|
8043
|
+
registerSecretRedactions([
|
|
8044
|
+
...selectedProvider?.apiKey ? [
|
|
8045
|
+
{
|
|
8046
|
+
name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
|
|
8047
|
+
value: selectedProvider.apiKey
|
|
8048
|
+
}
|
|
8049
|
+
] : !resolvedModel && fallbackProviderApiKey ? [{ name: "MODEL_PROVIDER_API_KEY", value: fallbackProviderApiKey }] : [],
|
|
8050
|
+
...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(
|
|
8051
|
+
([name, value]) => publicProviderHeaders.has(name.toLowerCase()) ? [] : [{ name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`, value }]
|
|
8052
|
+
),
|
|
8053
|
+
...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(
|
|
8054
|
+
([name, value]) => publicProviderQueries.has(name.toLowerCase()) ? [] : [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }]
|
|
8055
|
+
)
|
|
8056
|
+
]);
|
|
7895
8057
|
modelRunSettings = resolvedModel ? settingsWithResolvedModelContext(runSettings, resolvedModel.configured) : runSettings;
|
|
7896
8058
|
{
|
|
7897
8059
|
const workspaceModelPolicy = await getWorkspaceModelPolicy2(db, input.workspaceId);
|
|
@@ -7917,6 +8079,13 @@ function createRunAgentTurnActivity(services) {
|
|
|
7917
8079
|
input.workspaceId,
|
|
7918
8080
|
effectiveCodexCredentialId ?? ""
|
|
7919
8081
|
);
|
|
8082
|
+
const registerSnapshot = async (snapshotPromise) => {
|
|
8083
|
+
const snapshot = await snapshotPromise;
|
|
8084
|
+
registerSecretRedactions([
|
|
8085
|
+
{ name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken }
|
|
8086
|
+
]);
|
|
8087
|
+
return snapshot;
|
|
8088
|
+
};
|
|
7920
8089
|
return {
|
|
7921
8090
|
clientVersion: CODEX_CLIENT_VERSION,
|
|
7922
8091
|
// Backend sticky cache-routing key — the SAME id as the body's
|
|
@@ -7926,11 +8095,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
7926
8095
|
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
7927
8096
|
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
7928
8097
|
sessionId: input.sessionId,
|
|
7929
|
-
getToken: resolver.getToken,
|
|
7930
|
-
refresh: resolver.refresh,
|
|
8098
|
+
getToken: () => registerSnapshot(resolver.getToken()),
|
|
8099
|
+
refresh: () => registerSnapshot(resolver.refresh()),
|
|
7931
8100
|
resolveModel: buildModelResolver(
|
|
7932
|
-
|
|
7933
|
-
|
|
8101
|
+
CODEX_FALLBACK_MODEL_SLUGS,
|
|
8102
|
+
CODEX_FALLBACK_MODEL_SLUGS[0]
|
|
7934
8103
|
),
|
|
7935
8104
|
onUsageHeaders: (snapshot) => {
|
|
7936
8105
|
latestCodexUsage = snapshot;
|
|
@@ -8057,7 +8226,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8057
8226
|
} catch (error) {
|
|
8058
8227
|
if (shouldRecoverCompactionProviderFailure(error)) throw error;
|
|
8059
8228
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
8060
|
-
const errorMessage = compactionFailureReasonFromError(error);
|
|
8229
|
+
const errorMessage = String(redact(compactionFailureReasonFromError(error)));
|
|
8061
8230
|
if (!await settle({
|
|
8062
8231
|
events: [
|
|
8063
8232
|
{
|
|
@@ -8121,7 +8290,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8121
8290
|
return claimedResult({ status: "idle" });
|
|
8122
8291
|
}
|
|
8123
8292
|
const turnResources = mergeResourceRefs(session.resources, turn.resources);
|
|
8124
|
-
const
|
|
8293
|
+
const resolvedToolPolicy = resolveSessionToolPolicy({
|
|
8125
8294
|
...session.toolPolicy ? { toolPolicy: session.toolPolicy } : {},
|
|
8126
8295
|
sessionTools: session.tools,
|
|
8127
8296
|
turnTools: turn.tools,
|
|
@@ -8130,7 +8299,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
8130
8299
|
defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
|
|
8131
8300
|
(tool) => tool.id
|
|
8132
8301
|
)
|
|
8133
|
-
})
|
|
8302
|
+
});
|
|
8303
|
+
const effectivePolicyTools = resolvedToolPolicy.toolRefs;
|
|
8304
|
+
defaultNativeToolsAllowed = sessionToolPolicyAllowsDefaultNativeTools(
|
|
8305
|
+
resolvedToolPolicy.effectivePolicy
|
|
8306
|
+
);
|
|
8134
8307
|
const turnTools = withCodexAppsTool(
|
|
8135
8308
|
runSettings,
|
|
8136
8309
|
withFirstPartyTools(runSettings, effectivePolicyTools)
|
|
@@ -8298,6 +8471,17 @@ function createRunAgentTurnActivity(services) {
|
|
|
8298
8471
|
cancellationSignal,
|
|
8299
8472
|
void 0
|
|
8300
8473
|
);
|
|
8474
|
+
registerSecretRedactions([
|
|
8475
|
+
...Object.entries(sandboxGitTokens ?? {}).flatMap(
|
|
8476
|
+
([provider, value]) => value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : []
|
|
8477
|
+
),
|
|
8478
|
+
...sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : [],
|
|
8479
|
+
...(sandboxGitCredentialBindings ?? []).map((binding) => ({
|
|
8480
|
+
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
8481
|
+
value: binding.token
|
|
8482
|
+
})),
|
|
8483
|
+
...sandboxToolspaceToken ? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }] : []
|
|
8484
|
+
]);
|
|
8301
8485
|
const sandboxToolspaceTokenFile = sandboxToolspaceToken ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId) : void 0;
|
|
8302
8486
|
const initialGitCredentials = sandboxGitCredentialBindings ? {
|
|
8303
8487
|
bindings: sandboxGitCredentialBindings,
|
|
@@ -8330,6 +8514,12 @@ function createRunAgentTurnActivity(services) {
|
|
|
8330
8514
|
);
|
|
8331
8515
|
if (binding) {
|
|
8332
8516
|
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
8517
|
+
registerSecretRedactions([
|
|
8518
|
+
{
|
|
8519
|
+
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
8520
|
+
value: binding.token
|
|
8521
|
+
}
|
|
8522
|
+
]);
|
|
8333
8523
|
}
|
|
8334
8524
|
pendingBinding = binding;
|
|
8335
8525
|
return binding ? {
|
|
@@ -8399,7 +8589,17 @@ function createRunAgentTurnActivity(services) {
|
|
|
8399
8589
|
toolspaceTokenRenewal = null;
|
|
8400
8590
|
await previous?.stop();
|
|
8401
8591
|
if (toolspaceTokenRenewalClosed) return;
|
|
8402
|
-
const mint = async () =>
|
|
8592
|
+
const mint = async () => {
|
|
8593
|
+
const material = await mintSandboxToolspaceToken(
|
|
8594
|
+
runSettings,
|
|
8595
|
+
connectionScope,
|
|
8596
|
+
toolspaceAuthority
|
|
8597
|
+
);
|
|
8598
|
+
if (material) {
|
|
8599
|
+
registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
|
|
8600
|
+
}
|
|
8601
|
+
return material;
|
|
8602
|
+
};
|
|
8403
8603
|
const write = async (material) => {
|
|
8404
8604
|
const runAs = sandboxRunAs(runSettings);
|
|
8405
8605
|
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
@@ -8571,8 +8771,13 @@ function createRunAgentTurnActivity(services) {
|
|
|
8571
8771
|
runCredentialRenewal = controller;
|
|
8572
8772
|
};
|
|
8573
8773
|
if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
|
|
8574
|
-
|
|
8575
|
-
|
|
8774
|
+
const managedOwnership = managedSandboxOwnershipForTurn(
|
|
8775
|
+
machinePrimary,
|
|
8776
|
+
input.attemptId,
|
|
8777
|
+
session.sandboxGroupId
|
|
8778
|
+
);
|
|
8779
|
+
sandboxHolderId = managedOwnership?.holderId ?? null;
|
|
8780
|
+
sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
|
|
8576
8781
|
if (machinePrimary) {
|
|
8577
8782
|
const machineOpStream = settings.agentOpStreamEnabled === true ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord.enrollmentId))?.opStream === true : false;
|
|
8578
8783
|
const established = await establishSelfhostedTurnSession(
|
|
@@ -8593,21 +8798,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
8593
8798
|
}
|
|
8594
8799
|
);
|
|
8595
8800
|
machinePrimarySession = established.session;
|
|
8596
|
-
const lease = await waitForTurnOperation(
|
|
8597
|
-
acquireSelfhostedLeaseForTurn(
|
|
8598
|
-
{ db, settings },
|
|
8599
|
-
{
|
|
8600
|
-
accountId: input.accountId,
|
|
8601
|
-
workspaceId: input.workspaceId,
|
|
8602
|
-
sandboxGroupId: session.sandboxGroupId,
|
|
8603
|
-
sessionId: input.sessionId
|
|
8604
|
-
},
|
|
8605
|
-
"turn",
|
|
8606
|
-
sandboxHolderId
|
|
8607
|
-
),
|
|
8608
|
-
cancellationSignal,
|
|
8609
|
-
async (lateLease) => await lateLease.release()
|
|
8610
|
-
);
|
|
8611
8801
|
setupBoxSession = established.session;
|
|
8612
8802
|
resolvedSandbox = {
|
|
8613
8803
|
// Wrap in the SAME routing proxy so a mid-turn swap (to another machine
|
|
@@ -8649,8 +8839,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
8649
8839
|
},
|
|
8650
8840
|
established
|
|
8651
8841
|
),
|
|
8652
|
-
|
|
8653
|
-
|
|
8842
|
+
// For a machine route this is the active-pointer epoch, not a cloud
|
|
8843
|
+
// lease epoch. Cloud-only heartbeat/snapshot/capture paths require
|
|
8844
|
+
// sandboxGroupId and remain disabled for this branch.
|
|
8845
|
+
leaseEpoch: activeSandboxPointer.activeEpoch,
|
|
8846
|
+
release: async () => void 0
|
|
8654
8847
|
};
|
|
8655
8848
|
} else if (establishPolicy === "on-demand") {
|
|
8656
8849
|
} else {
|
|
@@ -8679,7 +8872,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8679
8872
|
// running a DIFFERENT image (solo → recreate on the new image; N-holders →
|
|
8680
8873
|
// SandboxImageConflictError surfaced as an actionable turn error). Prefer the
|
|
8681
8874
|
// explicit Modal image ref, else the docker image. The selfhosted branch
|
|
8682
|
-
// (establishSelfhostedTurnSession
|
|
8875
|
+
// (establishSelfhostedTurnSession) NEVER passes
|
|
8683
8876
|
// an image — B3 lives only on this Modal else-branch.
|
|
8684
8877
|
...runSettings.modalImageRef ?? runSettings.dockerImage ? {
|
|
8685
8878
|
image: runSettings.modalImageRef ?? runSettings.dockerImage
|
|
@@ -8691,7 +8884,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8691
8884
|
...rigVersion ? { rigVersionId: rigVersion.id } : {}
|
|
8692
8885
|
},
|
|
8693
8886
|
"turn",
|
|
8694
|
-
|
|
8887
|
+
managedOwnership.holderId
|
|
8695
8888
|
),
|
|
8696
8889
|
cancellationSignal,
|
|
8697
8890
|
async (lateSandbox) => await lateSandbox.release()
|
|
@@ -8752,7 +8945,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8752
8945
|
throwIfWorkerShuttingDown();
|
|
8753
8946
|
throwIfTurnCancelled();
|
|
8754
8947
|
const mcpCredentialRootSessionId = connectionCredentials?.mcpCredentials && hostCredentialRootSessionId ? hostCredentialRootSessionId : input.sessionId;
|
|
8755
|
-
const
|
|
8948
|
+
const rawResolveCredential = connectionTokenResolverForTurn({
|
|
8756
8949
|
db,
|
|
8757
8950
|
settings: runSettings,
|
|
8758
8951
|
connectionCredentials: connectionCredentials ?? null,
|
|
@@ -8763,6 +8956,15 @@ function createRunAgentTurnActivity(services) {
|
|
|
8763
8956
|
attemptId: input.attemptId,
|
|
8764
8957
|
turn
|
|
8765
8958
|
});
|
|
8959
|
+
const resolveCredential = async (request) => {
|
|
8960
|
+
const result = await rawResolveCredential(request);
|
|
8961
|
+
if (result.status === "ok") {
|
|
8962
|
+
registerSecretRedactions(
|
|
8963
|
+
headerSecretRedactions("MCP", result.headers, Object.keys(result.headers))
|
|
8964
|
+
);
|
|
8965
|
+
}
|
|
8966
|
+
return result;
|
|
8967
|
+
};
|
|
8766
8968
|
preparedTools = await waitForTurnOperation(
|
|
8767
8969
|
withCodex(
|
|
8768
8970
|
() => runtime.prepareTools(runSettings, turnTools, {
|
|
@@ -8797,6 +8999,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
8797
8999
|
variableNames: Object.keys(workspaceVariableSet.values)
|
|
8798
9000
|
}
|
|
8799
9001
|
} : {};
|
|
9002
|
+
const hostedWebSearch = hostedWebSearchForTurn(
|
|
9003
|
+
resolvedModel,
|
|
9004
|
+
runSettings.webSearchEnabled,
|
|
9005
|
+
defaultNativeToolsAllowed
|
|
9006
|
+
);
|
|
8800
9007
|
const agent = runtime.buildAgent(modelRunSettings, turnResources, {
|
|
8801
9008
|
reasoningEffort: turn.reasoningEffort,
|
|
8802
9009
|
...humanInputResume ? { humanInputResponse: humanInputResume } : {},
|
|
@@ -8848,10 +9055,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
8848
9055
|
// store/compaction follow the provider's compaction mode (registry
|
|
8849
9056
|
// providers resolve to "client"); encrypted reasoning is only
|
|
8850
9057
|
// round-tripped on the Responses wire API; hosted web search is attached
|
|
8851
|
-
// only when the model opts in
|
|
8852
|
-
//
|
|
9058
|
+
// only when BOTH the model opts in and this immutable turn still tracks
|
|
9059
|
+
// workspace defaults (an explicit session/turn remains narrowed); the
|
|
9060
|
+
// effective context window drives the compaction threshold.
|
|
9061
|
+
hostedWebSearch,
|
|
8853
9062
|
...resolvedModel ? {
|
|
8854
|
-
hostedWebSearch: resolvedModel.configured.hostedWebSearch,
|
|
8855
9063
|
encryptedReasoning: resolvedModel.provider.api === "responses" && runSettings.openaiReasoningEncryptedContent,
|
|
8856
9064
|
contextWindowTokens: resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
8857
9065
|
// The ChatGPT/Codex backend rejects the SDK's HOSTED sandbox tools —
|
|
@@ -9147,7 +9355,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9147
9355
|
} catch (compactError) {
|
|
9148
9356
|
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
9149
9357
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
9150
|
-
const errorMessage = compactionFailureReasonFromError(compactError);
|
|
9358
|
+
const errorMessage = String(redact(compactionFailureReasonFromError(compactError)));
|
|
9151
9359
|
observability.error("context compaction failed", {
|
|
9152
9360
|
sessionId: input.sessionId,
|
|
9153
9361
|
turnId,
|
|
@@ -9409,6 +9617,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9409
9617
|
} : {},
|
|
9410
9618
|
contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
|
|
9411
9619
|
contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
9620
|
+
callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
|
|
9412
9621
|
...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
|
|
9413
9622
|
});
|
|
9414
9623
|
};
|
|
@@ -9481,7 +9690,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9481
9690
|
} catch (limitError) {
|
|
9482
9691
|
let serializedRunState = null;
|
|
9483
9692
|
try {
|
|
9484
|
-
serializedRunState = stream.state.toString();
|
|
9693
|
+
serializedRunState = String(redact(stream.state.toString()));
|
|
9485
9694
|
} catch {
|
|
9486
9695
|
serializedRunState = null;
|
|
9487
9696
|
}
|
|
@@ -9501,7 +9710,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
9501
9710
|
executionGeneration,
|
|
9502
9711
|
attemptId: input.attemptId,
|
|
9503
9712
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
9504
|
-
|
|
9713
|
+
callId: pendingToolCall.callId,
|
|
9714
|
+
callType: pendingToolCall.callType,
|
|
9715
|
+
callItem: redact(pendingToolCall.callItem)
|
|
9505
9716
|
});
|
|
9506
9717
|
if (!registered.accepted) {
|
|
9507
9718
|
throw new TurnAttemptFencedError(
|
|
@@ -9521,7 +9732,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9521
9732
|
attemptId: input.attemptId,
|
|
9522
9733
|
callId: completedToolCall.callId,
|
|
9523
9734
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
9524
|
-
resultItem: completedToolCall.resultItem
|
|
9735
|
+
resultItem: redact(completedToolCall.resultItem)
|
|
9525
9736
|
});
|
|
9526
9737
|
if (!recorded.accepted) {
|
|
9527
9738
|
throw new TurnAttemptFencedError(
|
|
@@ -9713,7 +9924,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9713
9924
|
activityStatus = "requires_action";
|
|
9714
9925
|
return claimedResult({ status: "requires_action" });
|
|
9715
9926
|
}
|
|
9716
|
-
const finalOutput = String(stream.finalOutput ?? "");
|
|
9927
|
+
const finalOutput = String(redact(String(stream.finalOutput ?? "")));
|
|
9717
9928
|
await reconcileConversationTruth();
|
|
9718
9929
|
if (machinePrimarySession) {
|
|
9719
9930
|
try {
|
|
@@ -9778,8 +9989,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
9778
9989
|
sessionId: input.sessionId,
|
|
9779
9990
|
turnId: activeTurnId,
|
|
9780
9991
|
reason: recoveryKind,
|
|
9781
|
-
code: overflow?.code,
|
|
9782
|
-
error: overflow?.message ?? compactionNeeded?.message,
|
|
9992
|
+
code: overflow?.code ? String(redact(overflow.code)) : void 0,
|
|
9993
|
+
error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
|
|
9783
9994
|
signalTokens: compactionNeeded?.signalTokens,
|
|
9784
9995
|
thresholdTokens: compactionNeeded?.thresholdTokens
|
|
9785
9996
|
});
|
|
@@ -9803,7 +10014,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
9803
10014
|
} catch (compactError) {
|
|
9804
10015
|
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
9805
10016
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
9806
|
-
compactionFailureMessage =
|
|
10017
|
+
compactionFailureMessage = String(
|
|
10018
|
+
redact(compactionFailureReasonFromError(compactError))
|
|
10019
|
+
);
|
|
9807
10020
|
observability.warn("context compaction recovery compaction failed", {
|
|
9808
10021
|
sessionId: input.sessionId,
|
|
9809
10022
|
turnId: activeTurnId,
|
|
@@ -9878,7 +10091,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
9878
10091
|
turnMetricOutcome = "recovering";
|
|
9879
10092
|
return claimedResult({ status: "recovering" });
|
|
9880
10093
|
} catch (recoveryError) {
|
|
9881
|
-
console.error(
|
|
10094
|
+
console.error(
|
|
10095
|
+
"sandbox lease supersession recovery failed",
|
|
10096
|
+
safeErrorDiagnostic(recoveryError, (value) => String(redact(value)))
|
|
10097
|
+
);
|
|
9882
10098
|
throw recoveryError;
|
|
9883
10099
|
}
|
|
9884
10100
|
}
|
|
@@ -9910,7 +10126,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
9910
10126
|
turnMetricOutcome = "recovering";
|
|
9911
10127
|
return claimedResult({ status: "recovering" });
|
|
9912
10128
|
} catch (recoveryError) {
|
|
9913
|
-
console.error(
|
|
10129
|
+
console.error(
|
|
10130
|
+
"worker-shutdown recovery checkpoint failed",
|
|
10131
|
+
safeErrorDiagnostic(recoveryError, (value) => String(redact(value)))
|
|
10132
|
+
);
|
|
9914
10133
|
throw recoveryError;
|
|
9915
10134
|
}
|
|
9916
10135
|
}
|
|
@@ -10336,7 +10555,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10336
10555
|
usageLimit,
|
|
10337
10556
|
error instanceof Error ? error.message : String(error)
|
|
10338
10557
|
);
|
|
10339
|
-
if (
|
|
10558
|
+
if (rotationResumeMs === null) {
|
|
10340
10559
|
const providerResetAt = capacityAuthoritativeResetAt ?? (usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1e3) : null);
|
|
10341
10560
|
const armed = await armCodexCapacityWait(db, {
|
|
10342
10561
|
accountId: input.accountId,
|
|
@@ -10345,8 +10564,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
10345
10564
|
turnId,
|
|
10346
10565
|
attemptId: input.attemptId,
|
|
10347
10566
|
workflowId: input.workflowId,
|
|
10348
|
-
goalId: goal.id,
|
|
10349
|
-
goalVersion: goal.version,
|
|
10567
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
10568
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
10350
10569
|
earliestResetAt: providerResetAt,
|
|
10351
10570
|
resetKind: providerResetAt ? "authoritative" : "bounded_refresh",
|
|
10352
10571
|
failurePayload,
|
|
@@ -10365,11 +10584,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
10365
10584
|
input.sessionId,
|
|
10366
10585
|
armed.events
|
|
10367
10586
|
);
|
|
10368
|
-
turnMetricOutcome = "
|
|
10369
|
-
activityStatus = "
|
|
10587
|
+
turnMetricOutcome = "recovering";
|
|
10588
|
+
activityStatus = "waiting_capacity";
|
|
10370
10589
|
activityError = error;
|
|
10371
10590
|
return claimedResult({
|
|
10372
|
-
status: "
|
|
10591
|
+
status: "waiting_capacity",
|
|
10373
10592
|
capacityWait: {
|
|
10374
10593
|
waiterId: armed.waiter.id,
|
|
10375
10594
|
generation: armed.waiter.generation,
|
|
@@ -10460,7 +10679,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10460
10679
|
activityStatus = "idle";
|
|
10461
10680
|
return claimedResult({ status: "idle" });
|
|
10462
10681
|
}
|
|
10463
|
-
const failure = agentRunFailurePayload(error, { isCodexTurn });
|
|
10682
|
+
const failure = redact(agentRunFailurePayload(error, { isCodexTurn }));
|
|
10464
10683
|
if (isSessionEventPersistenceError(error)) {
|
|
10465
10684
|
observability.error("session event persistence failed", {
|
|
10466
10685
|
accountId: input.accountId,
|
|
@@ -10570,8 +10789,12 @@ function createRunAgentTurnActivity(services) {
|
|
|
10570
10789
|
);
|
|
10571
10790
|
}
|
|
10572
10791
|
await drainAttemptOwnedSandboxWriters({
|
|
10573
|
-
|
|
10574
|
-
|
|
10792
|
+
// Normal turn completion owns the same process boundary as
|
|
10793
|
+
// Pause/Steer: yielded provider shells must be terminated, polled,
|
|
10794
|
+
// and durably settled before workspace capture. Only receipt
|
|
10795
|
+
// publication remains conditional on acknowledgeQuiescence.
|
|
10796
|
+
toolCancellationFence,
|
|
10797
|
+
cancellationReason: cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
|
|
10575
10798
|
gitCredentialRenewals: gitRenewalsToStop,
|
|
10576
10799
|
toolspaceTokenRenewal: toolspaceRenewalToStop,
|
|
10577
10800
|
runCredentialRenewal: runRenewalToStop
|
|
@@ -10619,14 +10842,20 @@ function createRunAgentTurnActivity(services) {
|
|
|
10619
10842
|
});
|
|
10620
10843
|
},
|
|
10621
10844
|
onReceiptFailure: (error) => {
|
|
10622
|
-
console.error(
|
|
10845
|
+
console.error(
|
|
10846
|
+
"agent turn quiescence receipt exhausted; signalling proof",
|
|
10847
|
+
safeErrorDiagnostic(error, (value) => String(redact(value)))
|
|
10848
|
+
);
|
|
10623
10849
|
},
|
|
10624
10850
|
onPublishFailure: (error) => {
|
|
10625
|
-
console.error(
|
|
10851
|
+
console.error(
|
|
10852
|
+
"agent turn quiescence event fanout failed",
|
|
10853
|
+
safeErrorDiagnostic(error, (value) => String(redact(value)))
|
|
10854
|
+
);
|
|
10626
10855
|
},
|
|
10627
10856
|
onSignalFailure: (error, attempt, retryMs) => {
|
|
10628
10857
|
console.error("agent turn quiescence proof signal failed; retrying", {
|
|
10629
|
-
error,
|
|
10858
|
+
error: safeErrorDiagnostic(error, (value) => String(redact(value))),
|
|
10630
10859
|
attempt,
|
|
10631
10860
|
retryMs
|
|
10632
10861
|
});
|
|
@@ -10812,14 +11041,20 @@ function createRunAgentTurnActivity(services) {
|
|
|
10812
11041
|
resolvedSandbox = null;
|
|
10813
11042
|
await waitForTurnFinalizerStep(
|
|
10814
11043
|
sandboxToRelease.release().catch((releaseError) => {
|
|
10815
|
-
console.error(
|
|
11044
|
+
console.error(
|
|
11045
|
+
"sandbox lease release failed (turn outcome unaffected)",
|
|
11046
|
+
safeErrorDiagnostic(releaseError, (value) => String(redact(value)))
|
|
11047
|
+
);
|
|
10816
11048
|
}),
|
|
10817
11049
|
finalizerSignal
|
|
10818
11050
|
);
|
|
10819
11051
|
}
|
|
10820
11052
|
} catch (error) {
|
|
10821
11053
|
finalizationError ??= error;
|
|
10822
|
-
console.error(
|
|
11054
|
+
console.error(
|
|
11055
|
+
"agent turn finalization failed (turn outcome unaffected)",
|
|
11056
|
+
safeErrorDiagnostic(error, (value) => String(redact(value)))
|
|
11057
|
+
);
|
|
10823
11058
|
} finally {
|
|
10824
11059
|
cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
|
|
10825
11060
|
const completedAt = performance.now();
|
|
@@ -10868,7 +11103,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
10868
11103
|
"opengeni.duration_ms": Math.round(durationSeconds * 1e3),
|
|
10869
11104
|
"opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1e3)
|
|
10870
11105
|
},
|
|
10871
|
-
error: finalizationError
|
|
11106
|
+
error: finalizationError || activityError ? safeErrorForTelemetry(
|
|
11107
|
+
finalizationError ?? activityError,
|
|
11108
|
+
(value) => String(redact(value))
|
|
11109
|
+
) : void 0
|
|
10872
11110
|
});
|
|
10873
11111
|
assertPhysicalToolQuiescenceForCancellation({
|
|
10874
11112
|
acknowledgeQuiescence,
|
|
@@ -10907,6 +11145,10 @@ async function assertGitHubTokenMintSelectionAuthorized(db, workspaceId, install
|
|
|
10907
11145
|
);
|
|
10908
11146
|
}
|
|
10909
11147
|
}
|
|
11148
|
+
var STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE = "unable to connect. is the computer able to access the url?";
|
|
11149
|
+
function isExactStatuslessUpstreamConnectivityMessage(message) {
|
|
11150
|
+
return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
|
|
11151
|
+
}
|
|
10910
11152
|
function isTransientProviderError(error) {
|
|
10911
11153
|
const status = typeof error === "object" && error !== null && "status" in error ? Number(error.status) : void 0;
|
|
10912
11154
|
if (status !== void 0 && Number.isFinite(status)) {
|
|
@@ -10917,6 +11159,9 @@ function isTransientProviderError(error) {
|
|
|
10917
11159
|
return true;
|
|
10918
11160
|
}
|
|
10919
11161
|
const message = error instanceof Error ? error.message : String(error);
|
|
11162
|
+
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
11163
|
+
return true;
|
|
11164
|
+
}
|
|
10920
11165
|
return /overloaded|an error occurred while processing your request|connection error|service unavailable|bad gateway|gateway timeout/i.test(
|
|
10921
11166
|
message
|
|
10922
11167
|
);
|
|
@@ -10984,6 +11229,13 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
10984
11229
|
};
|
|
10985
11230
|
}
|
|
10986
11231
|
if (isTransientProviderError(error)) {
|
|
11232
|
+
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
11233
|
+
return {
|
|
11234
|
+
error: "OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
|
|
11235
|
+
code: "upstream_connectivity_unavailable",
|
|
11236
|
+
retryable: true
|
|
11237
|
+
};
|
|
11238
|
+
}
|
|
10987
11239
|
return { error: message, code: "provider_unavailable", retryable: true };
|
|
10988
11240
|
}
|
|
10989
11241
|
return { error: message };
|
|
@@ -11432,19 +11684,27 @@ import {
|
|
|
11432
11684
|
accrueWarmSeconds as accrueWarmSeconds2,
|
|
11433
11685
|
confirmDrainCold,
|
|
11434
11686
|
appendSessionEventToSandboxGroup,
|
|
11687
|
+
claimTerminalRetainedProcesses,
|
|
11688
|
+
countActiveRetainedProcessesByOwnerState,
|
|
11689
|
+
countExpiredDrainingSandboxLeases,
|
|
11435
11690
|
countQueuedTurns,
|
|
11436
11691
|
countSandboxLeasesByLiveness,
|
|
11692
|
+
deferRetainedProcessReconciliation,
|
|
11437
11693
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
11438
11694
|
getBillingBalance as getBillingBalance4,
|
|
11439
11695
|
listCreditBalancesByAccount,
|
|
11440
11696
|
listLiveModalSandboxLeaseAttributions,
|
|
11441
11697
|
listMeterableWarmLeases,
|
|
11442
11698
|
persistDrainSnapshot,
|
|
11699
|
+
recordRetainedProcessReconciliationProof,
|
|
11443
11700
|
readWorkspaceArchiveCapturePreflight as readWorkspaceArchiveCapturePreflight2,
|
|
11444
11701
|
readLease as readLease3,
|
|
11445
11702
|
reapExpiredSessionListSnapshots,
|
|
11446
11703
|
reapStaleLeaseHoldersGlobal,
|
|
11447
|
-
|
|
11704
|
+
retainedProcessReconciliationProof,
|
|
11705
|
+
retainedProcessSettlementIdentity as retainedProcessSettlementIdentity2,
|
|
11706
|
+
rlsContextForWorkspace,
|
|
11707
|
+
settleRetainedProcess as settleRetainedProcess2
|
|
11448
11708
|
} from "@opengeni/db";
|
|
11449
11709
|
import { sandboxWarmRateMicrosPerSecond as sandboxWarmRateMicrosPerSecond2 } from "@opengeni/config";
|
|
11450
11710
|
import {
|
|
@@ -11452,13 +11712,19 @@ import {
|
|
|
11452
11712
|
createSandboxClientForBackend,
|
|
11453
11713
|
deletePriorPersistedSnapshot as deletePriorPersistedSnapshot2,
|
|
11454
11714
|
deserializeSandboxSessionStateEnvelope,
|
|
11715
|
+
isExecSessionLostBanner,
|
|
11455
11716
|
isProviderSandboxNotFoundError as isProviderSandboxNotFoundError3,
|
|
11717
|
+
parseExecBannerExitCode,
|
|
11456
11718
|
sweepModalOrphanSandboxes,
|
|
11457
11719
|
terminateModalSandboxById
|
|
11458
11720
|
} from "@opengeni/runtime";
|
|
11721
|
+
var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
11722
|
+
var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
|
|
11723
|
+
var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
|
|
11459
11724
|
function createSandboxLeaseActivities(services, options = {}) {
|
|
11460
11725
|
const terminateBox = options.terminateBox ?? terminateProviderBox;
|
|
11461
11726
|
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
11727
|
+
const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
|
|
11462
11728
|
async function reapSandboxLeases2() {
|
|
11463
11729
|
const service = await services();
|
|
11464
11730
|
const { db, settings, observability } = service;
|
|
@@ -11502,6 +11768,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
11502
11768
|
metered.workspaceIds,
|
|
11503
11769
|
observability
|
|
11504
11770
|
);
|
|
11771
|
+
await reconcileTerminalRetainedProcesses(db, settings, observability, probeRetainedProcess);
|
|
11505
11772
|
const drainable = await reapStaleLeaseHoldersGlobal(db, {
|
|
11506
11773
|
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
11507
11774
|
// Dead-worker turn holders: a live holder is touched every 10s from the
|
|
@@ -11573,6 +11840,276 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
11573
11840
|
}
|
|
11574
11841
|
return { reapSandboxLeases: reapSandboxLeases2 };
|
|
11575
11842
|
}
|
|
11843
|
+
async function reconcileTerminalRetainedProcesses(db, settings, observability, probe) {
|
|
11844
|
+
const claimId = crypto.randomUUID();
|
|
11845
|
+
let claims;
|
|
11846
|
+
try {
|
|
11847
|
+
claims = await claimTerminalRetainedProcesses(db, {
|
|
11848
|
+
claimId,
|
|
11849
|
+
limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,
|
|
11850
|
+
claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS
|
|
11851
|
+
});
|
|
11852
|
+
} catch (error) {
|
|
11853
|
+
recordRetainedProcessReconciliation(observability, "claim_failed");
|
|
11854
|
+
observability.warn("sandbox reaper: retained-process claim failed", {
|
|
11855
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11856
|
+
});
|
|
11857
|
+
return;
|
|
11858
|
+
}
|
|
11859
|
+
for (const claim of claims) {
|
|
11860
|
+
let process2 = claim.process;
|
|
11861
|
+
const expected = retainedProcessSettlementIdentity2(process2);
|
|
11862
|
+
let proof = retainedProcessReconciliationProof(process2);
|
|
11863
|
+
if (!proof) {
|
|
11864
|
+
const lease = await readLease3(db, process2.workspaceId, process2.sandboxGroupId).catch(
|
|
11865
|
+
() => null
|
|
11866
|
+
);
|
|
11867
|
+
if (!lease || lease.id !== process2.leaseId || lease.leaseEpoch !== process2.leaseEpoch || lease.backend !== process2.providerBackend || lease.instanceId !== process2.providerInstanceId || process2.routeTargetId !== null) {
|
|
11868
|
+
await deferRetainedProcessClaim(
|
|
11869
|
+
db,
|
|
11870
|
+
settings,
|
|
11871
|
+
observability,
|
|
11872
|
+
process2,
|
|
11873
|
+
expected,
|
|
11874
|
+
claim.claimId,
|
|
11875
|
+
"identity_mismatch"
|
|
11876
|
+
);
|
|
11877
|
+
continue;
|
|
11878
|
+
}
|
|
11879
|
+
let observation;
|
|
11880
|
+
try {
|
|
11881
|
+
observation = await probe(settings, lease, process2);
|
|
11882
|
+
} catch (error) {
|
|
11883
|
+
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
11884
|
+
processId: process2.id,
|
|
11885
|
+
providerBackend: process2.providerBackend,
|
|
11886
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11887
|
+
});
|
|
11888
|
+
observation = { status: "deferred", reason: "provider_error" };
|
|
11889
|
+
}
|
|
11890
|
+
if (observation.status === "deferred") {
|
|
11891
|
+
await deferRetainedProcessClaim(
|
|
11892
|
+
db,
|
|
11893
|
+
settings,
|
|
11894
|
+
observability,
|
|
11895
|
+
process2,
|
|
11896
|
+
expected,
|
|
11897
|
+
claim.claimId,
|
|
11898
|
+
observation.reason
|
|
11899
|
+
);
|
|
11900
|
+
continue;
|
|
11901
|
+
}
|
|
11902
|
+
try {
|
|
11903
|
+
process2 = await recordRetainedProcessReconciliationProof(db, {
|
|
11904
|
+
accountId: process2.accountId,
|
|
11905
|
+
workspaceId: process2.workspaceId,
|
|
11906
|
+
sessionId: process2.sessionId,
|
|
11907
|
+
processId: process2.id,
|
|
11908
|
+
expected,
|
|
11909
|
+
claimId: claim.claimId,
|
|
11910
|
+
proof: observation.proof
|
|
11911
|
+
});
|
|
11912
|
+
proof = observation.proof;
|
|
11913
|
+
recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);
|
|
11914
|
+
} catch (error) {
|
|
11915
|
+
recordRetainedProcessReconciliation(observability, "proof_checkpoint_failed");
|
|
11916
|
+
observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
|
|
11917
|
+
processId: process2.id,
|
|
11918
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11919
|
+
});
|
|
11920
|
+
continue;
|
|
11921
|
+
}
|
|
11922
|
+
}
|
|
11923
|
+
try {
|
|
11924
|
+
await settleRetainedProcess2(db, {
|
|
11925
|
+
accountId: process2.accountId,
|
|
11926
|
+
workspaceId: process2.workspaceId,
|
|
11927
|
+
sessionId: process2.sessionId,
|
|
11928
|
+
processId: process2.id,
|
|
11929
|
+
expected,
|
|
11930
|
+
reconciliationClaimId: claim.claimId,
|
|
11931
|
+
outcome: proof.outcome,
|
|
11932
|
+
exitCode: proof.exitCode,
|
|
11933
|
+
reason: proof.reason,
|
|
11934
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
11935
|
+
});
|
|
11936
|
+
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
11937
|
+
} catch (error) {
|
|
11938
|
+
recordRetainedProcessReconciliation(observability, "settlement_failed");
|
|
11939
|
+
observability.warn("sandbox reaper: retained-process settlement failed", {
|
|
11940
|
+
processId: process2.id,
|
|
11941
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11942
|
+
});
|
|
11943
|
+
await deferRetainedProcessClaim(
|
|
11944
|
+
db,
|
|
11945
|
+
settings,
|
|
11946
|
+
observability,
|
|
11947
|
+
process2,
|
|
11948
|
+
expected,
|
|
11949
|
+
claim.claimId,
|
|
11950
|
+
"settlement_failed"
|
|
11951
|
+
);
|
|
11952
|
+
}
|
|
11953
|
+
}
|
|
11954
|
+
}
|
|
11955
|
+
async function deferRetainedProcessClaim(db, settings, observability, process2, expected, claimId, outcome) {
|
|
11956
|
+
const exponent = Math.min(4, Math.max(0, process2.reconcileAttempts - 1));
|
|
11957
|
+
const retryAfterMs = Math.min(
|
|
11958
|
+
5 * 6e4,
|
|
11959
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
|
|
11960
|
+
);
|
|
11961
|
+
try {
|
|
11962
|
+
await deferRetainedProcessReconciliation(db, {
|
|
11963
|
+
accountId: process2.accountId,
|
|
11964
|
+
workspaceId: process2.workspaceId,
|
|
11965
|
+
sessionId: process2.sessionId,
|
|
11966
|
+
processId: process2.id,
|
|
11967
|
+
expected,
|
|
11968
|
+
claimId,
|
|
11969
|
+
outcome,
|
|
11970
|
+
retryAfterMs
|
|
11971
|
+
});
|
|
11972
|
+
recordRetainedProcessReconciliation(observability, outcome);
|
|
11973
|
+
} catch (error) {
|
|
11974
|
+
recordRetainedProcessReconciliation(observability, "defer_failed");
|
|
11975
|
+
observability.warn("sandbox reaper: retained-process defer failed", {
|
|
11976
|
+
processId: process2.id,
|
|
11977
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11978
|
+
});
|
|
11979
|
+
}
|
|
11980
|
+
}
|
|
11981
|
+
var RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
|
|
11982
|
+
async function withRetainedProcessProbeTimeout(promise) {
|
|
11983
|
+
let timer;
|
|
11984
|
+
try {
|
|
11985
|
+
return await Promise.race([
|
|
11986
|
+
promise,
|
|
11987
|
+
new Promise((_resolve, reject) => {
|
|
11988
|
+
timer = setTimeout(
|
|
11989
|
+
() => reject(RETAINED_PROCESS_PROBE_TIMEOUT),
|
|
11990
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
11991
|
+
);
|
|
11992
|
+
timer.unref?.();
|
|
11993
|
+
})
|
|
11994
|
+
]);
|
|
11995
|
+
} finally {
|
|
11996
|
+
if (timer) clearTimeout(timer);
|
|
11997
|
+
}
|
|
11998
|
+
}
|
|
11999
|
+
async function probeRetainedProcessAtProvider(settings, lease, process2) {
|
|
12000
|
+
if (lease.id !== process2.leaseId || lease.sandboxGroupId !== process2.sandboxGroupId || lease.leaseEpoch !== process2.leaseEpoch || lease.backend !== process2.providerBackend || lease.instanceId !== process2.providerInstanceId || lease.resumeBackendId !== process2.providerBackend || process2.routeTargetId !== null) {
|
|
12001
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
12002
|
+
}
|
|
12003
|
+
if (!lease.resumeState) {
|
|
12004
|
+
return { status: "deferred", reason: "resume_state_missing" };
|
|
12005
|
+
}
|
|
12006
|
+
const envelopeBackend = lease.resumeState.backendId;
|
|
12007
|
+
if (envelopeBackend !== void 0 && envelopeBackend !== process2.providerBackend) {
|
|
12008
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
12009
|
+
}
|
|
12010
|
+
if (providerIdentityFromResumeState(lease.resumeState) !== process2.providerInstanceId) {
|
|
12011
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
12012
|
+
}
|
|
12013
|
+
let client;
|
|
12014
|
+
try {
|
|
12015
|
+
client = createSandboxClientForBackend(process2.providerBackend, settings);
|
|
12016
|
+
} catch {
|
|
12017
|
+
return { status: "deferred", reason: "provider_error" };
|
|
12018
|
+
}
|
|
12019
|
+
if (!client || client.backendId !== process2.providerBackend || !client.resume) {
|
|
12020
|
+
return { status: "deferred", reason: "backend_unsupported" };
|
|
12021
|
+
}
|
|
12022
|
+
const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
|
|
12023
|
+
let resumedState;
|
|
12024
|
+
try {
|
|
12025
|
+
resumedState = await deserializeSandboxSessionStateEnvelope(
|
|
12026
|
+
client,
|
|
12027
|
+
envelopeSessionState
|
|
12028
|
+
);
|
|
12029
|
+
} catch {
|
|
12030
|
+
return { status: "deferred", reason: "provider_error" };
|
|
12031
|
+
}
|
|
12032
|
+
if (resumedState === void 0) {
|
|
12033
|
+
return { status: "deferred", reason: "resume_state_missing" };
|
|
12034
|
+
}
|
|
12035
|
+
let session;
|
|
12036
|
+
try {
|
|
12037
|
+
session = await withRetainedProcessProbeTimeout(
|
|
12038
|
+
client.resume(resumedState)
|
|
12039
|
+
);
|
|
12040
|
+
} catch (error) {
|
|
12041
|
+
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
12042
|
+
return { status: "deferred", reason: "provider_timeout" };
|
|
12043
|
+
}
|
|
12044
|
+
if (isProviderSandboxNotFoundError3(client.backendId, error)) {
|
|
12045
|
+
return {
|
|
12046
|
+
status: "proved",
|
|
12047
|
+
proof: {
|
|
12048
|
+
outcome: "lost",
|
|
12049
|
+
exitCode: null,
|
|
12050
|
+
reason: "provider_instance_not_found"
|
|
12051
|
+
}
|
|
12052
|
+
};
|
|
12053
|
+
}
|
|
12054
|
+
return { status: "deferred", reason: "provider_error" };
|
|
12055
|
+
}
|
|
12056
|
+
if (typeof session.writeStdin !== "function") {
|
|
12057
|
+
return { status: "deferred", reason: "backend_unsupported" };
|
|
12058
|
+
}
|
|
12059
|
+
let result;
|
|
12060
|
+
try {
|
|
12061
|
+
result = await withRetainedProcessProbeTimeout(
|
|
12062
|
+
session.writeStdin({
|
|
12063
|
+
sessionId: process2.providerSessionId,
|
|
12064
|
+
chars: "",
|
|
12065
|
+
yieldTimeMs: 1e3,
|
|
12066
|
+
maxOutputTokens: 2e3
|
|
12067
|
+
})
|
|
12068
|
+
);
|
|
12069
|
+
} catch (error) {
|
|
12070
|
+
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
12071
|
+
return { status: "deferred", reason: "provider_timeout" };
|
|
12072
|
+
}
|
|
12073
|
+
if (isProviderSandboxNotFoundError3(client.backendId, error)) {
|
|
12074
|
+
return {
|
|
12075
|
+
status: "proved",
|
|
12076
|
+
proof: {
|
|
12077
|
+
outcome: "lost",
|
|
12078
|
+
exitCode: null,
|
|
12079
|
+
reason: "provider_instance_not_found"
|
|
12080
|
+
}
|
|
12081
|
+
};
|
|
12082
|
+
}
|
|
12083
|
+
return { status: "deferred", reason: "provider_error" };
|
|
12084
|
+
}
|
|
12085
|
+
return classifyRetainedProcessPollResult(result, process2.providerSessionId);
|
|
12086
|
+
}
|
|
12087
|
+
function classifyRetainedProcessPollResult(result, providerSessionId) {
|
|
12088
|
+
if (typeof result !== "string") {
|
|
12089
|
+
return { status: "deferred", reason: "provider_unknown" };
|
|
12090
|
+
}
|
|
12091
|
+
if (isExecSessionLostBanner(result, providerSessionId)) {
|
|
12092
|
+
return {
|
|
12093
|
+
status: "proved",
|
|
12094
|
+
proof: {
|
|
12095
|
+
outcome: "lost",
|
|
12096
|
+
exitCode: null,
|
|
12097
|
+
reason: "provider_session_lost_banner"
|
|
12098
|
+
}
|
|
12099
|
+
};
|
|
12100
|
+
}
|
|
12101
|
+
const exitCode = parseExecBannerExitCode(result);
|
|
12102
|
+
if (exitCode !== null) {
|
|
12103
|
+
return {
|
|
12104
|
+
status: "proved",
|
|
12105
|
+
proof: { outcome: "exited", exitCode, reason: "provider_exit_banner" }
|
|
12106
|
+
};
|
|
12107
|
+
}
|
|
12108
|
+
return {
|
|
12109
|
+
status: "deferred",
|
|
12110
|
+
reason: result.includes("Process running with session ID") ? "provider_running" : "provider_unknown"
|
|
12111
|
+
};
|
|
12112
|
+
}
|
|
11576
12113
|
async function accrueWarmTick(db, settings, observability) {
|
|
11577
12114
|
const workspaceIds = /* @__PURE__ */ new Set();
|
|
11578
12115
|
let accrued = 0;
|
|
@@ -11626,6 +12163,26 @@ async function refreshQueueLeaseAndCreditGauges(db, observability) {
|
|
|
11626
12163
|
error: error instanceof Error ? error.message : String(error)
|
|
11627
12164
|
});
|
|
11628
12165
|
}
|
|
12166
|
+
try {
|
|
12167
|
+
recordRetainedProcessInventoryGauges(
|
|
12168
|
+
observability,
|
|
12169
|
+
await countActiveRetainedProcessesByOwnerState(db)
|
|
12170
|
+
);
|
|
12171
|
+
} catch (error) {
|
|
12172
|
+
observability.warn("sandbox reaper: retained-process gauge refresh failed", {
|
|
12173
|
+
error: error instanceof Error ? error.message : String(error)
|
|
12174
|
+
});
|
|
12175
|
+
}
|
|
12176
|
+
try {
|
|
12177
|
+
recordExpiredDrainingSandboxLeaseGauges(
|
|
12178
|
+
observability,
|
|
12179
|
+
await countExpiredDrainingSandboxLeases(db)
|
|
12180
|
+
);
|
|
12181
|
+
} catch (error) {
|
|
12182
|
+
observability.warn("sandbox reaper: expired-draining gauge refresh failed", {
|
|
12183
|
+
error: error instanceof Error ? error.message : String(error)
|
|
12184
|
+
});
|
|
12185
|
+
}
|
|
11629
12186
|
try {
|
|
11630
12187
|
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
11631
12188
|
} catch (error) {
|
|
@@ -13600,7 +14157,10 @@ var documentActivity = proxyActivities({
|
|
|
13600
14157
|
});
|
|
13601
14158
|
|
|
13602
14159
|
// src/http.ts
|
|
13603
|
-
import {
|
|
14160
|
+
import {
|
|
14161
|
+
assertRuntimeDatabasePosture,
|
|
14162
|
+
dbSql
|
|
14163
|
+
} from "@opengeni/db";
|
|
13604
14164
|
function createWorkerHttpHandler(input) {
|
|
13605
14165
|
const { settings, observability, checks, lifecycle } = input;
|
|
13606
14166
|
const timeoutMs = input.timeoutMs ?? 2e3;
|
|
@@ -13675,8 +14235,12 @@ async function runReadinessChecks(checks, timeoutMs = 2e3) {
|
|
|
13675
14235
|
checks: result
|
|
13676
14236
|
};
|
|
13677
14237
|
}
|
|
13678
|
-
function dbReadyCheck(db) {
|
|
14238
|
+
function dbReadyCheck(db, posture) {
|
|
13679
14239
|
return async () => {
|
|
14240
|
+
if (posture) {
|
|
14241
|
+
await assertRuntimeDatabasePosture(db, posture);
|
|
14242
|
+
return;
|
|
14243
|
+
}
|
|
13680
14244
|
await db.execute(dbSql`select 1`);
|
|
13681
14245
|
};
|
|
13682
14246
|
}
|
|
@@ -14155,7 +14719,7 @@ async function createOpenGeniWorkerService(options) {
|
|
|
14155
14719
|
settings,
|
|
14156
14720
|
observability,
|
|
14157
14721
|
checks: {
|
|
14158
|
-
db: dbReadyCheck(options.activityDependencies.db),
|
|
14722
|
+
db: dbReadyCheck(options.activityDependencies.db, options.databasePosture),
|
|
14159
14723
|
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
14160
14724
|
temporal: temporalReadyCheck(workerBundle.connection)
|
|
14161
14725
|
},
|
|
@@ -14240,9 +14804,19 @@ async function startWorker() {
|
|
|
14240
14804
|
...searchPath ? { searchPath } : {},
|
|
14241
14805
|
rlsStrategy: settings.rlsStrategy
|
|
14242
14806
|
});
|
|
14807
|
+
const databasePosture = {
|
|
14808
|
+
rlsStrategy: settings.rlsStrategy,
|
|
14809
|
+
expectedRole: settings.runtimeDatabaseRole,
|
|
14810
|
+
targetSchema: settings.dbSchema.trim() || "public"
|
|
14811
|
+
};
|
|
14243
14812
|
const controlPlaneAuth = resolveNatsControlPlaneAuth2(settings);
|
|
14244
14813
|
let bus;
|
|
14245
14814
|
try {
|
|
14815
|
+
await retryStartupDependency(
|
|
14816
|
+
"PostgreSQL runtime posture",
|
|
14817
|
+
() => assertRuntimeDatabasePosture2(dbClient.db, databasePosture),
|
|
14818
|
+
{ ...retryOptions, onRetry }
|
|
14819
|
+
);
|
|
14246
14820
|
bus = await retryStartupDependency(
|
|
14247
14821
|
"NATS",
|
|
14248
14822
|
() => createNatsEventBus2(
|
|
@@ -14255,6 +14829,7 @@ async function startWorker() {
|
|
|
14255
14829
|
await runOpenGeniWorker({
|
|
14256
14830
|
role,
|
|
14257
14831
|
settings,
|
|
14832
|
+
databasePosture,
|
|
14258
14833
|
activityDependencies: {
|
|
14259
14834
|
observability,
|
|
14260
14835
|
db: dbClient.db,
|