@opengeni/worker-bundle 0.16.17 → 0.16.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +83 -32
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/environment.d.ts +1 -0
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +6 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +93 -0
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +2 -0
- package/dist/activities/toolspace-token-renewal.d.ts +1 -1
- package/dist/activities/workspace-capture.d.ts +1 -0
- package/dist/activities.d.ts +3 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/host-export.js +1 -1
- package/dist/index.js +1465 -453
- package/dist/index.js.map +1 -1
- package/dist/sandbox-resume.d.ts +6 -0
- package/dist/sandbox-routing.d.ts +11 -6
- package/dist/workflow-bundle.js +2 -1
- package/package.json +12 -12
- package/src/activities/agent-turn.ts +559 -434
- package/src/activities/codex-capacity.ts +10 -1
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +9 -5
- package/src/activities/environment.ts +13 -24
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +108 -7
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +892 -0
- package/src/activities/rig-verification.ts +13 -34
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +18 -4
- package/src/activities/sandbox-lease.ts +5 -2
- package/src/activities/scheduled-tasks.ts +19 -3
- package/src/activities/toolspace-token-renewal.ts +9 -2
- package/src/activities/workspace-capture.ts +33 -3
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/observability-metrics.ts +3 -3
- package/src/sandbox-resume.ts +45 -12
- package/src/sandbox-routing.ts +35 -17
- package/src/worker-service-lifecycle.ts +8 -4
- package/src/workflows/file-upload-reaper.ts +4 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createHostExportPump
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-RAMNYSAW.js";
|
|
4
4
|
|
|
5
5
|
// src/index.ts
|
|
6
6
|
import {
|
|
@@ -72,6 +72,7 @@ import {
|
|
|
72
72
|
getCodexRotationSettings,
|
|
73
73
|
getWorkspaceModelPolicy as getWorkspaceModelPolicy2,
|
|
74
74
|
getWorkspaceGrant,
|
|
75
|
+
getWorkspace as getWorkspace2,
|
|
75
76
|
listCodexAccountStatuses as listCodexAccountStatuses2,
|
|
76
77
|
fetchCodexUsageForAccount as fetchCodexUsageForAccount2,
|
|
77
78
|
getSessionCodexState,
|
|
@@ -110,7 +111,7 @@ import {
|
|
|
110
111
|
SandboxLeaseSupersededError as SandboxLeaseSupersededError2,
|
|
111
112
|
SandboxImageConflictError,
|
|
112
113
|
isSessionEventPersistenceError,
|
|
113
|
-
getEnrollment,
|
|
114
|
+
getEnrollment as getEnrollment2,
|
|
114
115
|
abandonRecordingForTurnAttempt,
|
|
115
116
|
markSessionAttemptQuiesced,
|
|
116
117
|
getOrCreatePreferenceRegistrySnapshot,
|
|
@@ -127,9 +128,10 @@ import {
|
|
|
127
128
|
sandboxStateEntryFromRunState,
|
|
128
129
|
maxTurnsExceededRunState,
|
|
129
130
|
modelResponseServiceTierFromSdkEvent,
|
|
130
|
-
|
|
131
|
+
modelTerminalResponseFromSdkEvent,
|
|
131
132
|
normalizeModelCallUsage,
|
|
132
133
|
normalizeSdkEvent,
|
|
134
|
+
normalizeProtocolJsonValue,
|
|
133
135
|
projectHistoryForProvider as projectHistoryForProvider2,
|
|
134
136
|
sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2,
|
|
135
137
|
projectModelInputForCapabilities,
|
|
@@ -164,6 +166,7 @@ import {
|
|
|
164
166
|
runOwnedSandboxSetup,
|
|
165
167
|
RoutingMutationOutcomeUnknownError,
|
|
166
168
|
WorkspaceArchiveIntegrityError as WorkspaceArchiveIntegrityError2,
|
|
169
|
+
sdkBackendIdForSandboxBackend,
|
|
167
170
|
swapTargetEstablishability,
|
|
168
171
|
deleteRecordingArtifacts as deleteRecordingArtifacts2,
|
|
169
172
|
stopRecording as stopRecordingOnBox2
|
|
@@ -590,10 +593,21 @@ function selectCodexCredentialLeaseForTurn(args) {
|
|
|
590
593
|
};
|
|
591
594
|
}
|
|
592
595
|
if (!rotationEnabled) {
|
|
593
|
-
const
|
|
596
|
+
const active = activeCredentialId ? accounts.find((account) => account.id === activeCredentialId) : void 0;
|
|
597
|
+
if (!active?.allocatorEnabled) {
|
|
598
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
599
|
+
}
|
|
600
|
+
if (!isCodexCredentialHealthy(active, args.now)) {
|
|
601
|
+
return {
|
|
602
|
+
credentialId: null,
|
|
603
|
+
decision: { kind: "allCapped", earliestResetAt: availableAt(active, args.now) },
|
|
604
|
+
advanceActivePointer: false
|
|
605
|
+
};
|
|
606
|
+
}
|
|
594
607
|
return {
|
|
595
|
-
credentialId,
|
|
596
|
-
decision:
|
|
608
|
+
credentialId: active.id,
|
|
609
|
+
decision: { kind: "active", credentialId: active.id, moved: false },
|
|
610
|
+
advanceActivePointer: false
|
|
597
611
|
};
|
|
598
612
|
}
|
|
599
613
|
if (!leasingEnabled && pinDisposition !== "clearStale" && args.sessionPinnedCredentialId) {
|
|
@@ -871,7 +885,9 @@ function codexCapacityDecision(context, settings) {
|
|
|
871
885
|
sessionLastCredentialId: context.sessionLastCredentialId,
|
|
872
886
|
now
|
|
873
887
|
});
|
|
874
|
-
|
|
888
|
+
const selectedAccount = selected.credentialId ? context.accounts.find((account) => account.id === selected.credentialId) : void 0;
|
|
889
|
+
const selectedIsAvailable = selectedAccount !== void 0 && (selectedAccount.id === context.existingCredentialId ? isCodexCredentialHealthy(selectedAccount, now) : isCodexCredentialEligible(selectedAccount, now));
|
|
890
|
+
if (selected.credentialId && selectedIsAvailable) {
|
|
875
891
|
return {
|
|
876
892
|
kind: "available",
|
|
877
893
|
credentialId: selected.credentialId,
|
|
@@ -1326,6 +1342,7 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
1326
1342
|
joinInternalContext(internalContext, attachmentContext),
|
|
1327
1343
|
fileAttachments.map((attachment) => attachment.resource),
|
|
1328
1344
|
options.providerApi,
|
|
1345
|
+
options.materializeModelHistory,
|
|
1329
1346
|
options.projectModelHistory
|
|
1330
1347
|
);
|
|
1331
1348
|
}
|
|
@@ -1342,6 +1359,7 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
1342
1359
|
internalContext,
|
|
1343
1360
|
[],
|
|
1344
1361
|
options.providerApi,
|
|
1362
|
+
options.materializeModelHistory,
|
|
1345
1363
|
options.projectModelHistory
|
|
1346
1364
|
);
|
|
1347
1365
|
}
|
|
@@ -1351,9 +1369,10 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
1351
1369
|
if (!state) {
|
|
1352
1370
|
throw new Error("No saved run state is available for approval decision");
|
|
1353
1371
|
}
|
|
1372
|
+
const serializedRunState = resumeRunState(state);
|
|
1354
1373
|
const prepared = await runtime.prepareInput(agent, {
|
|
1355
1374
|
kind: "approval",
|
|
1356
|
-
serializedRunState:
|
|
1375
|
+
serializedRunState: options.materializeSerializedRunState ? await options.materializeSerializedRunState(serializedRunState) : serializedRunState,
|
|
1357
1376
|
approvalId: String(payload.approvalId ?? ""),
|
|
1358
1377
|
decision: payload.decision === "approve" ? "approve" : "reject",
|
|
1359
1378
|
...typeof payload.message === "string" ? { message: payload.message } : {}
|
|
@@ -1379,9 +1398,10 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
1379
1398
|
if (!resume) {
|
|
1380
1399
|
throw new Error("Human-input response does not resolve to a durable request");
|
|
1381
1400
|
}
|
|
1401
|
+
const serializedRunState = resumeRunState(state);
|
|
1382
1402
|
const prepared = await runtime.prepareInput(agent, {
|
|
1383
1403
|
kind: "human_input",
|
|
1384
|
-
serializedRunState:
|
|
1404
|
+
serializedRunState: options.materializeSerializedRunState ? await options.materializeSerializedRunState(serializedRunState) : serializedRunState,
|
|
1385
1405
|
toolCallId: resume.toolCallId
|
|
1386
1406
|
});
|
|
1387
1407
|
return {
|
|
@@ -1400,13 +1420,14 @@ function joinInternalContext(...parts) {
|
|
|
1400
1420
|
const content = parts.map((part) => part?.trim()).filter((part) => Boolean(part));
|
|
1401
1421
|
return content.length > 0 ? content.join("\n\n") : void 0;
|
|
1402
1422
|
}
|
|
1403
|
-
async function messageInput(db, runtime, agent, trigger, text, internalContext, currentAttachmentRefs = [], providerApi = "responses", projectModelHistory) {
|
|
1423
|
+
async function messageInput(db, runtime, agent, trigger, text, internalContext, currentAttachmentRefs = [], providerApi = "responses", materializeModelHistory, projectModelHistory) {
|
|
1404
1424
|
const stored = await getActiveSessionHistoryItems(db, trigger.workspaceId, trigger.sessionId);
|
|
1405
1425
|
const envelope = await getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId);
|
|
1406
1426
|
const canonicalView = projectRejectedProviderArtifacts(stored);
|
|
1407
1427
|
const providerView = projectHistoryForProvider(canonicalView, providerApi);
|
|
1408
1428
|
const referencedHistory = withCurrentUserAttachmentRefs(providerView, currentAttachmentRefs);
|
|
1409
|
-
const
|
|
1429
|
+
const materializedHistory = materializeModelHistory ? await materializeModelHistory(referencedHistory) : referencedHistory;
|
|
1430
|
+
const historyItems = projectModelHistory ? await projectModelHistory(materializedHistory) : materializedHistory;
|
|
1410
1431
|
const prepared = await runtime.prepareInput(agent, {
|
|
1411
1432
|
kind: "message",
|
|
1412
1433
|
...text ? { text } : {},
|
|
@@ -1490,10 +1511,13 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
|
|
|
1490
1511
|
};
|
|
1491
1512
|
}
|
|
1492
1513
|
const canonicalItems = projectRejectedProviderArtifacts(active);
|
|
1493
|
-
const projectForWire = async (input) =>
|
|
1494
|
-
options.
|
|
1495
|
-
|
|
1496
|
-
|
|
1514
|
+
const projectForWire = async (input) => {
|
|
1515
|
+
const materialized = options.materializeHistory ? await options.materializeHistory(input) : input;
|
|
1516
|
+
return sanitizeHistoryItemsForModel(
|
|
1517
|
+
options.projectModelInput ? await options.projectModelInput(materialized) : materialized,
|
|
1518
|
+
settings.modelToolOutputTruncationTokens
|
|
1519
|
+
);
|
|
1520
|
+
};
|
|
1497
1521
|
const items = await projectForWire(canonicalItems);
|
|
1498
1522
|
const decision = decideCompaction({
|
|
1499
1523
|
items,
|
|
@@ -1615,7 +1639,6 @@ async function compactContextRemoteV2(db, settings, scope, canonicalItems, items
|
|
|
1615
1639
|
expectedAttemptId: scope.attemptId,
|
|
1616
1640
|
replacementItems: replacementHistory.slice(0, -1),
|
|
1617
1641
|
summaryItem: tailItem,
|
|
1618
|
-
replacementInputTokens: estimatedTokensAfter,
|
|
1619
1642
|
...options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {},
|
|
1620
1643
|
eventPayload: {
|
|
1621
1644
|
trigger: options.trigger ?? "auto",
|
|
@@ -1668,7 +1691,6 @@ async function compactContextPortable(db, settings, scope, canonicalItems, items
|
|
|
1668
1691
|
expectedAttemptId: scope.attemptId,
|
|
1669
1692
|
replacementItems: replacementHistory.slice(0, -1),
|
|
1670
1693
|
summaryItem,
|
|
1671
|
-
replacementInputTokens: estimatedTokensAfter,
|
|
1672
1694
|
...options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {},
|
|
1673
1695
|
eventPayload: {
|
|
1674
1696
|
trigger: options.trigger ?? "auto",
|
|
@@ -1745,6 +1767,7 @@ import {
|
|
|
1745
1767
|
import {
|
|
1746
1768
|
gitCredentialBindingIdForRepository,
|
|
1747
1769
|
gitCredentialProviderForRepository,
|
|
1770
|
+
gitRemoteIdentity,
|
|
1748
1771
|
signDelegatedAccessToken
|
|
1749
1772
|
} from "@opengeni/contracts";
|
|
1750
1773
|
import {
|
|
@@ -1810,8 +1833,13 @@ function assertWorkspaceEcho(kind, scope, echoedWorkspaceId) {
|
|
|
1810
1833
|
async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironment = {}, options = {}) {
|
|
1811
1834
|
const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
|
|
1812
1835
|
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
|
|
1836
|
+
if (options.skipToolspace) {
|
|
1837
|
+
delete environment.OPENGENI_TOOLSPACE_URL;
|
|
1838
|
+
delete environment.OPENGENI_TOOLSPACE_TOKEN_FILE;
|
|
1839
|
+
delete environment.OPENGENI_OGTOOL_PACKAGE_SPEC;
|
|
1840
|
+
}
|
|
1813
1841
|
const toolspaceScope = options.scope;
|
|
1814
|
-
const toolspaceToken = toolspaceScope && options.toolspaceAuthority ? await mintSandboxToolspaceToken(settings, toolspaceScope, options.toolspaceAuthority) : void 0;
|
|
1842
|
+
const toolspaceToken = !options.skipToolspace && toolspaceScope && options.toolspaceAuthority ? await mintSandboxToolspaceToken(settings, toolspaceScope, options.toolspaceAuthority) : void 0;
|
|
1815
1843
|
if (toolspaceToken && toolspaceScope) {
|
|
1816
1844
|
environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(
|
|
1817
1845
|
settings,
|
|
@@ -2137,7 +2165,7 @@ function gitCredentialSelections(resources) {
|
|
|
2137
2165
|
);
|
|
2138
2166
|
}
|
|
2139
2167
|
bindingProviders.set(credentialBindingId, provider);
|
|
2140
|
-
const normalizedRemote =
|
|
2168
|
+
const normalizedRemote = gitRemoteIdentity(resource.uri, provider);
|
|
2141
2169
|
const claimedBinding = remoteBindings.get(normalizedRemote);
|
|
2142
2170
|
if (claimedBinding && claimedBinding !== bindingKey) {
|
|
2143
2171
|
throw new Error(
|
|
@@ -2193,14 +2221,6 @@ function normalizedGitHost(uri) {
|
|
|
2193
2221
|
return "";
|
|
2194
2222
|
}
|
|
2195
2223
|
}
|
|
2196
|
-
function normalizedGitRemote(uri) {
|
|
2197
|
-
try {
|
|
2198
|
-
const url = new URL(uri);
|
|
2199
|
-
return `${url.protocol.toLowerCase()}//${url.host.toLowerCase()}${url.pathname.replace(/\/+$/, "").replace(/\.git$/, "")}`;
|
|
2200
|
-
} catch {
|
|
2201
|
-
return uri.trim().replace(/\/+$/, "").replace(/\.git$/, "").toLowerCase();
|
|
2202
|
-
}
|
|
2203
|
-
}
|
|
2204
2224
|
function repositoryCredentialProvider(resource) {
|
|
2205
2225
|
return gitCredentialProviderForRepository(resource);
|
|
2206
2226
|
}
|
|
@@ -2303,7 +2323,7 @@ function startGitCredentialRenewalLoop(options) {
|
|
|
2303
2323
|
} catch {
|
|
2304
2324
|
}
|
|
2305
2325
|
scheduleRefresh(nextDelayMs);
|
|
2306
|
-
} catch
|
|
2326
|
+
} catch {
|
|
2307
2327
|
if (stopped) return;
|
|
2308
2328
|
const scheduledRetryMs = retryDelayMs;
|
|
2309
2329
|
retryDelayMs = Math.min(retryDelayMs * 2, GIT_CREDENTIAL_MAX_RETRY_MS);
|
|
@@ -2311,7 +2331,7 @@ function startGitCredentialRenewalLoop(options) {
|
|
|
2311
2331
|
options.onFailure?.({
|
|
2312
2332
|
providers,
|
|
2313
2333
|
retryDelayMs: scheduledRetryMs,
|
|
2314
|
-
errorClass:
|
|
2334
|
+
errorClass: "GitCredentialRenewalOperationError"
|
|
2315
2335
|
});
|
|
2316
2336
|
} catch {
|
|
2317
2337
|
}
|
|
@@ -2402,14 +2422,14 @@ function startExpiringMaterialRenewalLoop(options) {
|
|
|
2402
2422
|
} catch {
|
|
2403
2423
|
}
|
|
2404
2424
|
scheduleRefresh(nextDelayMs);
|
|
2405
|
-
} catch
|
|
2425
|
+
} catch {
|
|
2406
2426
|
if (stopped) return;
|
|
2407
2427
|
const scheduledRetryMs = retryDelayMs;
|
|
2408
2428
|
retryDelayMs = Math.min(retryDelayMs * 2, options.policy.maxRetryMs);
|
|
2409
2429
|
try {
|
|
2410
2430
|
options.onFailure?.({
|
|
2411
2431
|
retryDelayMs: scheduledRetryMs,
|
|
2412
|
-
errorClass:
|
|
2432
|
+
errorClass: options.publicErrorClass
|
|
2413
2433
|
});
|
|
2414
2434
|
} catch {
|
|
2415
2435
|
}
|
|
@@ -2451,6 +2471,7 @@ function startRunCredentialRenewalLoop(options) {
|
|
|
2451
2471
|
resolve: options.resolve,
|
|
2452
2472
|
write: options.write,
|
|
2453
2473
|
expiresAt: (material) => material?.expiresAt ?? null,
|
|
2474
|
+
publicErrorClass: "RunCredentialRenewalOperationError",
|
|
2454
2475
|
policy: {
|
|
2455
2476
|
defaultRefreshMs: RUN_CREDENTIAL_DEFAULT_REFRESH_MS,
|
|
2456
2477
|
expiryLeadMs: RUN_CREDENTIAL_EXPIRY_LEAD_MS,
|
|
@@ -2490,6 +2511,7 @@ function startToolspaceTokenRenewalLoop(options) {
|
|
|
2490
2511
|
},
|
|
2491
2512
|
write: options.write,
|
|
2492
2513
|
expiresAt: (material) => material.expiresAt,
|
|
2514
|
+
publicErrorClass: "ToolspaceTokenRenewalOperationError",
|
|
2493
2515
|
policy: {
|
|
2494
2516
|
defaultRefreshMs: TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS,
|
|
2495
2517
|
expiryLeadMs: TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
|
|
@@ -2892,10 +2914,6 @@ function workspacePackRuntimeFromPacks(packs) {
|
|
|
2892
2914
|
skills
|
|
2893
2915
|
};
|
|
2894
2916
|
}
|
|
2895
|
-
async function resolveWorkspaceAgentInstructions(db, workspaceId) {
|
|
2896
|
-
const workspace = await getWorkspace(db, workspaceId);
|
|
2897
|
-
return workspace?.agentInstructions ?? null;
|
|
2898
|
-
}
|
|
2899
2917
|
function settingsWithPackSandboxImage(settings, sandboxImage, sandboxProviderImages = null) {
|
|
2900
2918
|
if (!sandboxImage) {
|
|
2901
2919
|
return settings;
|
|
@@ -3150,16 +3168,6 @@ function childCompletionSummary(child, goal, terminalStatus) {
|
|
|
3150
3168
|
return lines.join("\n");
|
|
3151
3169
|
}
|
|
3152
3170
|
|
|
3153
|
-
// src/activities/redaction.ts
|
|
3154
|
-
import {
|
|
3155
|
-
createSecretRedactor,
|
|
3156
|
-
identityRedactor,
|
|
3157
|
-
isCredentialHeaderName,
|
|
3158
|
-
redactSensitiveData,
|
|
3159
|
-
redactSensitiveText,
|
|
3160
|
-
redactSerializedJson
|
|
3161
|
-
} from "@opengeni/contracts";
|
|
3162
|
-
|
|
3163
3171
|
// src/activities/streaming.ts
|
|
3164
3172
|
import { Context } from "@temporalio/activity";
|
|
3165
3173
|
var TRAILING_FLUSH_MS = 33;
|
|
@@ -3310,7 +3318,6 @@ var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery
|
|
|
3310
3318
|
|
|
3311
3319
|
// src/sandbox-resume.ts
|
|
3312
3320
|
import { sandboxArchiveCaptureTimeoutMs } from "@opengeni/config";
|
|
3313
|
-
import { redactSensitiveText as redactSensitiveText2 } from "@opengeni/contracts";
|
|
3314
3321
|
import { randomUUID } from "crypto";
|
|
3315
3322
|
import {
|
|
3316
3323
|
acquireLease,
|
|
@@ -3398,14 +3405,18 @@ var SnapshotTimeoutError = class extends Error {
|
|
|
3398
3405
|
}
|
|
3399
3406
|
};
|
|
3400
3407
|
function safeSnapshotError(error) {
|
|
3401
|
-
const
|
|
3402
|
-
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
0,
|
|
3406
|
-
2048
|
|
3407
|
-
)
|
|
3408
|
+
const fields = {
|
|
3409
|
+
errorClass: "SnapshotOperationError",
|
|
3410
|
+
errorCode: "snapshot_operation_failed",
|
|
3411
|
+
origin: "sandbox-resume"
|
|
3408
3412
|
};
|
|
3413
|
+
try {
|
|
3414
|
+
const rawStatus = error && typeof error === "object" ? error.status ?? error.statusCode : void 0;
|
|
3415
|
+
const status = Number(rawStatus);
|
|
3416
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
|
|
3417
|
+
} catch {
|
|
3418
|
+
}
|
|
3419
|
+
return fields;
|
|
3409
3420
|
}
|
|
3410
3421
|
async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
|
|
3411
3422
|
if (signal?.aborted) return false;
|
|
@@ -3572,6 +3583,10 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
|
|
|
3572
3583
|
if (typeof persistable.persistWorkspace !== "function") {
|
|
3573
3584
|
return false;
|
|
3574
3585
|
}
|
|
3586
|
+
const workspacePersistence = persistable.state?.workspacePersistence ?? persistable.state?.providerState?.workspacePersistence;
|
|
3587
|
+
if (!force && workspacePersistence === "snapshot_filesystem") {
|
|
3588
|
+
return false;
|
|
3589
|
+
}
|
|
3575
3590
|
if (signal?.aborted) {
|
|
3576
3591
|
return false;
|
|
3577
3592
|
}
|
|
@@ -3581,7 +3596,7 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
|
|
|
3581
3596
|
return false;
|
|
3582
3597
|
}
|
|
3583
3598
|
const instanceId = lease.instanceId;
|
|
3584
|
-
const nativeModalPersistence =
|
|
3599
|
+
const nativeModalPersistence = workspacePersistence === "snapshot_filesystem" || workspacePersistence === "snapshot_directory";
|
|
3585
3600
|
const checkpointBinding = nativeModalPersistence ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal) : null;
|
|
3586
3601
|
if (signal?.aborted) return false;
|
|
3587
3602
|
const captureId = randomUUID();
|
|
@@ -4199,6 +4214,7 @@ import {
|
|
|
4199
4214
|
verifyRetainedProcessMutationSettlement,
|
|
4200
4215
|
verifyWorkspaceMutationSettlement,
|
|
4201
4216
|
getSandbox,
|
|
4217
|
+
getEnrollment,
|
|
4202
4218
|
markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
|
|
4203
4219
|
readLease as readLease2,
|
|
4204
4220
|
readActiveSandbox,
|
|
@@ -4215,6 +4231,7 @@ import {
|
|
|
4215
4231
|
RoutingBackendRecoveryRequiredError,
|
|
4216
4232
|
RoutingSandboxSession,
|
|
4217
4233
|
resolveModalCheckpointProviderBindingForSession as resolveModalCheckpointProviderBindingForSession2,
|
|
4234
|
+
sandboxBackendForSdkBackendId,
|
|
4218
4235
|
verifySandboxExecReadiness as verifySandboxExecReadiness2
|
|
4219
4236
|
} from "@opengeni/runtime";
|
|
4220
4237
|
function relayConfigFromSettings(settings) {
|
|
@@ -4300,7 +4317,8 @@ async function resolveCurrentHomeBackend(services, ids, established) {
|
|
|
4300
4317
|
};
|
|
4301
4318
|
}
|
|
4302
4319
|
const resumeState = lease.resumeState;
|
|
4303
|
-
const
|
|
4320
|
+
const durableResumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
|
|
4321
|
+
const resumeBackend = sandboxBackendForSdkBackendId(durableResumeBackend) ?? durableResumeBackend;
|
|
4304
4322
|
if (!resumeState || resumeBackend !== ids.backend || providerIdentityFromResumeState(resumeState) !== lease.instanceId) {
|
|
4305
4323
|
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
4306
4324
|
}
|
|
@@ -4341,7 +4359,8 @@ async function resolveCurrentHomeBackend(services, ids, established) {
|
|
|
4341
4359
|
marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease)
|
|
4342
4360
|
);
|
|
4343
4361
|
}
|
|
4344
|
-
|
|
4362
|
+
const reboundBackend = sandboxBackendForSdkBackendId(rebound.backendId) ?? rebound.backendId;
|
|
4363
|
+
if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
|
|
4345
4364
|
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
4346
4365
|
}
|
|
4347
4366
|
services.onHomeSandboxRebound?.({
|
|
@@ -4539,6 +4558,11 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
4539
4558
|
} : null;
|
|
4540
4559
|
},
|
|
4541
4560
|
controlRpcFactory: controlRpcFactory(bus),
|
|
4561
|
+
resolveSelfhostedOpStream: async (sandbox) => {
|
|
4562
|
+
if (!sandbox.enrollmentId) return void 0;
|
|
4563
|
+
const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
|
|
4564
|
+
return opStreamDepsFor(services, enrollment?.opStream === true);
|
|
4565
|
+
},
|
|
4542
4566
|
relay: relayConfigFromSettings(settings),
|
|
4543
4567
|
// A selfhosted swap target runs real commands too, so give it the same split
|
|
4544
4568
|
// deadlines the machine-primary establish path uses (short control, long exec).
|
|
@@ -4674,6 +4698,11 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
4674
4698
|
} : null;
|
|
4675
4699
|
},
|
|
4676
4700
|
controlRpcFactory: controlRpcFactory(bus),
|
|
4701
|
+
resolveSelfhostedOpStream: async (sandbox) => {
|
|
4702
|
+
if (!sandbox.enrollmentId) return void 0;
|
|
4703
|
+
const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
|
|
4704
|
+
return opStreamDepsFor(services, enrollment?.opStream === true);
|
|
4705
|
+
},
|
|
4677
4706
|
relay: relayConfigFromSettings(settings),
|
|
4678
4707
|
...selfhostedResolverTimeouts(settings),
|
|
4679
4708
|
...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
|
|
@@ -4778,7 +4807,7 @@ function opStreamDepsFor(services, machineAdvertisesOpStream) {
|
|
|
4778
4807
|
async function establishSelfhostedTurnSession(services, args) {
|
|
4779
4808
|
const { settings, bus, onOp } = services;
|
|
4780
4809
|
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
4781
|
-
const opStream = opStreamDepsFor(services,
|
|
4810
|
+
const opStream = opStreamDepsFor(services, args.opStream);
|
|
4782
4811
|
const { client, session } = await buildSelfhostedBackendSession({
|
|
4783
4812
|
workspaceId: args.workspaceId,
|
|
4784
4813
|
agentId: args.agentId,
|
|
@@ -4794,8 +4823,8 @@ async function establishSelfhostedTurnSession(services, args) {
|
|
|
4794
4823
|
// Meter every control op (out-of-band telemetry) — no-op when unwired.
|
|
4795
4824
|
...onOp !== void 0 ? { onOp } : {},
|
|
4796
4825
|
// The streaming exec transport — present iff the machine advertised the
|
|
4797
|
-
// capability AND the server flag is on
|
|
4798
|
-
// legacy
|
|
4826
|
+
// capability AND the server flag is on. It is required when execTimeoutMs=0;
|
|
4827
|
+
// legacy request/reply remains available only for explicitly bounded exec.
|
|
4799
4828
|
...opStream !== void 0 ? { opStream } : {}
|
|
4800
4829
|
});
|
|
4801
4830
|
return {
|
|
@@ -4846,11 +4875,11 @@ function eventAttributes(attributes) {
|
|
|
4846
4875
|
if (!attributes) {
|
|
4847
4876
|
return void 0;
|
|
4848
4877
|
}
|
|
4849
|
-
const
|
|
4878
|
+
const projected = {};
|
|
4850
4879
|
for (const [key, value] of Object.entries(attributes)) {
|
|
4851
|
-
|
|
4880
|
+
projected[key] = eventAttributeValue(value);
|
|
4852
4881
|
}
|
|
4853
|
-
return
|
|
4882
|
+
return projected;
|
|
4854
4883
|
}
|
|
4855
4884
|
function eventAttributeValue(value) {
|
|
4856
4885
|
if (value === null || value === void 0 || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
@@ -5343,6 +5372,22 @@ function recordBatchFlush(observability, input) {
|
|
|
5343
5372
|
value: input.durationSeconds
|
|
5344
5373
|
});
|
|
5345
5374
|
}
|
|
5375
|
+
function recordSessionEventAppendLatency(observability, input) {
|
|
5376
|
+
observability.observeHistogram({
|
|
5377
|
+
name: "opengeni_session_event_append_seconds",
|
|
5378
|
+
help: "Duration in seconds of an appendSessionEvents DB write (the durable write path).",
|
|
5379
|
+
buckets: STREAM_IO_BUCKETS,
|
|
5380
|
+
value: input.durationSeconds
|
|
5381
|
+
});
|
|
5382
|
+
}
|
|
5383
|
+
function recordSessionEventPublishLatency(observability, input) {
|
|
5384
|
+
observability.observeHistogram({
|
|
5385
|
+
name: "opengeni_session_event_publish_seconds",
|
|
5386
|
+
help: "Duration in seconds of the best-effort NATS live fan-out publish (the delivery path).",
|
|
5387
|
+
buckets: STREAM_IO_BUCKETS,
|
|
5388
|
+
value: input.durationSeconds
|
|
5389
|
+
});
|
|
5390
|
+
}
|
|
5346
5391
|
var MODEL_INPUT_TOKENS_BUCKETS = [
|
|
5347
5392
|
1e3,
|
|
5348
5393
|
5e3,
|
|
@@ -5628,12 +5673,667 @@ async function discardUnpublishedRecording(args) {
|
|
|
5628
5673
|
}
|
|
5629
5674
|
function scrubFreeText(value) {
|
|
5630
5675
|
if (value === null || value === void 0) return null;
|
|
5631
|
-
const
|
|
5632
|
-
return
|
|
5676
|
+
const exact = String(value);
|
|
5677
|
+
return exact.length === 0 ? null : exact;
|
|
5633
5678
|
}
|
|
5634
5679
|
|
|
5635
|
-
// src/activities/
|
|
5680
|
+
// src/activities/retained-screenshots.ts
|
|
5681
|
+
import {
|
|
5682
|
+
COMPUTER_SCREENSHOT_MAX_BYTES,
|
|
5683
|
+
COMPUTER_SCREENSHOT_MAX_DIMENSION,
|
|
5684
|
+
COMPUTER_SCREENSHOT_MAX_PIXELS,
|
|
5685
|
+
COMPUTER_SCREENSHOT_RETENTION_MS,
|
|
5686
|
+
COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES,
|
|
5687
|
+
RetainedArtifactMetadataSchema,
|
|
5688
|
+
retainedScreenshotReferenceFromFile
|
|
5689
|
+
} from "@opengeni/contracts";
|
|
5690
|
+
import {
|
|
5691
|
+
RetainedScreenshotQuotaExceededError,
|
|
5692
|
+
getRetainedScreenshotArtifact,
|
|
5693
|
+
prepareRetainedScreenshotArtifact,
|
|
5694
|
+
recordRetainedScreenshotArtifactError,
|
|
5695
|
+
settleRetainedScreenshotArtifactReady
|
|
5696
|
+
} from "@opengeni/db";
|
|
5636
5697
|
import { createHash as createHash3 } from "crypto";
|
|
5698
|
+
var PNG_SIGNATURE = Uint8Array.of(137, 80, 78, 71, 13, 10, 26, 10);
|
|
5699
|
+
var PNG_ALLOWED_CRITICAL_CHUNKS = /* @__PURE__ */ new Set(["IHDR", "PLTE", "IDAT", "IEND"]);
|
|
5700
|
+
var PNG_MAX_CHUNKS = 16384;
|
|
5701
|
+
var RETAINED_IMAGE_MARKER = "retained_artifact";
|
|
5702
|
+
var ScreenshotValidationError = class extends Error {
|
|
5703
|
+
constructor(reason, message) {
|
|
5704
|
+
super(message);
|
|
5705
|
+
this.reason = reason;
|
|
5706
|
+
this.name = "ScreenshotValidationError";
|
|
5707
|
+
}
|
|
5708
|
+
};
|
|
5709
|
+
var RetainedScreenshotUnavailableError = class extends Error {
|
|
5710
|
+
constructor(artifactId, reason) {
|
|
5711
|
+
super(`Retained screenshot ${artifactId} is unavailable: ${reason}`);
|
|
5712
|
+
this.artifactId = artifactId;
|
|
5713
|
+
this.reason = reason;
|
|
5714
|
+
this.name = "RetainedScreenshotUnavailableError";
|
|
5715
|
+
}
|
|
5716
|
+
};
|
|
5717
|
+
function typedScreenshotFromSdkEvent(event) {
|
|
5718
|
+
if (!event || typeof event !== "object") return null;
|
|
5719
|
+
const streamEvent = event;
|
|
5720
|
+
if (streamEvent.type !== "run_item_stream_event") return null;
|
|
5721
|
+
const item = streamEvent.item;
|
|
5722
|
+
if (!item || item.type !== "tool_call_output_item") return null;
|
|
5723
|
+
const output = item.output;
|
|
5724
|
+
if (!output || typeof output !== "object" || Array.isArray(output)) return null;
|
|
5725
|
+
const outputRecord = output;
|
|
5726
|
+
if (outputRecord.type !== "image" || !outputRecord.image || typeof outputRecord.image !== "object") {
|
|
5727
|
+
return null;
|
|
5728
|
+
}
|
|
5729
|
+
const image = outputRecord.image;
|
|
5730
|
+
if (!(image.data instanceof Uint8Array) || typeof image.mediaType !== "string") return null;
|
|
5731
|
+
const raw = item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem) ? item.rawItem : {};
|
|
5732
|
+
const callId = raw.callId ?? raw.call_id;
|
|
5733
|
+
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
5734
|
+
const toolOutputId = raw.id ?? item.id ?? callId;
|
|
5735
|
+
if (typeof toolOutputId !== "string" || toolOutputId.length === 0) return null;
|
|
5736
|
+
return {
|
|
5737
|
+
callId,
|
|
5738
|
+
toolOutputId,
|
|
5739
|
+
bytes: image.data,
|
|
5740
|
+
mediaType: image.mediaType
|
|
5741
|
+
};
|
|
5742
|
+
}
|
|
5743
|
+
function validateComputerScreenshot(input, limits = {}) {
|
|
5744
|
+
const maxBytes = limits.maxBytes ?? COMPUTER_SCREENSHOT_MAX_BYTES;
|
|
5745
|
+
const maxDimension = limits.maxDimension ?? COMPUTER_SCREENSHOT_MAX_DIMENSION;
|
|
5746
|
+
const maxPixels = limits.maxPixels ?? COMPUTER_SCREENSHOT_MAX_PIXELS;
|
|
5747
|
+
if (input.mediaType !== "image/png") {
|
|
5748
|
+
throw new ScreenshotValidationError(
|
|
5749
|
+
"unsupported",
|
|
5750
|
+
"computer screenshot MIME must be image/png"
|
|
5751
|
+
);
|
|
5752
|
+
}
|
|
5753
|
+
if (input.bytes.byteLength === 0) {
|
|
5754
|
+
throw new ScreenshotValidationError("invalid_content", "computer screenshot is empty");
|
|
5755
|
+
}
|
|
5756
|
+
if (input.bytes.byteLength > maxBytes) {
|
|
5757
|
+
throw new ScreenshotValidationError(
|
|
5758
|
+
"oversized",
|
|
5759
|
+
`computer screenshot exceeds ${maxBytes} bytes`
|
|
5760
|
+
);
|
|
5761
|
+
}
|
|
5762
|
+
if (input.bytes.byteLength < PNG_SIGNATURE.byteLength + 25) {
|
|
5763
|
+
throw new ScreenshotValidationError("invalid_content", "computer screenshot PNG is truncated");
|
|
5764
|
+
}
|
|
5765
|
+
for (let index = 0; index < PNG_SIGNATURE.byteLength; index += 1) {
|
|
5766
|
+
if (input.bytes[index] !== PNG_SIGNATURE[index]) {
|
|
5767
|
+
throw new ScreenshotValidationError(
|
|
5768
|
+
"invalid_content",
|
|
5769
|
+
"computer screenshot PNG signature is invalid"
|
|
5770
|
+
);
|
|
5771
|
+
}
|
|
5772
|
+
}
|
|
5773
|
+
let offset = PNG_SIGNATURE.byteLength;
|
|
5774
|
+
let chunks = 0;
|
|
5775
|
+
let width = 0;
|
|
5776
|
+
let height = 0;
|
|
5777
|
+
let sawIhdr = false;
|
|
5778
|
+
let sawIdat = false;
|
|
5779
|
+
let endedIdat = false;
|
|
5780
|
+
let sawIend = false;
|
|
5781
|
+
while (offset < input.bytes.byteLength) {
|
|
5782
|
+
chunks += 1;
|
|
5783
|
+
if (chunks > PNG_MAX_CHUNKS || offset + 12 > input.bytes.byteLength) {
|
|
5784
|
+
throw new ScreenshotValidationError(
|
|
5785
|
+
"invalid_content",
|
|
5786
|
+
"computer screenshot PNG chunk table is invalid"
|
|
5787
|
+
);
|
|
5788
|
+
}
|
|
5789
|
+
const length = readUint32(input.bytes, offset);
|
|
5790
|
+
const typeOffset = offset + 4;
|
|
5791
|
+
const dataOffset = typeOffset + 4;
|
|
5792
|
+
const dataEnd = dataOffset + length;
|
|
5793
|
+
const crcOffset = dataEnd;
|
|
5794
|
+
const nextOffset = crcOffset + 4;
|
|
5795
|
+
if (!Number.isSafeInteger(nextOffset) || nextOffset > input.bytes.byteLength) {
|
|
5796
|
+
throw new ScreenshotValidationError(
|
|
5797
|
+
"invalid_content",
|
|
5798
|
+
"computer screenshot PNG chunk is truncated"
|
|
5799
|
+
);
|
|
5800
|
+
}
|
|
5801
|
+
const type = asciiChunkType(input.bytes, typeOffset);
|
|
5802
|
+
const expectedCrc = readUint32(input.bytes, crcOffset);
|
|
5803
|
+
const actualCrc = crc32(input.bytes.subarray(typeOffset, dataEnd));
|
|
5804
|
+
if (expectedCrc !== actualCrc) {
|
|
5805
|
+
throw new ScreenshotValidationError(
|
|
5806
|
+
"invalid_content",
|
|
5807
|
+
`computer screenshot PNG ${type} CRC is invalid`
|
|
5808
|
+
);
|
|
5809
|
+
}
|
|
5810
|
+
if (chunks === 1 && type !== "IHDR") {
|
|
5811
|
+
throw new ScreenshotValidationError(
|
|
5812
|
+
"invalid_content",
|
|
5813
|
+
"computer screenshot PNG must begin with IHDR"
|
|
5814
|
+
);
|
|
5815
|
+
}
|
|
5816
|
+
if (isCriticalChunk(type) && !PNG_ALLOWED_CRITICAL_CHUNKS.has(type)) {
|
|
5817
|
+
throw new ScreenshotValidationError(
|
|
5818
|
+
"invalid_content",
|
|
5819
|
+
`computer screenshot PNG has unsupported critical chunk ${type}`
|
|
5820
|
+
);
|
|
5821
|
+
}
|
|
5822
|
+
if (type === "IHDR") {
|
|
5823
|
+
if (sawIhdr || length !== 13) {
|
|
5824
|
+
throw new ScreenshotValidationError(
|
|
5825
|
+
"invalid_content",
|
|
5826
|
+
"computer screenshot PNG IHDR is invalid"
|
|
5827
|
+
);
|
|
5828
|
+
}
|
|
5829
|
+
sawIhdr = true;
|
|
5830
|
+
width = readUint32(input.bytes, dataOffset);
|
|
5831
|
+
height = readUint32(input.bytes, dataOffset + 4);
|
|
5832
|
+
const bitDepth = input.bytes[dataOffset + 8];
|
|
5833
|
+
const colorType = input.bytes[dataOffset + 9];
|
|
5834
|
+
const compression = input.bytes[dataOffset + 10];
|
|
5835
|
+
const filter = input.bytes[dataOffset + 11];
|
|
5836
|
+
const interlace = input.bytes[dataOffset + 12];
|
|
5837
|
+
if (!validPngColorMode(bitDepth, colorType) || compression !== 0 || filter !== 0 || interlace !== 0 && interlace !== 1) {
|
|
5838
|
+
throw new ScreenshotValidationError(
|
|
5839
|
+
"invalid_content",
|
|
5840
|
+
"computer screenshot PNG IHDR fields are invalid"
|
|
5841
|
+
);
|
|
5842
|
+
}
|
|
5843
|
+
if (width <= 0 || height <= 0 || width > maxDimension || height > maxDimension || width * height > maxPixels) {
|
|
5844
|
+
throw new ScreenshotValidationError(
|
|
5845
|
+
"oversized",
|
|
5846
|
+
"computer screenshot dimensions exceed policy"
|
|
5847
|
+
);
|
|
5848
|
+
}
|
|
5849
|
+
} else if (type === "IDAT") {
|
|
5850
|
+
if (!sawIhdr || sawIend || endedIdat || length === 0) {
|
|
5851
|
+
throw new ScreenshotValidationError(
|
|
5852
|
+
"invalid_content",
|
|
5853
|
+
"computer screenshot PNG IDAT ordering is invalid"
|
|
5854
|
+
);
|
|
5855
|
+
}
|
|
5856
|
+
sawIdat = true;
|
|
5857
|
+
} else if (sawIdat && type !== "IEND") {
|
|
5858
|
+
endedIdat = true;
|
|
5859
|
+
}
|
|
5860
|
+
if (type === "IEND") {
|
|
5861
|
+
if (!sawIhdr || !sawIdat || sawIend || length !== 0) {
|
|
5862
|
+
throw new ScreenshotValidationError(
|
|
5863
|
+
"invalid_content",
|
|
5864
|
+
"computer screenshot PNG IEND is invalid"
|
|
5865
|
+
);
|
|
5866
|
+
}
|
|
5867
|
+
sawIend = true;
|
|
5868
|
+
if (nextOffset !== input.bytes.byteLength) {
|
|
5869
|
+
throw new ScreenshotValidationError(
|
|
5870
|
+
"invalid_content",
|
|
5871
|
+
"computer screenshot PNG has trailing/polyglot bytes"
|
|
5872
|
+
);
|
|
5873
|
+
}
|
|
5874
|
+
}
|
|
5875
|
+
offset = nextOffset;
|
|
5876
|
+
}
|
|
5877
|
+
if (!sawIhdr || !sawIdat || !sawIend || offset !== input.bytes.byteLength) {
|
|
5878
|
+
throw new ScreenshotValidationError("invalid_content", "computer screenshot PNG is incomplete");
|
|
5879
|
+
}
|
|
5880
|
+
return {
|
|
5881
|
+
bytes: input.bytes,
|
|
5882
|
+
mediaType: "image/png",
|
|
5883
|
+
sizeBytes: input.bytes.byteLength,
|
|
5884
|
+
sha256: createHash3("sha256").update(input.bytes).digest("hex"),
|
|
5885
|
+
width,
|
|
5886
|
+
height
|
|
5887
|
+
};
|
|
5888
|
+
}
|
|
5889
|
+
function retainedScreenshotIdentity(input) {
|
|
5890
|
+
const settlementKey = createHash3("sha256").update("opengeni:computer-screenshot:v1\0").update(input.sessionId).update("\0").update(input.turnId).update("\0").update(input.attemptId).update("\0").update(input.toolCallId).update("\0").update(input.toolOutputId).digest("hex");
|
|
5891
|
+
const uuidBytes = Uint8Array.from(Buffer.from(settlementKey.slice(0, 32), "hex"));
|
|
5892
|
+
uuidBytes[6] = uuidBytes[6] & 15 | 128;
|
|
5893
|
+
uuidBytes[8] = uuidBytes[8] & 63 | 128;
|
|
5894
|
+
const hex = Buffer.from(uuidBytes).toString("hex");
|
|
5895
|
+
return {
|
|
5896
|
+
artifactId: `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`,
|
|
5897
|
+
settlementKey
|
|
5898
|
+
};
|
|
5899
|
+
}
|
|
5900
|
+
async function retainComputerScreenshot(input) {
|
|
5901
|
+
const identity = retainedScreenshotIdentity({
|
|
5902
|
+
sessionId: input.sessionId,
|
|
5903
|
+
turnId: input.turnId,
|
|
5904
|
+
attemptId: input.attemptId,
|
|
5905
|
+
toolCallId: input.output.callId,
|
|
5906
|
+
toolOutputId: input.output.toolOutputId
|
|
5907
|
+
});
|
|
5908
|
+
let screenshot;
|
|
5909
|
+
try {
|
|
5910
|
+
screenshot = validateComputerScreenshot(input.output);
|
|
5911
|
+
} catch (error) {
|
|
5912
|
+
if (error instanceof ScreenshotValidationError) {
|
|
5913
|
+
return unavailable(identity.artifactId, error.reason);
|
|
5914
|
+
}
|
|
5915
|
+
throw error;
|
|
5916
|
+
}
|
|
5917
|
+
if (!input.objectStorage) return unavailable(identity.artifactId, "missing_storage");
|
|
5918
|
+
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
5919
|
+
const retentionExpiresAt = new Date(
|
|
5920
|
+
now.getTime() + (input.retentionMs ?? COMPUTER_SCREENSHOT_RETENTION_MS)
|
|
5921
|
+
);
|
|
5922
|
+
const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/computer-screenshot.png`;
|
|
5923
|
+
let prepared;
|
|
5924
|
+
try {
|
|
5925
|
+
prepared = (await prepareRetainedScreenshotArtifact(input.db, {
|
|
5926
|
+
artifactId: identity.artifactId,
|
|
5927
|
+
accountId: input.accountId,
|
|
5928
|
+
workspaceId: input.workspaceId,
|
|
5929
|
+
sessionId: input.sessionId,
|
|
5930
|
+
turnId: input.turnId,
|
|
5931
|
+
attemptId: input.attemptId,
|
|
5932
|
+
settlementKey: identity.settlementKey,
|
|
5933
|
+
toolCallId: input.output.callId,
|
|
5934
|
+
toolOutputId: input.output.toolOutputId,
|
|
5935
|
+
mediaType: screenshot.mediaType,
|
|
5936
|
+
sizeBytes: screenshot.sizeBytes,
|
|
5937
|
+
sha256: screenshot.sha256,
|
|
5938
|
+
width: screenshot.width,
|
|
5939
|
+
height: screenshot.height,
|
|
5940
|
+
retentionExpiresAt,
|
|
5941
|
+
bucket: input.objectStorage.bucket,
|
|
5942
|
+
objectKey,
|
|
5943
|
+
workspaceQuotaBytes: input.workspaceQuotaBytes ?? COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES
|
|
5944
|
+
})).artifact;
|
|
5945
|
+
} catch (error) {
|
|
5946
|
+
if (error instanceof RetainedScreenshotQuotaExceededError) {
|
|
5947
|
+
return unavailable(identity.artifactId, "quota_exceeded");
|
|
5948
|
+
}
|
|
5949
|
+
throw error;
|
|
5950
|
+
}
|
|
5951
|
+
if (prepared.status === "ready") {
|
|
5952
|
+
return await verifyReadyArtifact(input.objectStorage, prepared) ? reference(prepared) : unavailable(identity.artifactId, "missing_storage");
|
|
5953
|
+
}
|
|
5954
|
+
if (prepared.status !== "pending" && prepared.status !== "reconciling") {
|
|
5955
|
+
return unavailable(identity.artifactId, unavailableReasonForStatus(prepared.status));
|
|
5956
|
+
}
|
|
5957
|
+
try {
|
|
5958
|
+
await input.objectStorage.putObject({
|
|
5959
|
+
key: objectKey,
|
|
5960
|
+
contentType: screenshot.mediaType,
|
|
5961
|
+
body: screenshot.bytes,
|
|
5962
|
+
sha256: screenshot.sha256
|
|
5963
|
+
});
|
|
5964
|
+
const head = await input.objectStorage.headFile(prepared.file);
|
|
5965
|
+
assertStoredScreenshotHead(head, screenshot);
|
|
5966
|
+
} catch (error) {
|
|
5967
|
+
await recordRetainedScreenshotArtifactError(input.db, {
|
|
5968
|
+
accountId: input.accountId,
|
|
5969
|
+
workspaceId: input.workspaceId,
|
|
5970
|
+
artifactId: identity.artifactId,
|
|
5971
|
+
settlementKey: identity.settlementKey,
|
|
5972
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5973
|
+
}).catch(() => void 0);
|
|
5974
|
+
return unavailable(identity.artifactId, "storage_write_failed");
|
|
5975
|
+
}
|
|
5976
|
+
try {
|
|
5977
|
+
const settled = await settleRetainedScreenshotArtifactReady(input.db, {
|
|
5978
|
+
accountId: input.accountId,
|
|
5979
|
+
workspaceId: input.workspaceId,
|
|
5980
|
+
artifactId: identity.artifactId,
|
|
5981
|
+
settlementKey: identity.settlementKey
|
|
5982
|
+
});
|
|
5983
|
+
return reference(settled);
|
|
5984
|
+
} catch (error) {
|
|
5985
|
+
const stillReferenced = await getRetainedScreenshotArtifact(
|
|
5986
|
+
input.db,
|
|
5987
|
+
input.workspaceId,
|
|
5988
|
+
input.sessionId,
|
|
5989
|
+
identity.artifactId
|
|
5990
|
+
).catch(() => void 0);
|
|
5991
|
+
if (stillReferenced?.status === "ready") {
|
|
5992
|
+
return await verifyReadyArtifact(input.objectStorage, stillReferenced) ? reference(stillReferenced) : unavailable(identity.artifactId, "missing_storage");
|
|
5993
|
+
}
|
|
5994
|
+
if (stillReferenced === null || stillReferenced?.status === "cleanup_queued" || stillReferenced?.status === "cleanup_pending" || stillReferenced?.status === "failed" || stillReferenced?.status === "expired" || stillReferenced?.status === "deleted") {
|
|
5995
|
+
await input.objectStorage.deleteObject(objectKey).catch(() => void 0);
|
|
5996
|
+
} else if (stillReferenced) {
|
|
5997
|
+
await recordRetainedScreenshotArtifactError(input.db, {
|
|
5998
|
+
accountId: input.accountId,
|
|
5999
|
+
workspaceId: input.workspaceId,
|
|
6000
|
+
artifactId: identity.artifactId,
|
|
6001
|
+
settlementKey: identity.settlementKey,
|
|
6002
|
+
error: error instanceof Error ? error.message : String(error)
|
|
6003
|
+
}).catch(() => void 0);
|
|
6004
|
+
}
|
|
6005
|
+
return unavailable(identity.artifactId, "pending");
|
|
6006
|
+
}
|
|
6007
|
+
}
|
|
6008
|
+
function compactRetainedScreenshotHistory(history, receiptsByCallId) {
|
|
6009
|
+
if (receiptsByCallId.size === 0) return history;
|
|
6010
|
+
return history.map((item) => {
|
|
6011
|
+
const callId = historyCallId(item);
|
|
6012
|
+
const receipt = callId ? receiptsByCallId.get(callId) : void 0;
|
|
6013
|
+
if (!receipt || !Array.isArray(item.output)) return item;
|
|
6014
|
+
let changed = false;
|
|
6015
|
+
const output = item.output.map((entry) => {
|
|
6016
|
+
if (!isInlineImageContent(entry)) return entry;
|
|
6017
|
+
changed = true;
|
|
6018
|
+
return {
|
|
6019
|
+
...entry,
|
|
6020
|
+
image: { type: RETAINED_IMAGE_MARKER, artifact: receipt }
|
|
6021
|
+
};
|
|
6022
|
+
});
|
|
6023
|
+
return changed ? { ...item, output } : item;
|
|
6024
|
+
});
|
|
6025
|
+
}
|
|
6026
|
+
function collectRetainedScreenshotReceipts(history, target = /* @__PURE__ */ new Map()) {
|
|
6027
|
+
for (const item of history) {
|
|
6028
|
+
const callId = historyCallId(item);
|
|
6029
|
+
if (!callId || !Array.isArray(item.output)) continue;
|
|
6030
|
+
for (const entry of item.output) {
|
|
6031
|
+
const receipt = retainedReceiptFromImageContent(entry);
|
|
6032
|
+
if (receipt) {
|
|
6033
|
+
target.set(callId, receipt);
|
|
6034
|
+
break;
|
|
6035
|
+
}
|
|
6036
|
+
}
|
|
6037
|
+
}
|
|
6038
|
+
return target;
|
|
6039
|
+
}
|
|
6040
|
+
function compactRetainedScreenshotRunState(serialized, receiptsByCallId) {
|
|
6041
|
+
if (receiptsByCallId.size === 0) return serialized;
|
|
6042
|
+
const parsed = parseSerializedRunState(serialized);
|
|
6043
|
+
if (!parsed) return serialized;
|
|
6044
|
+
let changed = false;
|
|
6045
|
+
visitSerializedRunStateItemArrays(parsed, (items) => {
|
|
6046
|
+
const compacted = compactRetainedScreenshotHistory(items, receiptsByCallId);
|
|
6047
|
+
if (compacted !== items && compacted.some((item, index) => item !== items[index])) {
|
|
6048
|
+
items.splice(0, items.length, ...compacted);
|
|
6049
|
+
changed = true;
|
|
6050
|
+
}
|
|
6051
|
+
});
|
|
6052
|
+
return changed ? JSON.stringify(parsed) : serialized;
|
|
6053
|
+
}
|
|
6054
|
+
function collectRetainedScreenshotRunStateReceipts(serialized, target = /* @__PURE__ */ new Map()) {
|
|
6055
|
+
const parsed = parseSerializedRunState(serialized);
|
|
6056
|
+
if (!parsed) return target;
|
|
6057
|
+
visitSerializedRunStateItemArrays(parsed, (items) => {
|
|
6058
|
+
collectRetainedScreenshotReceipts(items, target);
|
|
6059
|
+
});
|
|
6060
|
+
return target;
|
|
6061
|
+
}
|
|
6062
|
+
async function materializeRetainedScreenshotHistory(input) {
|
|
6063
|
+
return await materializeRetainedScreenshotHistoryWithCache(input, /* @__PURE__ */ new Map());
|
|
6064
|
+
}
|
|
6065
|
+
async function materializeRetainedScreenshotRunState(input) {
|
|
6066
|
+
const parsed = parseSerializedRunState(input.serialized);
|
|
6067
|
+
if (!parsed) return input.serialized;
|
|
6068
|
+
const cache = /* @__PURE__ */ new Map();
|
|
6069
|
+
let changed = false;
|
|
6070
|
+
await visitSerializedRunStateItemArraysAsync(parsed, async (items) => {
|
|
6071
|
+
const materialized = await materializeRetainedScreenshotHistoryWithCache(
|
|
6072
|
+
{ ...input, history: items },
|
|
6073
|
+
cache
|
|
6074
|
+
);
|
|
6075
|
+
if (materialized.some((item, index) => item !== items[index])) {
|
|
6076
|
+
items.splice(0, items.length, ...materialized);
|
|
6077
|
+
changed = true;
|
|
6078
|
+
}
|
|
6079
|
+
});
|
|
6080
|
+
return changed ? JSON.stringify(parsed) : input.serialized;
|
|
6081
|
+
}
|
|
6082
|
+
async function materializeRetainedScreenshotHistoryWithCache(input, cache) {
|
|
6083
|
+
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
6084
|
+
const materializeEntry = async (entry) => {
|
|
6085
|
+
const receipt = retainedReceiptFromImageContent(entry);
|
|
6086
|
+
if (!receipt) return entry;
|
|
6087
|
+
if (!receipt.available) {
|
|
6088
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, receipt.reason);
|
|
6089
|
+
}
|
|
6090
|
+
let dataUrl = cache.get(receipt.artifactId);
|
|
6091
|
+
if (!dataUrl) {
|
|
6092
|
+
if (!input.objectStorage) {
|
|
6093
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
|
|
6094
|
+
}
|
|
6095
|
+
const artifact = await getRetainedScreenshotArtifact(
|
|
6096
|
+
input.db,
|
|
6097
|
+
input.workspaceId,
|
|
6098
|
+
input.sessionId,
|
|
6099
|
+
receipt.artifactId
|
|
6100
|
+
);
|
|
6101
|
+
if (!artifact) throw new RetainedScreenshotUnavailableError(receipt.artifactId, "deleted");
|
|
6102
|
+
if (artifact.status !== "ready") {
|
|
6103
|
+
throw new RetainedScreenshotUnavailableError(
|
|
6104
|
+
receipt.artifactId,
|
|
6105
|
+
unavailableReasonForStatus(artifact.status)
|
|
6106
|
+
);
|
|
6107
|
+
}
|
|
6108
|
+
if (artifact.retentionExpiresAt.getTime() <= now.getTime()) {
|
|
6109
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "expired");
|
|
6110
|
+
}
|
|
6111
|
+
const bytes = await input.objectStorage.getFileBytes(artifact.file).catch(() => null);
|
|
6112
|
+
if (!bytes)
|
|
6113
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
|
|
6114
|
+
const validated = validateComputerScreenshot({
|
|
6115
|
+
bytes,
|
|
6116
|
+
mediaType: artifact.mediaType
|
|
6117
|
+
});
|
|
6118
|
+
if (validated.sizeBytes !== artifact.sizeBytes || validated.sha256 !== artifact.sha256 || validated.width !== artifact.width || validated.height !== artifact.height || receipt.sha256 !== artifact.sha256 || receipt.originalBytes !== artifact.sizeBytes) {
|
|
6119
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "invalid_content");
|
|
6120
|
+
}
|
|
6121
|
+
dataUrl = `data:${validated.mediaType};base64,${Buffer.from(validated.bytes).toString("base64")}`;
|
|
6122
|
+
cache.set(receipt.artifactId, dataUrl);
|
|
6123
|
+
}
|
|
6124
|
+
return { ...entry, image: dataUrl };
|
|
6125
|
+
};
|
|
6126
|
+
const materialized = [];
|
|
6127
|
+
for (const item of input.history) {
|
|
6128
|
+
if (!Array.isArray(item.output)) {
|
|
6129
|
+
materialized.push(item);
|
|
6130
|
+
continue;
|
|
6131
|
+
}
|
|
6132
|
+
let changed = false;
|
|
6133
|
+
const output = [];
|
|
6134
|
+
for (const entry of item.output) {
|
|
6135
|
+
const next = await materializeEntry(entry);
|
|
6136
|
+
changed ||= next !== entry;
|
|
6137
|
+
output.push(next);
|
|
6138
|
+
}
|
|
6139
|
+
materialized.push(changed ? { ...item, output } : item);
|
|
6140
|
+
}
|
|
6141
|
+
return materialized;
|
|
6142
|
+
}
|
|
6143
|
+
function parseSerializedRunState(serialized) {
|
|
6144
|
+
try {
|
|
6145
|
+
const parsed = JSON.parse(serialized);
|
|
6146
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
6147
|
+
} catch {
|
|
6148
|
+
return null;
|
|
6149
|
+
}
|
|
6150
|
+
}
|
|
6151
|
+
function visitSerializedRunStateItemArrays(root, visitor) {
|
|
6152
|
+
const originalInput = root.originalInput;
|
|
6153
|
+
if (Array.isArray(originalInput)) {
|
|
6154
|
+
visitor(originalInput);
|
|
6155
|
+
}
|
|
6156
|
+
if (Array.isArray(root.generatedItems)) {
|
|
6157
|
+
for (const wrapper of root.generatedItems) {
|
|
6158
|
+
if (!wrapper || typeof wrapper !== "object" || Array.isArray(wrapper)) continue;
|
|
6159
|
+
const record = wrapper;
|
|
6160
|
+
const rawItem = record.rawItem;
|
|
6161
|
+
if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
|
|
6162
|
+
const items = [rawItem];
|
|
6163
|
+
visitor(items);
|
|
6164
|
+
if (items[0] !== rawItem) record.rawItem = items[0];
|
|
6165
|
+
}
|
|
6166
|
+
}
|
|
6167
|
+
if (Array.isArray(root.modelResponses)) {
|
|
6168
|
+
for (const response of root.modelResponses) {
|
|
6169
|
+
if (!response || typeof response !== "object" || Array.isArray(response)) continue;
|
|
6170
|
+
const output = response.output;
|
|
6171
|
+
if (Array.isArray(output)) visitor(output);
|
|
6172
|
+
}
|
|
6173
|
+
}
|
|
6174
|
+
const lastModelResponse = root.lastModelResponse;
|
|
6175
|
+
if (lastModelResponse && typeof lastModelResponse === "object" && !Array.isArray(lastModelResponse)) {
|
|
6176
|
+
const output = lastModelResponse.output;
|
|
6177
|
+
if (Array.isArray(output)) visitor(output);
|
|
6178
|
+
}
|
|
6179
|
+
}
|
|
6180
|
+
async function visitSerializedRunStateItemArraysAsync(root, visitor) {
|
|
6181
|
+
const originalInput = root.originalInput;
|
|
6182
|
+
if (Array.isArray(originalInput)) {
|
|
6183
|
+
await visitor(originalInput);
|
|
6184
|
+
}
|
|
6185
|
+
if (Array.isArray(root.generatedItems)) {
|
|
6186
|
+
for (const wrapper of root.generatedItems) {
|
|
6187
|
+
if (!wrapper || typeof wrapper !== "object" || Array.isArray(wrapper)) continue;
|
|
6188
|
+
const record = wrapper;
|
|
6189
|
+
const rawItem = record.rawItem;
|
|
6190
|
+
if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
|
|
6191
|
+
const items = [rawItem];
|
|
6192
|
+
await visitor(items);
|
|
6193
|
+
if (items[0] !== rawItem) record.rawItem = items[0];
|
|
6194
|
+
}
|
|
6195
|
+
}
|
|
6196
|
+
if (Array.isArray(root.modelResponses)) {
|
|
6197
|
+
for (const response of root.modelResponses) {
|
|
6198
|
+
if (!response || typeof response !== "object" || Array.isArray(response)) continue;
|
|
6199
|
+
const output = response.output;
|
|
6200
|
+
if (Array.isArray(output)) await visitor(output);
|
|
6201
|
+
}
|
|
6202
|
+
}
|
|
6203
|
+
const lastModelResponse = root.lastModelResponse;
|
|
6204
|
+
if (lastModelResponse && typeof lastModelResponse === "object" && !Array.isArray(lastModelResponse)) {
|
|
6205
|
+
const output = lastModelResponse.output;
|
|
6206
|
+
if (Array.isArray(output)) await visitor(output);
|
|
6207
|
+
}
|
|
6208
|
+
}
|
|
6209
|
+
function reference(artifact) {
|
|
6210
|
+
if (!artifact.sessionId) {
|
|
6211
|
+
throw new Error(`Ready retained screenshot is detached: ${artifact.artifactId}`);
|
|
6212
|
+
}
|
|
6213
|
+
const value = retainedScreenshotReferenceFromFile({
|
|
6214
|
+
...artifact.file,
|
|
6215
|
+
sessionId: artifact.sessionId,
|
|
6216
|
+
width: artifact.width,
|
|
6217
|
+
height: artifact.height,
|
|
6218
|
+
expiresAt: artifact.retentionExpiresAt.toISOString()
|
|
6219
|
+
});
|
|
6220
|
+
if (!value)
|
|
6221
|
+
throw new Error(`Ready retained screenshot receipt is invalid: ${artifact.artifactId}`);
|
|
6222
|
+
return value;
|
|
6223
|
+
}
|
|
6224
|
+
function unavailable(artifactId, reason) {
|
|
6225
|
+
return RetainedArtifactMetadataSchema.parse({
|
|
6226
|
+
available: false,
|
|
6227
|
+
artifactId,
|
|
6228
|
+
reason
|
|
6229
|
+
});
|
|
6230
|
+
}
|
|
6231
|
+
function unavailableReasonForStatus(status) {
|
|
6232
|
+
switch (status) {
|
|
6233
|
+
case "pending":
|
|
6234
|
+
case "reconciling":
|
|
6235
|
+
return "pending";
|
|
6236
|
+
case "cleanup_queued":
|
|
6237
|
+
case "cleanup_pending":
|
|
6238
|
+
return "failed";
|
|
6239
|
+
case "failed":
|
|
6240
|
+
case "expired":
|
|
6241
|
+
case "deleted":
|
|
6242
|
+
return status;
|
|
6243
|
+
case "ready":
|
|
6244
|
+
return "missing_storage";
|
|
6245
|
+
}
|
|
6246
|
+
}
|
|
6247
|
+
async function verifyReadyArtifact(storage, artifact) {
|
|
6248
|
+
try {
|
|
6249
|
+
assertStoredScreenshotHead(await storage.headFile(artifact.file), {
|
|
6250
|
+
sizeBytes: artifact.sizeBytes,
|
|
6251
|
+
mediaType: "image/png",
|
|
6252
|
+
sha256: artifact.sha256
|
|
6253
|
+
});
|
|
6254
|
+
return true;
|
|
6255
|
+
} catch {
|
|
6256
|
+
return false;
|
|
6257
|
+
}
|
|
6258
|
+
}
|
|
6259
|
+
function assertStoredScreenshotHead(head, expected) {
|
|
6260
|
+
if (head.ContentLength !== expected.sizeBytes) {
|
|
6261
|
+
throw new Error("retained screenshot object size mismatch");
|
|
6262
|
+
}
|
|
6263
|
+
if (head.ContentType !== expected.mediaType) {
|
|
6264
|
+
throw new Error("retained screenshot object MIME mismatch");
|
|
6265
|
+
}
|
|
6266
|
+
if (head.Metadata?.sha256 !== expected.sha256) {
|
|
6267
|
+
throw new Error("retained screenshot object SHA-256 metadata mismatch");
|
|
6268
|
+
}
|
|
6269
|
+
}
|
|
6270
|
+
function retainedReceiptFromImageContent(entry) {
|
|
6271
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return null;
|
|
6272
|
+
const image = entry.image;
|
|
6273
|
+
if (!image || typeof image !== "object" || Array.isArray(image)) return null;
|
|
6274
|
+
const marker = image;
|
|
6275
|
+
if (marker.type !== RETAINED_IMAGE_MARKER) return null;
|
|
6276
|
+
const parsed = RetainedArtifactMetadataSchema.safeParse(marker.artifact);
|
|
6277
|
+
if (!parsed.success) return null;
|
|
6278
|
+
return parsed.data.available && parsed.data.kind !== "computer_screenshot" ? null : parsed.data;
|
|
6279
|
+
}
|
|
6280
|
+
function isInlineImageContent(entry) {
|
|
6281
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
|
|
6282
|
+
const record = entry;
|
|
6283
|
+
return record.type === "input_image" && typeof record.image === "string" && record.image.startsWith("data:image/png;base64,");
|
|
6284
|
+
}
|
|
6285
|
+
function historyCallId(item) {
|
|
6286
|
+
const value = item.callId ?? item.call_id;
|
|
6287
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
6288
|
+
}
|
|
6289
|
+
function readUint32(bytes, offset) {
|
|
6290
|
+
return (bytes[offset] << 24 | bytes[offset + 1] << 16 | bytes[offset + 2] << 8 | bytes[offset + 3]) >>> 0;
|
|
6291
|
+
}
|
|
6292
|
+
function asciiChunkType(bytes, offset) {
|
|
6293
|
+
const chars = [bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]];
|
|
6294
|
+
if (chars.some(
|
|
6295
|
+
(value) => value === void 0 || value < 65 || value > 90 && value < 97 || value > 122
|
|
6296
|
+
)) {
|
|
6297
|
+
throw new ScreenshotValidationError(
|
|
6298
|
+
"invalid_content",
|
|
6299
|
+
"computer screenshot PNG chunk type is invalid"
|
|
6300
|
+
);
|
|
6301
|
+
}
|
|
6302
|
+
return String.fromCharCode(...chars);
|
|
6303
|
+
}
|
|
6304
|
+
function isCriticalChunk(type) {
|
|
6305
|
+
return type.charCodeAt(0) >= 65 && type.charCodeAt(0) <= 90;
|
|
6306
|
+
}
|
|
6307
|
+
function validPngColorMode(bitDepth, colorType) {
|
|
6308
|
+
if (bitDepth === void 0 || colorType === void 0) return false;
|
|
6309
|
+
switch (colorType) {
|
|
6310
|
+
case 0:
|
|
6311
|
+
return [1, 2, 4, 8, 16].includes(bitDepth);
|
|
6312
|
+
case 2:
|
|
6313
|
+
case 4:
|
|
6314
|
+
case 6:
|
|
6315
|
+
return bitDepth === 8 || bitDepth === 16;
|
|
6316
|
+
case 3:
|
|
6317
|
+
return [1, 2, 4, 8].includes(bitDepth);
|
|
6318
|
+
default:
|
|
6319
|
+
return false;
|
|
6320
|
+
}
|
|
6321
|
+
}
|
|
6322
|
+
var CRC32_TABLE = Uint32Array.from({ length: 256 }, (_, value) => {
|
|
6323
|
+
let current = value;
|
|
6324
|
+
for (let bit = 0; bit < 8; bit += 1) {
|
|
6325
|
+
current = (current & 1) !== 0 ? 3988292384 ^ current >>> 1 : current >>> 1;
|
|
6326
|
+
}
|
|
6327
|
+
return current >>> 0;
|
|
6328
|
+
});
|
|
6329
|
+
function crc32(bytes) {
|
|
6330
|
+
let crc = 4294967295;
|
|
6331
|
+
for (const byte of bytes) crc = CRC32_TABLE[(crc ^ byte) & 255] ^ crc >>> 8;
|
|
6332
|
+
return (crc ^ 4294967295) >>> 0;
|
|
6333
|
+
}
|
|
6334
|
+
|
|
6335
|
+
// src/activities/workspace-capture.ts
|
|
6336
|
+
import { createHash as createHash4 } from "crypto";
|
|
5637
6337
|
import {
|
|
5638
6338
|
deleteWorkspaceCaptureRows,
|
|
5639
6339
|
insertFailedWorkspaceCapture,
|
|
@@ -5751,7 +6451,7 @@ async function readCaptureRepository(svc, root) {
|
|
|
5751
6451
|
return { complete: false, degradedReason: "repository_read_unavailable" };
|
|
5752
6452
|
}
|
|
5753
6453
|
try {
|
|
5754
|
-
const
|
|
6454
|
+
const workingRequest = svc.gitDiff({
|
|
5755
6455
|
path: root,
|
|
5756
6456
|
staged: false,
|
|
5757
6457
|
includeUntracked: true,
|
|
@@ -5760,7 +6460,24 @@ async function readCaptureRepository(svc, root) {
|
|
|
5760
6460
|
contextLines: 3,
|
|
5761
6461
|
maxBytesPerFile: PER_FILE_DIFF_GUARD_BYTES
|
|
5762
6462
|
});
|
|
5763
|
-
|
|
6463
|
+
const branchRequest = Promise.resolve().then(
|
|
6464
|
+
() => svc.gitDiff({
|
|
6465
|
+
path: root,
|
|
6466
|
+
staged: false,
|
|
6467
|
+
includeUntracked: true,
|
|
6468
|
+
fromRef: "origin/HEAD",
|
|
6469
|
+
pathspec: [],
|
|
6470
|
+
contextLines: 3,
|
|
6471
|
+
maxBytesPerFile: PER_FILE_DIFF_GUARD_BYTES
|
|
6472
|
+
})
|
|
6473
|
+
).then((value) => value).catch(() => null);
|
|
6474
|
+
const [diff, branchDiff] = await Promise.all([workingRequest, branchRequest]);
|
|
6475
|
+
return {
|
|
6476
|
+
complete: true,
|
|
6477
|
+
status,
|
|
6478
|
+
diff,
|
|
6479
|
+
...branchDiff ? { branchDiff } : {}
|
|
6480
|
+
};
|
|
5764
6481
|
} catch (error) {
|
|
5765
6482
|
const boxExiting = classifyCaptureEntryError(error);
|
|
5766
6483
|
if (boxExiting) throw boxExiting;
|
|
@@ -5913,7 +6630,7 @@ async function runCapture(input, ctx, startedAt, signal, stage) {
|
|
|
5913
6630
|
);
|
|
5914
6631
|
return;
|
|
5915
6632
|
}
|
|
5916
|
-
const { status, diff } = repository;
|
|
6633
|
+
const { status, diff, branchDiff } = repository;
|
|
5917
6634
|
const diffFiles = diff.files.filter((f) => !isUnderResidueDir(joinRepoPath(root, f.path)));
|
|
5918
6635
|
const statusFiles = status.files.filter((f) => !isUnderResidueDir(joinRepoPath(root, f.path)));
|
|
5919
6636
|
repos.push({
|
|
@@ -5924,7 +6641,12 @@ async function runCapture(input, ctx, startedAt, signal, stage) {
|
|
|
5924
6641
|
ahead: status.ahead,
|
|
5925
6642
|
behind: status.behind,
|
|
5926
6643
|
status: statusFiles,
|
|
5927
|
-
diff: diffFiles
|
|
6644
|
+
diff: diffFiles,
|
|
6645
|
+
...branchDiff ? {
|
|
6646
|
+
branchDiff: branchDiff.files.filter(
|
|
6647
|
+
(f) => !isUnderResidueDir(joinRepoPath(root, f.path))
|
|
6648
|
+
)
|
|
6649
|
+
} : {}
|
|
5928
6650
|
});
|
|
5929
6651
|
for (const f of diffFiles) {
|
|
5930
6652
|
additions += f.additions;
|
|
@@ -6335,7 +7057,7 @@ function joinRepoPath(repoRoot, repoRelPath) {
|
|
|
6335
7057
|
return `${repoRoot.replace(/\/+$/, "")}/${repoRelPath}`;
|
|
6336
7058
|
}
|
|
6337
7059
|
function sha256(bytes) {
|
|
6338
|
-
return
|
|
7060
|
+
return createHash4("sha256").update(bytes).digest("hex");
|
|
6339
7061
|
}
|
|
6340
7062
|
function utf8(s) {
|
|
6341
7063
|
return new TextEncoder().encode(s);
|
|
@@ -6415,11 +7137,21 @@ import {
|
|
|
6415
7137
|
DEFAULT_FIRST_PARTY_MCP_TOOLS,
|
|
6416
7138
|
evaluateWorkspaceModelPolicy as evaluateWorkspaceModelPolicy2,
|
|
6417
7139
|
readTurnExecutionPolicyV1,
|
|
7140
|
+
resolveWorkspaceAgentHumanInputEnabled,
|
|
6418
7141
|
resourceMountPath as resourceMountPath2
|
|
6419
7142
|
} from "@opengeni/contracts";
|
|
6420
|
-
import { createHash as
|
|
7143
|
+
import { createHash as createHash5, randomUUID as randomUUID2 } from "crypto";
|
|
6421
7144
|
var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
|
|
6422
7145
|
var PROVIDER_CONNECTIVITY_BACKOFF_MS = [2e3, 5e3, 15e3, 3e4, 6e4];
|
|
7146
|
+
var WorkspaceHumanInputDisabledError = class extends Error {
|
|
7147
|
+
constructor(kind) {
|
|
7148
|
+
super(`Workspace policy rejects structured human-input ${kind}`);
|
|
7149
|
+
this.name = "WorkspaceHumanInputDisabledError";
|
|
7150
|
+
}
|
|
7151
|
+
};
|
|
7152
|
+
function assertWorkspaceHumanInputAllowed(enabled, kind, attempted) {
|
|
7153
|
+
if (!enabled && attempted) throw new WorkspaceHumanInputDisabledError(kind);
|
|
7154
|
+
}
|
|
6423
7155
|
function credentialSubjectIdForTurnInitiator(turn) {
|
|
6424
7156
|
return directPersonalConnectionSubjectId(turn);
|
|
6425
7157
|
}
|
|
@@ -6539,10 +7271,10 @@ function filterUnmaterializedSandboxFileDownloads(downloads, materializedFileIds
|
|
|
6539
7271
|
return downloads.filter((download) => !materializedFileIds.has(download.fileId));
|
|
6540
7272
|
}
|
|
6541
7273
|
function codexWorkspaceMetricKey(workspaceId) {
|
|
6542
|
-
return
|
|
7274
|
+
return createHash5("sha256").update(workspaceId).digest("hex").slice(0, 12);
|
|
6543
7275
|
}
|
|
6544
7276
|
function stableHumanInputRequestId(sessionId, turnId, toolCallId) {
|
|
6545
|
-
const hex =
|
|
7277
|
+
const hex = createHash5("sha256").update("opengeni-human-input-v1\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
|
|
6546
7278
|
hex[12] = "5";
|
|
6547
7279
|
hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
|
|
6548
7280
|
const value = hex.join("");
|
|
@@ -6645,6 +7377,16 @@ async function waitForTurnStreamCleanup(batcherFlush, providerCompleted, signal)
|
|
|
6645
7377
|
await waitForTurnFinalizerStep(batcherFlush, signal);
|
|
6646
7378
|
await waitForTurnFinalizerStep(providerCompleted, signal);
|
|
6647
7379
|
}
|
|
7380
|
+
async function clearAttemptCredentialsWithSettledFence(input) {
|
|
7381
|
+
try {
|
|
7382
|
+
await input.runWorkspaceFencedClear();
|
|
7383
|
+
} catch (error) {
|
|
7384
|
+
const settledAttemptFence = error instanceof Error && error.name === "SandboxWorkspaceMutationFencedError" && error.code === "attempt_fenced" && (input.activityStatus === "idle" || input.activityStatus === "failed");
|
|
7385
|
+
if (!settledAttemptFence) throw error;
|
|
7386
|
+
input.onSettledAttemptFence();
|
|
7387
|
+
await input.clearExactAttempt();
|
|
7388
|
+
}
|
|
7389
|
+
}
|
|
6648
7390
|
function turnFinalizerCancellationSignal(temporalSignal, activityStatus) {
|
|
6649
7391
|
if (activityStatus !== "cancelled" || temporalSignal?.aborted) return temporalSignal;
|
|
6650
7392
|
const fenced = new AbortController();
|
|
@@ -6654,56 +7396,55 @@ function turnFinalizerCancellationSignal(temporalSignal, activityStatus) {
|
|
|
6654
7396
|
function compactionFailureReason(reason) {
|
|
6655
7397
|
return reason.startsWith("compaction summarization failed:") ? reason : `compaction summarization failed: ${reason}`;
|
|
6656
7398
|
}
|
|
6657
|
-
|
|
6658
|
-
|
|
6659
|
-
|
|
6660
|
-
|
|
7399
|
+
var MandatoryHistoryPersistenceError = class extends Error {
|
|
7400
|
+
constructor(stage, cause) {
|
|
7401
|
+
super(cause instanceof Error ? cause.message : String(cause));
|
|
7402
|
+
this.stage = stage;
|
|
7403
|
+
this.cause = cause;
|
|
7404
|
+
}
|
|
7405
|
+
name = "MandatoryHistoryPersistenceError";
|
|
7406
|
+
cause;
|
|
7407
|
+
};
|
|
7408
|
+
async function runMandatoryHistoryPersistenceStep(stage, operation) {
|
|
7409
|
+
try {
|
|
7410
|
+
return await operation();
|
|
7411
|
+
} catch (cause) {
|
|
7412
|
+
throw new MandatoryHistoryPersistenceError(stage, cause);
|
|
7413
|
+
}
|
|
7414
|
+
}
|
|
7415
|
+
function safeErrorDiagnostic(error) {
|
|
6661
7416
|
const diagnostic = {
|
|
6662
|
-
|
|
6663
|
-
|
|
7417
|
+
errorClass: "WorkerOperationError",
|
|
7418
|
+
errorCode: "worker_operation_failed",
|
|
7419
|
+
origin: "worker"
|
|
6664
7420
|
};
|
|
6665
|
-
|
|
6666
|
-
|
|
6667
|
-
|
|
6668
|
-
|
|
6669
|
-
|
|
6670
|
-
|
|
6671
|
-
|
|
6672
|
-
|
|
6673
|
-
|
|
6674
|
-
|
|
6675
|
-
if (
|
|
7421
|
+
try {
|
|
7422
|
+
let statusSource = error;
|
|
7423
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
7424
|
+
diagnostic.historyPersistenceStage = error.stage;
|
|
7425
|
+
statusSource = error.cause;
|
|
7426
|
+
}
|
|
7427
|
+
if (statusSource && typeof statusSource === "object") {
|
|
7428
|
+
const status = Number(
|
|
7429
|
+
statusSource.status ?? statusSource.statusCode
|
|
7430
|
+
);
|
|
7431
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
7432
|
+
diagnostic.status = status;
|
|
7433
|
+
}
|
|
6676
7434
|
}
|
|
7435
|
+
} catch {
|
|
6677
7436
|
}
|
|
6678
7437
|
return diagnostic;
|
|
6679
7438
|
}
|
|
6680
|
-
function safeErrorForTelemetry(error
|
|
6681
|
-
const diagnostic = safeErrorDiagnostic(error
|
|
6682
|
-
const safe = new Error(
|
|
6683
|
-
safe.name =
|
|
7439
|
+
function safeErrorForTelemetry(error) {
|
|
7440
|
+
const diagnostic = safeErrorDiagnostic(error);
|
|
7441
|
+
const safe = new Error("worker operation failed");
|
|
7442
|
+
safe.name = "WorkerOperationError";
|
|
7443
|
+
safe.code = diagnostic.errorCode;
|
|
7444
|
+
if (diagnostic.status !== void 0) safe.status = diagnostic.status;
|
|
7445
|
+
safe.origin = diagnostic.origin;
|
|
6684
7446
|
return safe;
|
|
6685
7447
|
}
|
|
6686
|
-
function headerSecretRedactions(prefix, headers, credentialHeaderNames) {
|
|
6687
|
-
const discovered = [];
|
|
6688
|
-
const explicitCredentialHeaders = credentialHeaderNames ? new Set([...credentialHeaderNames].map((name) => name.toLowerCase())) : null;
|
|
6689
|
-
for (const [headerName, value] of Object.entries(headers ?? {})) {
|
|
6690
|
-
if (!explicitCredentialHeaders?.has(headerName.toLowerCase()) && !isCredentialHeaderName(headerName)) {
|
|
6691
|
-
continue;
|
|
6692
|
-
}
|
|
6693
|
-
const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
|
|
6694
|
-
discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
|
|
6695
|
-
if (/^(?:proxy-)?authorization$/i.test(headerName)) {
|
|
6696
|
-
const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
|
|
6697
|
-
if (credential) {
|
|
6698
|
-
discovered.push({
|
|
6699
|
-
name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
|
|
6700
|
-
value: credential
|
|
6701
|
-
});
|
|
6702
|
-
}
|
|
6703
|
-
}
|
|
6704
|
-
}
|
|
6705
|
-
return discovered;
|
|
6706
|
-
}
|
|
6707
7448
|
function compactionFailureReasonFromError(error) {
|
|
6708
7449
|
if (error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError2) {
|
|
6709
7450
|
return compactionFailureReason(error.message);
|
|
@@ -6791,22 +7532,18 @@ async function recordCompletedModelCallBeforeOwnershipFences(input) {
|
|
|
6791
7532
|
}
|
|
6792
7533
|
await input.recordAttemptSignals?.();
|
|
6793
7534
|
}
|
|
6794
|
-
function
|
|
7535
|
+
function createModelResponseEventState(claimedSourceKeys = /* @__PURE__ */ new Set()) {
|
|
6795
7536
|
return {
|
|
6796
|
-
|
|
6797
|
-
|
|
6798
|
-
lastProviderContextTokensObserved: null,
|
|
7537
|
+
responseCount: 0,
|
|
7538
|
+
contextSignal: null,
|
|
6799
7539
|
claimedSourceKeys
|
|
6800
7540
|
};
|
|
6801
7541
|
}
|
|
6802
7542
|
function createCompactionModelUsageEventState(claimedSourceKeys = /* @__PURE__ */ new Set()) {
|
|
6803
7543
|
return { usageCount: 0, claimedSourceKeys };
|
|
6804
7544
|
}
|
|
6805
|
-
function
|
|
6806
|
-
return state.
|
|
6807
|
-
revision: state.providerContextRevision,
|
|
6808
|
-
totalTokens: state.lastProviderContextTokensObserved
|
|
6809
|
-
};
|
|
7545
|
+
function modelResponseContextSignal(state) {
|
|
7546
|
+
return state.contextSignal;
|
|
6810
7547
|
}
|
|
6811
7548
|
function assertModelResponseLatencyMode(input) {
|
|
6812
7549
|
if (input.requested === "standard") {
|
|
@@ -6825,14 +7562,14 @@ function assertModelResponseLatencyMode(input) {
|
|
|
6825
7562
|
);
|
|
6826
7563
|
}
|
|
6827
7564
|
}
|
|
6828
|
-
async function
|
|
6829
|
-
const
|
|
6830
|
-
if (!
|
|
6831
|
-
return { status: "
|
|
7565
|
+
async function processModelResponseTerminalEvent(input) {
|
|
7566
|
+
const terminal = modelTerminalResponseFromSdkEvent(input.event);
|
|
7567
|
+
if (!terminal) {
|
|
7568
|
+
return { status: "not_response" };
|
|
6832
7569
|
}
|
|
6833
|
-
const responseOrdinal = input.state.
|
|
7570
|
+
const responseOrdinal = input.state.responseCount + 1;
|
|
6834
7571
|
const sourceKey = modelUsageSourceKey({
|
|
6835
|
-
responseId:
|
|
7572
|
+
responseId: terminal.responseId,
|
|
6836
7573
|
dispatchId: input.dispatchId,
|
|
6837
7574
|
positionalKey: `response-${responseOrdinal}`
|
|
6838
7575
|
});
|
|
@@ -6840,19 +7577,21 @@ async function processModelResponseUsageEvent(input) {
|
|
|
6840
7577
|
return { status: "duplicate", sourceKey };
|
|
6841
7578
|
}
|
|
6842
7579
|
input.state.claimedSourceKeys.add(sourceKey);
|
|
6843
|
-
input.state.
|
|
6844
|
-
const
|
|
7580
|
+
input.state.responseCount = responseOrdinal;
|
|
7581
|
+
const responseUsage = terminal.usage;
|
|
7582
|
+
const normalizedUsage = normalizeModelCallUsage(responseUsage?.usage);
|
|
6845
7583
|
const accountContext = modelCallAccountContext({
|
|
6846
7584
|
servingCredentialId: input.servingCredentialId,
|
|
6847
7585
|
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
6848
7586
|
isFirstCallOfTurn: responseOrdinal === 1
|
|
6849
7587
|
});
|
|
6850
|
-
let authoritative =
|
|
7588
|
+
let authoritative = responseUsage === null;
|
|
6851
7589
|
await recordCompletedModelCallBeforeOwnershipFences({
|
|
6852
7590
|
renewLease: input.renewLease,
|
|
6853
7591
|
leaseLost: input.leaseLost,
|
|
6854
7592
|
leaseLostMessage: input.leaseLostMessage,
|
|
6855
7593
|
recordUsage: async () => {
|
|
7594
|
+
if (!responseUsage) return;
|
|
6856
7595
|
const billing = await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
6857
7596
|
accountId: input.accountId,
|
|
6858
7597
|
workspaceId: input.workspaceId,
|
|
@@ -6910,17 +7649,19 @@ async function processModelResponseUsageEvent(input) {
|
|
|
6910
7649
|
recordAttemptSignals: async () => {
|
|
6911
7650
|
if (!authoritative) return;
|
|
6912
7651
|
const observedTotal = normalizedUsage.totalTokens;
|
|
6913
|
-
|
|
6914
|
-
input.state.lastProviderContextTokensObserved = observedTotal;
|
|
6915
|
-
input.state.providerContextRevision += 1;
|
|
6916
|
-
}
|
|
7652
|
+
input.state.contextSignal = observedTotal !== null && observedTotal > 0 ? { revision: responseOrdinal, totalTokens: observedTotal } : null;
|
|
6917
7653
|
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
|
|
7654
|
+
await input.setLastInputTokens(
|
|
7655
|
+
observedInput !== null && observedInput > 0 ? observedInput : null
|
|
7656
|
+
);
|
|
6921
7657
|
}
|
|
6922
7658
|
});
|
|
6923
|
-
return {
|
|
7659
|
+
return {
|
|
7660
|
+
status: "processed",
|
|
7661
|
+
sourceKey,
|
|
7662
|
+
authoritative,
|
|
7663
|
+
usageReported: responseUsage !== null
|
|
7664
|
+
};
|
|
6924
7665
|
}
|
|
6925
7666
|
async function processCompactionModelUsageEvent(input) {
|
|
6926
7667
|
const usageOrdinal = input.state.usageCount + 1;
|
|
@@ -7032,14 +7773,9 @@ async function emitModelCallUsage(input) {
|
|
|
7032
7773
|
if (!authoritative) return false;
|
|
7033
7774
|
try {
|
|
7034
7775
|
input.observability.info("model call usage", {
|
|
7035
|
-
accountId: input.accountId,
|
|
7036
|
-
workspaceId: input.workspaceId,
|
|
7037
|
-
sessionId: input.sessionId,
|
|
7038
|
-
turnId: input.turnId,
|
|
7039
7776
|
provider: input.provider,
|
|
7040
7777
|
providerApi: input.providerApi,
|
|
7041
7778
|
model: input.model,
|
|
7042
|
-
sourceKey: input.sourceKey,
|
|
7043
7779
|
inputTokens: telemetry.inputTokens,
|
|
7044
7780
|
outputTokens: telemetry.outputTokens,
|
|
7045
7781
|
cachedTokens: telemetry.cachedTokens,
|
|
@@ -7053,7 +7789,6 @@ async function emitModelCallUsage(input) {
|
|
|
7053
7789
|
provider: input.provider,
|
|
7054
7790
|
providerApi: input.providerApi,
|
|
7055
7791
|
model: input.model,
|
|
7056
|
-
sourceKey: input.sourceKey,
|
|
7057
7792
|
rejectedFields: normalizedUsage.rejectedFields.join(",")
|
|
7058
7793
|
});
|
|
7059
7794
|
}
|
|
@@ -7072,20 +7807,27 @@ function applyCodexCacheTelemetry(observability, provider, normalizedUsage) {
|
|
|
7072
7807
|
promptTokens: normalizedUsage.telemetry.inputTokens
|
|
7073
7808
|
});
|
|
7074
7809
|
}
|
|
7075
|
-
function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens
|
|
7076
|
-
const
|
|
7077
|
-
|
|
7078
|
-
);
|
|
7079
|
-
if (sanitized.length <= persistedHistoryCount) {
|
|
7810
|
+
function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens) {
|
|
7811
|
+
const modelReady = sanitizeHistoryItemsForModel2(rawHistory, toolOutputTruncationTokens);
|
|
7812
|
+
if (modelReady.length <= persistedHistoryCount) {
|
|
7080
7813
|
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
7081
7814
|
}
|
|
7082
|
-
const rows =
|
|
7083
|
-
|
|
7084
|
-
item
|
|
7085
|
-
|
|
7815
|
+
const rows = [];
|
|
7816
|
+
for (const [offset, item] of modelReady.slice(persistedHistoryCount).entries()) {
|
|
7817
|
+
if (item.type === "message" && item.role === "system") {
|
|
7818
|
+
continue;
|
|
7819
|
+
}
|
|
7820
|
+
rows.push({
|
|
7821
|
+
position: nextPosition + rows.length,
|
|
7822
|
+
item: normalizeProtocolJsonValue(
|
|
7823
|
+
item,
|
|
7824
|
+
`$[${persistedHistoryCount + offset}]`
|
|
7825
|
+
)
|
|
7826
|
+
});
|
|
7827
|
+
}
|
|
7086
7828
|
return {
|
|
7087
7829
|
rows,
|
|
7088
|
-
nextWatermark:
|
|
7830
|
+
nextWatermark: modelReady.length,
|
|
7089
7831
|
nextPosition: nextPosition + rows.length
|
|
7090
7832
|
};
|
|
7091
7833
|
}
|
|
@@ -7109,12 +7851,6 @@ function selectRejectedProviderArtifactHistoryIds(activeHistory, candidates, wir
|
|
|
7109
7851
|
}
|
|
7110
7852
|
return selected;
|
|
7111
7853
|
}
|
|
7112
|
-
function secretRedactionModelInputFilter(redactValue) {
|
|
7113
|
-
return ({ modelData }) => ({
|
|
7114
|
-
...modelData,
|
|
7115
|
-
input: redactValue(modelData.input)
|
|
7116
|
-
});
|
|
7117
|
-
}
|
|
7118
7854
|
function isModelOrToolProgressHistoryItem(item) {
|
|
7119
7855
|
if (item.type === "message") {
|
|
7120
7856
|
return item.role === "assistant";
|
|
@@ -7432,9 +8168,6 @@ function createTurnSandboxProvisioner(establish, options = {}) {
|
|
|
7432
8168
|
}
|
|
7433
8169
|
};
|
|
7434
8170
|
}
|
|
7435
|
-
function sdkBackendIdForSandboxBackend(backend) {
|
|
7436
|
-
return backend === "local" ? "unix_local" : backend;
|
|
7437
|
-
}
|
|
7438
8171
|
function acceptsPromptCacheKeyForTurn(resolvedModel) {
|
|
7439
8172
|
if (!resolvedModel) {
|
|
7440
8173
|
return true;
|
|
@@ -7457,6 +8190,24 @@ async function refreshCappedCodexUsageRows(db, settings, workspaceId, accounts,
|
|
|
7457
8190
|
function codexCredentialLeaseDeadlineExpired(confirmedUntilMs, nowMs = performance.now()) {
|
|
7458
8191
|
return confirmedUntilMs === null || !Number.isFinite(confirmedUntilMs) || confirmedUntilMs <= nowMs;
|
|
7459
8192
|
}
|
|
8193
|
+
async function finalizeDurableTurnOpStreams(sessions, fallback) {
|
|
8194
|
+
const candidates = /* @__PURE__ */ new Set();
|
|
8195
|
+
for (const session of sessions) {
|
|
8196
|
+
const candidate = session;
|
|
8197
|
+
if (typeof candidate?.finalizeOpStreamOps === "function") {
|
|
8198
|
+
candidates.add(candidate);
|
|
8199
|
+
}
|
|
8200
|
+
}
|
|
8201
|
+
if (candidates.size === 0 && fallback) {
|
|
8202
|
+
candidates.add(fallback);
|
|
8203
|
+
}
|
|
8204
|
+
for (const candidate of candidates) {
|
|
8205
|
+
try {
|
|
8206
|
+
await candidate.finalizeOpStreamOps();
|
|
8207
|
+
} catch {
|
|
8208
|
+
}
|
|
8209
|
+
}
|
|
8210
|
+
}
|
|
7460
8211
|
function createRunAgentTurnActivity(services) {
|
|
7461
8212
|
return async function runAgentTurn2(input) {
|
|
7462
8213
|
const {
|
|
@@ -7500,6 +8251,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
7500
8251
|
acknowledgeQuiescence = true;
|
|
7501
8252
|
noteCancellationRequested();
|
|
7502
8253
|
};
|
|
8254
|
+
const acknowledgeRecoveryQuiescence = () => {
|
|
8255
|
+
acknowledgeQuiescence = true;
|
|
8256
|
+
noteCancellationRequested();
|
|
8257
|
+
};
|
|
7503
8258
|
let turnId;
|
|
7504
8259
|
let triggerEventId;
|
|
7505
8260
|
const claimedResult = (result) => {
|
|
@@ -7601,7 +8356,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7601
8356
|
workspaceId: input.workspaceId,
|
|
7602
8357
|
turnId,
|
|
7603
8358
|
reason,
|
|
7604
|
-
|
|
8359
|
+
...safeErrorDiagnostic(error)
|
|
7605
8360
|
});
|
|
7606
8361
|
observability.incrementCounter({
|
|
7607
8362
|
name: "opengeni_codex_lease_renewals_total",
|
|
@@ -7632,6 +8387,12 @@ function createRunAgentTurnActivity(services) {
|
|
|
7632
8387
|
let lazyOwnedSandbox = null;
|
|
7633
8388
|
let turnSandboxProvisioner = null;
|
|
7634
8389
|
let setupBoxSession = null;
|
|
8390
|
+
const finalizeTurnOpStreamOps = async () => {
|
|
8391
|
+
await finalizeDurableTurnOpStreams(
|
|
8392
|
+
[lazyOwnedSandbox?.session, resolvedSandbox?.established.session],
|
|
8393
|
+
machinePrimarySession
|
|
8394
|
+
);
|
|
8395
|
+
};
|
|
7635
8396
|
const onHomeSandboxRebound = (rebound) => {
|
|
7636
8397
|
const current = resolvedSandbox;
|
|
7637
8398
|
const previousSession = current?.established.session;
|
|
@@ -7862,7 +8623,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7862
8623
|
sessionId: input.sessionId,
|
|
7863
8624
|
sandboxGroupId: heartbeatGroupId,
|
|
7864
8625
|
leaseEpoch: heartbeatEpoch,
|
|
7865
|
-
|
|
8626
|
+
...safeErrorDiagnostic(error)
|
|
7866
8627
|
});
|
|
7867
8628
|
}).finally(() => {
|
|
7868
8629
|
rotationInFlight = null;
|
|
@@ -7970,6 +8731,28 @@ function createRunAgentTurnActivity(services) {
|
|
|
7970
8731
|
}
|
|
7971
8732
|
};
|
|
7972
8733
|
let persistedHistoryCount = 0;
|
|
8734
|
+
const retainedScreenshotReceiptsByCallId = /* @__PURE__ */ new Map();
|
|
8735
|
+
const computerScreenshotCallIds = /* @__PURE__ */ new Set();
|
|
8736
|
+
const materializeScreenshotHistory = async (history) => {
|
|
8737
|
+
collectRetainedScreenshotReceipts(history, retainedScreenshotReceiptsByCallId);
|
|
8738
|
+
return await materializeRetainedScreenshotHistory({
|
|
8739
|
+
db,
|
|
8740
|
+
objectStorage,
|
|
8741
|
+
workspaceId: input.workspaceId,
|
|
8742
|
+
sessionId: input.sessionId,
|
|
8743
|
+
history
|
|
8744
|
+
});
|
|
8745
|
+
};
|
|
8746
|
+
const materializeScreenshotRunState = async (serialized) => {
|
|
8747
|
+
collectRetainedScreenshotRunStateReceipts(serialized, retainedScreenshotReceiptsByCallId);
|
|
8748
|
+
return await materializeRetainedScreenshotRunState({
|
|
8749
|
+
db,
|
|
8750
|
+
objectStorage,
|
|
8751
|
+
workspaceId: input.workspaceId,
|
|
8752
|
+
sessionId: input.sessionId,
|
|
8753
|
+
serialized
|
|
8754
|
+
});
|
|
8755
|
+
};
|
|
7973
8756
|
let providerArtifactCandidates = {
|
|
7974
8757
|
knownHistoryItemIds: [],
|
|
7975
8758
|
historyItemIds: []
|
|
@@ -7979,36 +8762,42 @@ function createRunAgentTurnActivity(services) {
|
|
|
7979
8762
|
if (!stream || !turnId) {
|
|
7980
8763
|
return;
|
|
7981
8764
|
}
|
|
8765
|
+
const durableTurnId = turnId;
|
|
7982
8766
|
try {
|
|
7983
8767
|
const rawHistory = stream.state.history;
|
|
7984
8768
|
if (Array.isArray(rawHistory)) {
|
|
7985
|
-
const
|
|
8769
|
+
const durableHistory = compactRetainedScreenshotHistory(
|
|
7986
8770
|
rawHistory,
|
|
8771
|
+
retainedScreenshotReceiptsByCallId
|
|
8772
|
+
);
|
|
8773
|
+
const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
|
|
8774
|
+
durableHistory,
|
|
7987
8775
|
persistedHistoryCount,
|
|
7988
8776
|
nextHistoryPosition,
|
|
7989
|
-
modelRunSettings.modelToolOutputTruncationTokens
|
|
7990
|
-
(value) => redact(value)
|
|
8777
|
+
modelRunSettings.modelToolOutputTruncationTokens
|
|
7991
8778
|
);
|
|
7992
8779
|
const hasModelOrToolProgress = rows.some(
|
|
7993
8780
|
(row) => isModelOrToolProgressHistoryItem(row.item)
|
|
7994
8781
|
);
|
|
7995
8782
|
const shouldAppendRows = rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
|
|
7996
8783
|
if (shouldAppendRows) {
|
|
7997
|
-
|
|
7998
|
-
|
|
7999
|
-
|
|
8000
|
-
|
|
8001
|
-
|
|
8002
|
-
|
|
8003
|
-
|
|
8004
|
-
|
|
8005
|
-
|
|
8784
|
+
await runMandatoryHistoryPersistenceStep("history_append", async () => {
|
|
8785
|
+
const appended = await appendSessionHistoryItems(db, {
|
|
8786
|
+
accountId: input.accountId,
|
|
8787
|
+
workspaceId: input.workspaceId,
|
|
8788
|
+
sessionId: input.sessionId,
|
|
8789
|
+
turnId: durableTurnId,
|
|
8790
|
+
expectedExecutionGeneration: executionGeneration,
|
|
8791
|
+
expectedAttemptId: input.attemptId,
|
|
8792
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
8793
|
+
items: rows
|
|
8794
|
+
});
|
|
8795
|
+
if (!appended) {
|
|
8796
|
+
throw new TurnAttemptFencedError(
|
|
8797
|
+
"turn execution generation was fenced while saving conversation history"
|
|
8798
|
+
);
|
|
8799
|
+
}
|
|
8006
8800
|
});
|
|
8007
|
-
if (!appended) {
|
|
8008
|
-
throw new TurnAttemptFencedError(
|
|
8009
|
-
"turn execution generation was fenced while saving conversation history"
|
|
8010
|
-
);
|
|
8011
|
-
}
|
|
8012
8801
|
}
|
|
8013
8802
|
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
8014
8803
|
persistedHistoryCount = nextWatermark;
|
|
@@ -8017,32 +8806,25 @@ function createRunAgentTurnActivity(services) {
|
|
|
8017
8806
|
}
|
|
8018
8807
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
8019
8808
|
if (envelope) {
|
|
8020
|
-
await
|
|
8021
|
-
|
|
8022
|
-
|
|
8023
|
-
|
|
8024
|
-
|
|
8025
|
-
|
|
8809
|
+
await runMandatoryHistoryPersistenceStep(
|
|
8810
|
+
"sandbox_envelope",
|
|
8811
|
+
() => upsertSandboxSessionEnvelope(db, {
|
|
8812
|
+
accountId: input.accountId,
|
|
8813
|
+
workspaceId: input.workspaceId,
|
|
8814
|
+
sessionId: input.sessionId,
|
|
8815
|
+
envelope
|
|
8816
|
+
})
|
|
8817
|
+
);
|
|
8026
8818
|
}
|
|
8027
8819
|
} catch (persistError) {
|
|
8028
8820
|
console.error(
|
|
8029
8821
|
"session history dual-write failed (run unaffected)",
|
|
8030
|
-
safeErrorDiagnostic(persistError
|
|
8822
|
+
safeErrorDiagnostic(persistError)
|
|
8031
8823
|
);
|
|
8032
8824
|
if (options.requireDurable) throw persistError;
|
|
8033
8825
|
}
|
|
8034
8826
|
};
|
|
8035
|
-
let redact = createSecretRedactor([]);
|
|
8036
|
-
const secretRedactions = /* @__PURE__ */ new Map();
|
|
8037
8827
|
const publishedRunCredentialNotices = /* @__PURE__ */ new Set();
|
|
8038
|
-
const registerSecretRedactions = (secrets) => {
|
|
8039
|
-
for (const secret of secrets) {
|
|
8040
|
-
if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
|
|
8041
|
-
}
|
|
8042
|
-
redact = createSecretRedactor(
|
|
8043
|
-
[...secretRedactions].map(([value, name]) => ({ name, value }))
|
|
8044
|
-
);
|
|
8045
|
-
};
|
|
8046
8828
|
let variableSetId = "";
|
|
8047
8829
|
let rigId = "";
|
|
8048
8830
|
let rigVersionId = "";
|
|
@@ -8174,7 +8956,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8174
8956
|
publish = async (events, immediate = false) => {
|
|
8175
8957
|
const inputs = events.map((event) => ({
|
|
8176
8958
|
...event,
|
|
8177
|
-
payload:
|
|
8959
|
+
payload: event.payload,
|
|
8178
8960
|
turnId,
|
|
8179
8961
|
producerId,
|
|
8180
8962
|
producerSeq: ++producerSeq
|
|
@@ -8187,7 +8969,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
8187
8969
|
turnId,
|
|
8188
8970
|
executionGeneration,
|
|
8189
8971
|
input.attemptId,
|
|
8190
|
-
inputs
|
|
8972
|
+
inputs,
|
|
8973
|
+
{
|
|
8974
|
+
onAppend: ({ durationSeconds }) => recordSessionEventAppendLatency(observability, { durationSeconds }),
|
|
8975
|
+
onPublish: ({ durationSeconds }) => recordSessionEventPublishLatency(observability, { durationSeconds })
|
|
8976
|
+
}
|
|
8191
8977
|
);
|
|
8192
8978
|
if (inputs.length > 0 && !appended.accepted) {
|
|
8193
8979
|
throw new TurnAttemptFencedError("turn execution generation was fenced");
|
|
@@ -8232,12 +9018,18 @@ function createRunAgentTurnActivity(services) {
|
|
|
8232
9018
|
} : void 0;
|
|
8233
9019
|
const inputs = inputSettlement.events.map((event) => ({
|
|
8234
9020
|
...event,
|
|
8235
|
-
payload:
|
|
9021
|
+
payload: event.payload,
|
|
8236
9022
|
turnId,
|
|
8237
9023
|
producerId,
|
|
8238
9024
|
producerSeq: ++producerSeq
|
|
8239
9025
|
}));
|
|
8240
|
-
const
|
|
9026
|
+
const runState = inputSettlement.runState ? {
|
|
9027
|
+
...inputSettlement.runState,
|
|
9028
|
+
serializedRunState: compactRetainedScreenshotRunState(
|
|
9029
|
+
inputSettlement.runState.serializedRunState,
|
|
9030
|
+
retainedScreenshotReceiptsByCallId
|
|
9031
|
+
)
|
|
9032
|
+
} : void 0;
|
|
8241
9033
|
const result = await applySessionTurnSettlement(db, input.workspaceId, {
|
|
8242
9034
|
sessionId: input.sessionId,
|
|
8243
9035
|
turnId,
|
|
@@ -8247,7 +9039,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8247
9039
|
sessionStatus: inputSettlement.sessionStatus,
|
|
8248
9040
|
activeTurnId: inputSettlement.activeTurnId,
|
|
8249
9041
|
events: inputs,
|
|
8250
|
-
...
|
|
9042
|
+
...runState ? { runState } : {},
|
|
8251
9043
|
...recordingMutation ? { recording: recordingMutation } : {},
|
|
8252
9044
|
...compactionRequestFailure ? { compactionRequestFailure } : {}
|
|
8253
9045
|
});
|
|
@@ -8539,10 +9331,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
8539
9331
|
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult)
|
|
8540
9332
|
});
|
|
8541
9333
|
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
8542
|
-
workspaceId: input.workspaceId,
|
|
8543
9334
|
stage: shadowResult.stage,
|
|
8544
9335
|
reason: shadowResult.reason,
|
|
8545
|
-
|
|
9336
|
+
errorClass: "CodexFleetShadowOperationError",
|
|
9337
|
+
errorCode: "codex_fleet_shadow_failed",
|
|
9338
|
+
origin: "worker",
|
|
8546
9339
|
payloadBytes: shadowResult.payloadBytes
|
|
8547
9340
|
});
|
|
8548
9341
|
}
|
|
@@ -8830,14 +9623,18 @@ function createRunAgentTurnActivity(services) {
|
|
|
8830
9623
|
attemptId: input.attemptId,
|
|
8831
9624
|
executionGeneration: turn.executionGeneration
|
|
8832
9625
|
};
|
|
8833
|
-
const [
|
|
8834
|
-
|
|
9626
|
+
const [workspace, instructionPolicySnapshot, preferenceSnapshot] = await Promise.all([
|
|
9627
|
+
getWorkspace2(db, input.workspaceId),
|
|
8835
9628
|
getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
|
|
8836
9629
|
getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
|
|
8837
9630
|
if (error instanceof PreferenceRegistryInitiatorError) return null;
|
|
8838
9631
|
throw error;
|
|
8839
9632
|
})
|
|
8840
9633
|
]);
|
|
9634
|
+
if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
9635
|
+
const workspaceAgentInstructions = workspace.agentInstructions;
|
|
9636
|
+
const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
|
|
9637
|
+
assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
|
|
8841
9638
|
const workspaceGovernance = renderWorkspaceGovernanceContext({
|
|
8842
9639
|
instructionPolicy: instructionPolicySnapshot,
|
|
8843
9640
|
preferences: preferenceSnapshot
|
|
@@ -8861,31 +9658,13 @@ function createRunAgentTurnActivity(services) {
|
|
|
8861
9658
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
8862
9659
|
sandboxBackend: turn.sandboxBackend
|
|
8863
9660
|
};
|
|
8864
|
-
const sessionMcpHeaderNames = /* @__PURE__ */ new Map();
|
|
8865
9661
|
const runSettings = await settingsWithSessionMcpServersForRun(
|
|
8866
9662
|
db,
|
|
8867
9663
|
input.workspaceId,
|
|
8868
9664
|
input.sessionId,
|
|
8869
9665
|
input.attemptId,
|
|
8870
|
-
baseRunSettings
|
|
8871
|
-
{
|
|
8872
|
-
onResolvedServers: (servers) => {
|
|
8873
|
-
for (const server of servers) {
|
|
8874
|
-
sessionMcpHeaderNames.set(server.id, server.headerNames);
|
|
8875
|
-
}
|
|
8876
|
-
}
|
|
8877
|
-
}
|
|
9666
|
+
baseRunSettings
|
|
8878
9667
|
);
|
|
8879
|
-
registerSecretRedactions([
|
|
8880
|
-
...runSettings.accessKey ? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }] : [],
|
|
8881
|
-
...runSettings.mcpServers.flatMap((server) => {
|
|
8882
|
-
return headerSecretRedactions(
|
|
8883
|
-
`MCP_${server.id.toUpperCase()}_STATIC`,
|
|
8884
|
-
server.headers,
|
|
8885
|
-
sessionMcpHeaderNames.get(server.id)
|
|
8886
|
-
);
|
|
8887
|
-
})
|
|
8888
|
-
]);
|
|
8889
9668
|
const resolvedModel = runtime.resolveTurnModel(
|
|
8890
9669
|
capabilitySettings,
|
|
8891
9670
|
turnExecutionPolicy.productModelId
|
|
@@ -8901,38 +9680,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
8901
9680
|
);
|
|
8902
9681
|
const modelHistoryProjector = async (items) => projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
|
|
8903
9682
|
const compactionModelHistoryProjector = async (items) => await modelHistoryProjector(projectHistoryForProvider2(items, providerApi));
|
|
8904
|
-
const selectedProvider = resolvedModel?.provider;
|
|
8905
|
-
const publicProviderHeaders = new Set(
|
|
8906
|
-
selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? []
|
|
8907
|
-
);
|
|
8908
|
-
const publicProviderQueries = new Set(
|
|
8909
|
-
selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? []
|
|
8910
|
-
);
|
|
8911
|
-
const fallbackProviderApiKey = capabilitySettings.openaiProvider === "azure" ? capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken : capabilitySettings.openaiApiKey;
|
|
8912
|
-
registerSecretRedactions([
|
|
8913
|
-
...selectedProvider?.apiKey ? [
|
|
8914
|
-
{
|
|
8915
|
-
name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
|
|
8916
|
-
value: selectedProvider.apiKey
|
|
8917
|
-
}
|
|
8918
|
-
] : !resolvedModel && fallbackProviderApiKey ? [
|
|
8919
|
-
{
|
|
8920
|
-
name: "MODEL_PROVIDER_API_KEY",
|
|
8921
|
-
value: fallbackProviderApiKey
|
|
8922
|
-
}
|
|
8923
|
-
] : [],
|
|
8924
|
-
...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(
|
|
8925
|
-
([name, value]) => publicProviderHeaders.has(name.toLowerCase()) ? [] : [
|
|
8926
|
-
{
|
|
8927
|
-
name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`,
|
|
8928
|
-
value
|
|
8929
|
-
}
|
|
8930
|
-
]
|
|
8931
|
-
),
|
|
8932
|
-
...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(
|
|
8933
|
-
([name, value]) => publicProviderQueries.has(name.toLowerCase()) ? [] : [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }]
|
|
8934
|
-
)
|
|
8935
|
-
]);
|
|
8936
9683
|
modelRunSettings = resolvedModel ? settingsWithResolvedModelContext(runSettings, resolvedModel.configured) : runSettings;
|
|
8937
9684
|
{
|
|
8938
9685
|
const workspaceModelPolicy = await getWorkspaceModelPolicy2(db, input.workspaceId);
|
|
@@ -8958,13 +9705,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
8958
9705
|
input.workspaceId,
|
|
8959
9706
|
effectiveCodexCredentialId ?? ""
|
|
8960
9707
|
);
|
|
8961
|
-
const registerSnapshot = async (snapshotPromise) => {
|
|
8962
|
-
const snapshot = await snapshotPromise;
|
|
8963
|
-
registerSecretRedactions([
|
|
8964
|
-
{ name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken }
|
|
8965
|
-
]);
|
|
8966
|
-
return snapshot;
|
|
8967
|
-
};
|
|
8968
9708
|
return {
|
|
8969
9709
|
clientVersion: CODEX_CLIENT_VERSION,
|
|
8970
9710
|
// Backend sticky cache-routing key — the SAME id as the body's
|
|
@@ -8974,8 +9714,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
8974
9714
|
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
8975
9715
|
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
8976
9716
|
sessionId: input.sessionId,
|
|
8977
|
-
getToken: () =>
|
|
8978
|
-
refresh: () =>
|
|
9717
|
+
getToken: () => resolver.getToken(),
|
|
9718
|
+
refresh: () => resolver.refresh(),
|
|
8979
9719
|
resolveModel: buildModelResolver(
|
|
8980
9720
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
8981
9721
|
CODEX_FALLBACK_MODEL_SLUGS[0]
|
|
@@ -9175,6 +9915,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9175
9915
|
force: true,
|
|
9176
9916
|
clearRequestedCompaction: true,
|
|
9177
9917
|
trigger: "operator",
|
|
9918
|
+
materializeHistory: materializeScreenshotHistory,
|
|
9178
9919
|
projectModelInput: compactionModelHistoryProjector,
|
|
9179
9920
|
...compactionModeOptions
|
|
9180
9921
|
}
|
|
@@ -9201,7 +9942,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9201
9942
|
}
|
|
9202
9943
|
);
|
|
9203
9944
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
9204
|
-
const errorMessage = String(
|
|
9945
|
+
const errorMessage = String(compactionFailureReasonFromError(error));
|
|
9205
9946
|
if (!await settle({
|
|
9206
9947
|
events: [
|
|
9207
9948
|
{
|
|
@@ -9307,9 +10048,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9307
10048
|
rigDefaultEnvironmentValues,
|
|
9308
10049
|
workspaceVariableSet?.values ?? {}
|
|
9309
10050
|
);
|
|
9310
|
-
registerSecretRedactions(
|
|
9311
|
-
Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value }))
|
|
9312
|
-
);
|
|
9313
10051
|
const routingOn = routingEnabled(settings);
|
|
9314
10052
|
let activeSandboxPointer = routingOn ? await readActiveSandbox2(db, input.workspaceId, input.sessionId).catch(() => null) : null;
|
|
9315
10053
|
let activeSandboxRecord = null;
|
|
@@ -9366,7 +10104,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9366
10104
|
void 0
|
|
9367
10105
|
) : null;
|
|
9368
10106
|
if (initialRunCredentialMaterial) {
|
|
9369
|
-
registerSecretRedactions(initialRunCredentialMaterial.redactions);
|
|
9370
10107
|
for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
|
|
9371
10108
|
publishedRunCredentialNotices.add(JSON.stringify(payload));
|
|
9372
10109
|
await publish([{ type: "credential.auth_needed", payload }], true);
|
|
@@ -9379,7 +10116,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
9379
10116
|
cancellationSignal,
|
|
9380
10117
|
void 0
|
|
9381
10118
|
);
|
|
9382
|
-
const establishPolicy = lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none"
|
|
10119
|
+
const establishPolicy = lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none" && // Resolved run credentials must be written to one exact leased sandbox
|
|
10120
|
+
// before agent execution. A warm active pointer can bypass the lazy
|
|
10121
|
+
// provisioner entirely, which previously skipped materialization.
|
|
10122
|
+
!initialRunCredentialMaterial ? "on-demand" : "eager";
|
|
9383
10123
|
if (activeSandboxBackend !== "selfhosted") {
|
|
9384
10124
|
await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
|
|
9385
10125
|
}
|
|
@@ -9424,6 +10164,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9424
10164
|
sandboxWorkspaceEnvironmentValues,
|
|
9425
10165
|
{
|
|
9426
10166
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
10167
|
+
skipToolspace: activeSandboxBackend === "selfhosted",
|
|
9427
10168
|
deferGitHubToken: activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
|
|
9428
10169
|
scope: connectionScope,
|
|
9429
10170
|
...gitCredentialAuthority ? { authority: gitCredentialAuthority } : {},
|
|
@@ -9435,17 +10176,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9435
10176
|
cancellationSignal,
|
|
9436
10177
|
void 0
|
|
9437
10178
|
);
|
|
9438
|
-
registerSecretRedactions([
|
|
9439
|
-
...Object.entries(sandboxGitTokens ?? {}).flatMap(
|
|
9440
|
-
([provider, value]) => value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : []
|
|
9441
|
-
),
|
|
9442
|
-
...sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : [],
|
|
9443
|
-
...(sandboxGitCredentialBindings ?? []).map((binding) => ({
|
|
9444
|
-
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
9445
|
-
value: binding.token
|
|
9446
|
-
})),
|
|
9447
|
-
...sandboxToolspaceToken ? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }] : []
|
|
9448
|
-
]);
|
|
9449
10179
|
const sandboxToolspaceTokenFile = sandboxToolspaceToken ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId) : void 0;
|
|
9450
10180
|
const initialGitCredentials = sandboxGitCredentialBindings ? {
|
|
9451
10181
|
bindings: sandboxGitCredentialBindings,
|
|
@@ -9478,12 +10208,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9478
10208
|
);
|
|
9479
10209
|
if (binding) {
|
|
9480
10210
|
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
9481
|
-
registerSecretRedactions([
|
|
9482
|
-
{
|
|
9483
|
-
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
9484
|
-
value: binding.token
|
|
9485
|
-
}
|
|
9486
|
-
]);
|
|
9487
10211
|
}
|
|
9488
10212
|
pendingBinding = binding;
|
|
9489
10213
|
return binding ? {
|
|
@@ -9560,7 +10284,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9560
10284
|
toolspaceAuthority
|
|
9561
10285
|
);
|
|
9562
10286
|
if (material) {
|
|
9563
|
-
registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
|
|
9564
10287
|
}
|
|
9565
10288
|
return material;
|
|
9566
10289
|
};
|
|
@@ -9668,7 +10391,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9668
10391
|
);
|
|
9669
10392
|
return;
|
|
9670
10393
|
}
|
|
9671
|
-
registerSecretRedactions(material.redactions);
|
|
9672
10394
|
await runWorkspaceMutationForSandbox(
|
|
9673
10395
|
requireTargetSandbox(),
|
|
9674
10396
|
"runCredentialMaterialization",
|
|
@@ -9743,7 +10465,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9743
10465
|
sandboxHolderId = managedOwnership?.holderId ?? null;
|
|
9744
10466
|
sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
|
|
9745
10467
|
if (machinePrimary) {
|
|
9746
|
-
const machineOpStream = settings.agentOpStreamEnabled === true ? (await
|
|
10468
|
+
const machineOpStream = settings.agentOpStreamEnabled === true ? (await getEnrollment2(db, input.workspaceId, activeSandboxRecord.enrollmentId))?.opStream === true : false;
|
|
9747
10469
|
const established = await establishSelfhostedTurnSession(
|
|
9748
10470
|
{
|
|
9749
10471
|
db,
|
|
@@ -9775,6 +10497,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9775
10497
|
db,
|
|
9776
10498
|
settings,
|
|
9777
10499
|
bus,
|
|
10500
|
+
opJournal,
|
|
9778
10501
|
onOp: machineOpObserver.observer,
|
|
9779
10502
|
onSandboxOperation: sandboxOperationObserver,
|
|
9780
10503
|
onHomeSandboxRebound
|
|
@@ -9865,6 +10588,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9865
10588
|
db,
|
|
9866
10589
|
settings,
|
|
9867
10590
|
bus,
|
|
10591
|
+
opJournal,
|
|
9868
10592
|
onSandboxOperation: sandboxOperationObserver,
|
|
9869
10593
|
onHomeSandboxLost: publishSandboxLost,
|
|
9870
10594
|
onHomeSandboxRebound
|
|
@@ -9943,9 +10667,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9943
10667
|
const resolveCredential = async (request) => {
|
|
9944
10668
|
const result = await resolveFrozenCredential(request);
|
|
9945
10669
|
if (result.status === "ok") {
|
|
9946
|
-
registerSecretRedactions(
|
|
9947
|
-
headerSecretRedactions("MCP", result.headers, Object.keys(result.headers))
|
|
9948
|
-
);
|
|
9949
10670
|
}
|
|
9950
10671
|
return result;
|
|
9951
10672
|
};
|
|
@@ -9961,9 +10682,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9961
10682
|
clientVersion: CODEX_CLIENT_VERSION,
|
|
9962
10683
|
withAuthorization: async (use) => {
|
|
9963
10684
|
const snapshot = await resolver.getToken();
|
|
9964
|
-
registerSecretRedactions([
|
|
9965
|
-
{ name: "CODEX_APPS_ACCESS_TOKEN", value: snapshot.accessToken }
|
|
9966
|
-
]);
|
|
9967
10685
|
return await withCodexAppsRequestAuthorization(
|
|
9968
10686
|
db,
|
|
9969
10687
|
{ workspaceId: input.workspaceId, credentialId: codexAppsCredentialId },
|
|
@@ -10043,6 +10761,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10043
10761
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
10044
10762
|
...serviceTier ? { serviceTier } : {},
|
|
10045
10763
|
...humanInputResume ? { humanInputResponse: humanInputResume } : {},
|
|
10764
|
+
humanInputEnabled: agentHumanInputEnabled,
|
|
10046
10765
|
genesisTitleHint: isGenesisTurn,
|
|
10047
10766
|
persistentSessionSettings: {
|
|
10048
10767
|
titleIsSet: Boolean(session.title?.trim())
|
|
@@ -10061,14 +10780,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
10061
10780
|
...activeSandboxBackend !== "selfhosted" && sandboxGitTokens ? { gitTokenSeeds: sandboxGitTokens } : {},
|
|
10062
10781
|
...activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings ? { gitCredentialBindings: sandboxGitCredentialBindings } : {},
|
|
10063
10782
|
...activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
|
|
10064
|
-
// Toolspace is
|
|
10065
|
-
//
|
|
10066
|
-
//
|
|
10067
|
-
// machine's ONLY path to programmatic tool calling and grants no more
|
|
10068
|
-
// than toolspace:call for its own session (own-session-bound,
|
|
10069
|
-
// short-lived and renewable, per-active-turn budgeted, approval-tools
|
|
10070
|
-
// excluded). The runtime seeds it to the box's token file over the same
|
|
10071
|
-
// exec channel, off-manifest, on every backend.
|
|
10783
|
+
// Toolspace delivery is managed-sandbox-only. Connected Machines own any
|
|
10784
|
+
// manually configured API credentials and must never be contacted during
|
|
10785
|
+
// turn admission merely to seed OpenGeni tooling.
|
|
10072
10786
|
...sandboxToolspaceToken ? {
|
|
10073
10787
|
toolspaceTokenSeed: sandboxToolspaceToken,
|
|
10074
10788
|
toolspaceTokenSessionId: input.sessionId
|
|
@@ -10328,6 +11042,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10328
11042
|
db,
|
|
10329
11043
|
settings,
|
|
10330
11044
|
bus,
|
|
11045
|
+
opJournal,
|
|
10331
11046
|
onOp: machineOpObserver.observer,
|
|
10332
11047
|
onSandboxOperation: sandboxOperationObserver,
|
|
10333
11048
|
onHomeSandboxLost: publishSandboxLost,
|
|
@@ -10418,7 +11133,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10418
11133
|
}
|
|
10419
11134
|
);
|
|
10420
11135
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
10421
|
-
const errorMessage = String(
|
|
11136
|
+
const errorMessage = String(compactionFailureReasonFromError(error));
|
|
10422
11137
|
if (!await settle({
|
|
10423
11138
|
events: [
|
|
10424
11139
|
{
|
|
@@ -10503,10 +11218,12 @@ function createRunAgentTurnActivity(services) {
|
|
|
10503
11218
|
force: true,
|
|
10504
11219
|
clearRequestedCompaction: true,
|
|
10505
11220
|
trigger: "operator",
|
|
11221
|
+
materializeHistory: materializeScreenshotHistory,
|
|
10506
11222
|
projectModelInput: compactionModelHistoryProjector,
|
|
10507
11223
|
...compactionModeOptions
|
|
10508
11224
|
} : {
|
|
10509
11225
|
trigger: "auto",
|
|
11226
|
+
materializeHistory: materializeScreenshotHistory,
|
|
10510
11227
|
projectModelInput: compactionModelHistoryProjector,
|
|
10511
11228
|
...compactionModeOptions
|
|
10512
11229
|
}
|
|
@@ -10540,11 +11257,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
10540
11257
|
}
|
|
10541
11258
|
);
|
|
10542
11259
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
10543
|
-
const errorMessage = String(
|
|
11260
|
+
const errorMessage = String(compactionFailureReasonFromError(compactError));
|
|
10544
11261
|
observability.error("context compaction failed", {
|
|
10545
11262
|
sessionId: input.sessionId,
|
|
10546
11263
|
turnId,
|
|
10547
|
-
|
|
11264
|
+
...safeErrorDiagnostic(compactError)
|
|
10548
11265
|
});
|
|
10549
11266
|
if (!await settle({
|
|
10550
11267
|
events: [
|
|
@@ -10641,6 +11358,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
10641
11358
|
...unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {},
|
|
10642
11359
|
...runCredentialsNote ? { runCredentialsNote } : {},
|
|
10643
11360
|
providerApi,
|
|
11361
|
+
materializeModelHistory: materializeScreenshotHistory,
|
|
11362
|
+
materializeSerializedRunState: materializeScreenshotRunState,
|
|
10644
11363
|
projectModelHistory: modelHistoryProjector
|
|
10645
11364
|
});
|
|
10646
11365
|
runInput = prepared.input;
|
|
@@ -10666,16 +11385,17 @@ function createRunAgentTurnActivity(services) {
|
|
|
10666
11385
|
attemptId: input.attemptId
|
|
10667
11386
|
},
|
|
10668
11387
|
// Never reuse the persisted prior-turn signal for recovery. Proactive
|
|
10669
|
-
// guards provide their exact current signal; provider
|
|
10670
|
-
//
|
|
10671
|
-
// recovery proves progress separately by comparing the
|
|
10672
|
-
// with the current active-history estimate
|
|
11388
|
+
// guards provide their exact current provider-based signal; provider
|
|
11389
|
+
// overflows do not, so their diagnostic signal stays unknown/zero.
|
|
11390
|
+
// Forced recovery proves progress separately by comparing the
|
|
11391
|
+
// replacement with the current active-history estimate.
|
|
10673
11392
|
recoverySignalTokens,
|
|
10674
11393
|
compactSummarizer,
|
|
10675
11394
|
{
|
|
10676
11395
|
force: true,
|
|
10677
11396
|
...triggerLabel === "operator" ? { clearRequestedCompaction: true } : {},
|
|
10678
11397
|
trigger: triggerLabel,
|
|
11398
|
+
materializeHistory: materializeScreenshotHistory,
|
|
10679
11399
|
projectModelInput: compactionModelHistoryProjector,
|
|
10680
11400
|
...compactionModeOptions
|
|
10681
11401
|
}
|
|
@@ -10691,9 +11411,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10691
11411
|
}
|
|
10692
11412
|
return outcome;
|
|
10693
11413
|
};
|
|
10694
|
-
const
|
|
10695
|
-
claimedModelUsageSourceKeys
|
|
10696
|
-
);
|
|
11414
|
+
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
10697
11415
|
const runStreamAttempt = async () => {
|
|
10698
11416
|
if (!runInput) {
|
|
10699
11417
|
throw new Error("Run input was not prepared");
|
|
@@ -10702,19 +11420,25 @@ function createRunAgentTurnActivity(services) {
|
|
|
10702
11420
|
batcher = null;
|
|
10703
11421
|
let currentToolBatchCallIds = /* @__PURE__ */ new Set();
|
|
10704
11422
|
let currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
|
|
11423
|
+
let streamSawPerResponseUsage = false;
|
|
10705
11424
|
throwIfWorkerShuttingDown();
|
|
10706
11425
|
throwIfTurnCancelled();
|
|
10707
11426
|
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
10708
11427
|
const runStreamOnce = async () => {
|
|
10709
11428
|
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
10710
11429
|
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
11430
|
+
await attachRunCredentialRenewal(
|
|
11431
|
+
eagerSetupSession,
|
|
11432
|
+
resolvedSandbox
|
|
11433
|
+
);
|
|
11434
|
+
const eagerCredentialSetupSession = initialRunCredentialMaterial ? withRunCredentialsSession(eagerSetupSession, input.sessionId) : eagerSetupSession;
|
|
10711
11435
|
await runWorkspaceMutationForSandbox(
|
|
10712
11436
|
resolvedSandbox,
|
|
10713
11437
|
"eagerOwnedSandboxSetup",
|
|
10714
11438
|
async () => await runOwnedSandboxSetup(
|
|
10715
11439
|
agent,
|
|
10716
11440
|
ownedEstablished.session,
|
|
10717
|
-
|
|
11441
|
+
eagerCredentialSetupSession,
|
|
10718
11442
|
{
|
|
10719
11443
|
settings: runSettings,
|
|
10720
11444
|
environment: sandboxEnvironment,
|
|
@@ -10772,7 +11496,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10772
11496
|
} : {},
|
|
10773
11497
|
...sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox ? {
|
|
10774
11498
|
onToolspaceTokenSessionReady: async (tokenSession) => {
|
|
10775
|
-
const renewalSession =
|
|
11499
|
+
const renewalSession = setupBoxSession ?? tokenSession;
|
|
10776
11500
|
await attachToolspaceTokenRenewal(renewalSession);
|
|
10777
11501
|
}
|
|
10778
11502
|
} : {},
|
|
@@ -10785,9 +11509,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
10785
11509
|
}
|
|
10786
11510
|
} : {}
|
|
10787
11511
|
} : {},
|
|
10788
|
-
contextCompactionSignal: () =>
|
|
11512
|
+
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
10789
11513
|
contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
10790
|
-
callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
|
|
10791
11514
|
...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
|
|
10792
11515
|
});
|
|
10793
11516
|
};
|
|
@@ -10818,9 +11541,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
10818
11541
|
}
|
|
10819
11542
|
let stableToolCallIdsToClear = null;
|
|
10820
11543
|
let completedCurrentToolBatch = false;
|
|
10821
|
-
|
|
11544
|
+
let retainedScreenshotMetadata = null;
|
|
11545
|
+
const responseResult = await processModelResponseTerminalEvent({
|
|
10822
11546
|
event: next.value,
|
|
10823
|
-
state:
|
|
11547
|
+
state: modelResponseState,
|
|
10824
11548
|
dispatchId: modelUsageDispatchId,
|
|
10825
11549
|
settings,
|
|
10826
11550
|
db,
|
|
@@ -10851,7 +11575,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
10851
11575
|
model: turn.model,
|
|
10852
11576
|
...resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}
|
|
10853
11577
|
});
|
|
10854
|
-
if (
|
|
11578
|
+
if (responseResult.status === "processed") {
|
|
11579
|
+
streamSawPerResponseUsage ||= responseResult.usageReported;
|
|
10855
11580
|
currentToolBatchCallIds = /* @__PURE__ */ new Set();
|
|
10856
11581
|
currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
|
|
10857
11582
|
await reconcileConversationTruth();
|
|
@@ -10867,7 +11592,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10867
11592
|
} catch (limitError) {
|
|
10868
11593
|
let serializedRunState = null;
|
|
10869
11594
|
try {
|
|
10870
|
-
serializedRunState = String(
|
|
11595
|
+
serializedRunState = String(stream.state.toString());
|
|
10871
11596
|
} catch {
|
|
10872
11597
|
serializedRunState = null;
|
|
10873
11598
|
}
|
|
@@ -10889,7 +11614,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10889
11614
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
10890
11615
|
callId: pendingToolCall.callId,
|
|
10891
11616
|
callType: pendingToolCall.callType,
|
|
10892
|
-
callItem:
|
|
11617
|
+
callItem: pendingToolCall.callItem
|
|
10893
11618
|
});
|
|
10894
11619
|
if (!registered.accepted) {
|
|
10895
11620
|
throw new TurnAttemptFencedError(
|
|
@@ -10897,9 +11622,33 @@ function createRunAgentTurnActivity(services) {
|
|
|
10897
11622
|
);
|
|
10898
11623
|
}
|
|
10899
11624
|
currentToolBatchCallIds.add(pendingToolCall.callId);
|
|
11625
|
+
if (pendingToolCall.callName === "computer_screenshot") {
|
|
11626
|
+
computerScreenshotCallIds.add(pendingToolCall.callId);
|
|
11627
|
+
}
|
|
10900
11628
|
}
|
|
10901
11629
|
const completedToolCall = completedToolCallFromSdkEvent(next.value);
|
|
10902
11630
|
if (completedToolCall) {
|
|
11631
|
+
const typedScreenshot = typedScreenshotFromSdkEvent(next.value);
|
|
11632
|
+
if (typedScreenshot && typedScreenshot.callId === completedToolCall.callId && computerScreenshotCallIds.has(completedToolCall.callId)) {
|
|
11633
|
+
retainedScreenshotMetadata = await retainComputerScreenshot({
|
|
11634
|
+
db,
|
|
11635
|
+
objectStorage,
|
|
11636
|
+
accountId: input.accountId,
|
|
11637
|
+
workspaceId: input.workspaceId,
|
|
11638
|
+
sessionId: input.sessionId,
|
|
11639
|
+
turnId: activeTurnId,
|
|
11640
|
+
attemptId: input.attemptId,
|
|
11641
|
+
output: typedScreenshot
|
|
11642
|
+
});
|
|
11643
|
+
retainedScreenshotReceiptsByCallId.set(
|
|
11644
|
+
completedToolCall.callId,
|
|
11645
|
+
retainedScreenshotMetadata
|
|
11646
|
+
);
|
|
11647
|
+
}
|
|
11648
|
+
const durableResultItem = retainedScreenshotMetadata ? compactRetainedScreenshotHistory(
|
|
11649
|
+
[completedToolCall.resultItem],
|
|
11650
|
+
retainedScreenshotReceiptsByCallId
|
|
11651
|
+
)[0] ?? completedToolCall.resultItem : completedToolCall.resultItem;
|
|
10903
11652
|
const recorded = await recordPendingSessionToolCallResult(db, {
|
|
10904
11653
|
accountId: input.accountId,
|
|
10905
11654
|
workspaceId: input.workspaceId,
|
|
@@ -10909,7 +11658,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10909
11658
|
attemptId: input.attemptId,
|
|
10910
11659
|
callId: completedToolCall.callId,
|
|
10911
11660
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
10912
|
-
resultItem:
|
|
11661
|
+
resultItem: durableResultItem
|
|
10913
11662
|
});
|
|
10914
11663
|
if (!recorded.accepted) {
|
|
10915
11664
|
throw new TurnAttemptFencedError(
|
|
@@ -10928,7 +11677,16 @@ function createRunAgentTurnActivity(services) {
|
|
|
10928
11677
|
completedCurrentToolBatch = currentBatchIsStable;
|
|
10929
11678
|
}
|
|
10930
11679
|
}
|
|
10931
|
-
const normalized = normalizeSdkEvent(
|
|
11680
|
+
const normalized = normalizeSdkEvent(
|
|
11681
|
+
next.value,
|
|
11682
|
+
retainedScreenshotMetadata ? {
|
|
11683
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
11684
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available ? retainedScreenshotMetadata : {
|
|
11685
|
+
available: false,
|
|
11686
|
+
reason: retainedScreenshotMetadata.reason
|
|
11687
|
+
}
|
|
11688
|
+
} : {}
|
|
11689
|
+
);
|
|
10932
11690
|
for (const event of normalized) {
|
|
10933
11691
|
streamTiming.onEvent(event.type);
|
|
10934
11692
|
await batcher.push(event);
|
|
@@ -10952,6 +11710,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
10952
11710
|
currentToolBatchCallIds = /* @__PURE__ */ new Set();
|
|
10953
11711
|
currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
|
|
10954
11712
|
}
|
|
11713
|
+
for (const callId of stableToolCallIdsToClear) {
|
|
11714
|
+
computerScreenshotCallIds.delete(callId);
|
|
11715
|
+
}
|
|
10955
11716
|
}
|
|
10956
11717
|
}
|
|
10957
11718
|
} finally {
|
|
@@ -10964,7 +11725,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10964
11725
|
stream.completed.catch(() => void 0),
|
|
10965
11726
|
cancellationSignal
|
|
10966
11727
|
);
|
|
10967
|
-
if (
|
|
11728
|
+
if (!streamSawPerResponseUsage) {
|
|
10968
11729
|
const aggregateUsage = stream.state.usage;
|
|
10969
11730
|
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
10970
11731
|
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
@@ -11041,13 +11802,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
11041
11802
|
},
|
|
11042
11803
|
recordAttemptSignals: async () => {
|
|
11043
11804
|
if (!aggregateAuthoritative) return;
|
|
11044
|
-
|
|
11045
|
-
|
|
11046
|
-
modelResponseUsageState.providerContextRevision += 1;
|
|
11047
|
-
}
|
|
11048
|
-
if (aggregateInput !== null && aggregateInput > 0) {
|
|
11049
|
-
await setLastInputTokensFenced(aggregateInput);
|
|
11050
|
-
}
|
|
11805
|
+
modelResponseState.contextSignal = null;
|
|
11806
|
+
await setLastInputTokensFenced(null);
|
|
11051
11807
|
}
|
|
11052
11808
|
});
|
|
11053
11809
|
}
|
|
@@ -11056,6 +11812,13 @@ function createRunAgentTurnActivity(services) {
|
|
|
11056
11812
|
await reconcileConversationTruth();
|
|
11057
11813
|
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
11058
11814
|
const humanInputInterruptions = runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
|
|
11815
|
+
const latestWorkspace = await getWorkspace2(db, input.workspaceId);
|
|
11816
|
+
if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
11817
|
+
assertWorkspaceHumanInputAllowed(
|
|
11818
|
+
resolveWorkspaceAgentHumanInputEnabled(latestWorkspace.settings),
|
|
11819
|
+
"interruption",
|
|
11820
|
+
humanInputInterruptions.length > 0
|
|
11821
|
+
);
|
|
11059
11822
|
const humanInputRequests = await Promise.all(
|
|
11060
11823
|
humanInputInterruptions.map(async (interruption) => {
|
|
11061
11824
|
const id = stableHumanInputRequestId(
|
|
@@ -11121,17 +11884,13 @@ function createRunAgentTurnActivity(services) {
|
|
|
11121
11884
|
})) {
|
|
11122
11885
|
return claimedResult({ status: "cancelled" });
|
|
11123
11886
|
}
|
|
11887
|
+
await finalizeTurnOpStreamOps();
|
|
11124
11888
|
activityStatus = "requires_action";
|
|
11125
11889
|
return claimedResult({ status: "requires_action" });
|
|
11126
11890
|
}
|
|
11127
|
-
const finalOutput = String(
|
|
11128
|
-
await reconcileConversationTruth();
|
|
11129
|
-
|
|
11130
|
-
try {
|
|
11131
|
-
await machinePrimarySession.finalizeOpStreamOps();
|
|
11132
|
-
} catch {
|
|
11133
|
-
}
|
|
11134
|
-
}
|
|
11891
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
11892
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
11893
|
+
await finalizeTurnOpStreamOps();
|
|
11135
11894
|
if (!await settle({
|
|
11136
11895
|
events: [
|
|
11137
11896
|
{
|
|
@@ -11189,8 +11948,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11189
11948
|
sessionId: input.sessionId,
|
|
11190
11949
|
turnId: activeTurnId,
|
|
11191
11950
|
reason: recoveryKind,
|
|
11192
|
-
|
|
11193
|
-
error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
|
|
11951
|
+
...safeErrorDiagnostic(attemptError),
|
|
11194
11952
|
signalTokens: compactionNeeded?.signalTokens,
|
|
11195
11953
|
thresholdTokens: compactionNeeded?.thresholdTokens
|
|
11196
11954
|
});
|
|
@@ -11232,13 +11990,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
11232
11990
|
);
|
|
11233
11991
|
compactionRequestCleared = landmark.requestConsumed;
|
|
11234
11992
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
11235
|
-
compactionFailureMessage = String(
|
|
11236
|
-
redact(compactionFailureReasonFromError(compactError))
|
|
11237
|
-
);
|
|
11993
|
+
compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
|
|
11238
11994
|
observability.warn("context compaction recovery compaction failed", {
|
|
11239
11995
|
sessionId: input.sessionId,
|
|
11240
11996
|
turnId: activeTurnId,
|
|
11241
|
-
|
|
11997
|
+
...safeErrorDiagnostic(compactError)
|
|
11242
11998
|
});
|
|
11243
11999
|
}
|
|
11244
12000
|
if (!compactionHandled) {
|
|
@@ -11311,6 +12067,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11311
12067
|
turnMetricOutcome = "cancelled";
|
|
11312
12068
|
return claimedResult({ status: "cancelled" });
|
|
11313
12069
|
}
|
|
12070
|
+
acknowledgeRecoveryQuiescence();
|
|
11314
12071
|
await publishDurableSessionEvents(
|
|
11315
12072
|
bus,
|
|
11316
12073
|
input.workspaceId,
|
|
@@ -11328,7 +12085,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11328
12085
|
} catch (recoveryError) {
|
|
11329
12086
|
console.error(
|
|
11330
12087
|
"sandbox lease supersession recovery failed",
|
|
11331
|
-
safeErrorDiagnostic(recoveryError
|
|
12088
|
+
safeErrorDiagnostic(recoveryError)
|
|
11332
12089
|
);
|
|
11333
12090
|
throw recoveryError;
|
|
11334
12091
|
}
|
|
@@ -11355,6 +12112,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11355
12112
|
turnMetricOutcome = "cancelled";
|
|
11356
12113
|
return claimedResult({ status: "cancelled" });
|
|
11357
12114
|
}
|
|
12115
|
+
acknowledgeRecoveryQuiescence();
|
|
11358
12116
|
await publishDurableSessionEvents(
|
|
11359
12117
|
bus,
|
|
11360
12118
|
input.workspaceId,
|
|
@@ -11370,7 +12128,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11370
12128
|
} catch (recoveryError) {
|
|
11371
12129
|
console.error(
|
|
11372
12130
|
"sandbox deadline rotation recovery failed",
|
|
11373
|
-
safeErrorDiagnostic(recoveryError
|
|
12131
|
+
safeErrorDiagnostic(recoveryError)
|
|
11374
12132
|
);
|
|
11375
12133
|
throw recoveryError;
|
|
11376
12134
|
}
|
|
@@ -11393,6 +12151,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11393
12151
|
turnMetricOutcome = "cancelled";
|
|
11394
12152
|
return claimedResult({ status: "cancelled" });
|
|
11395
12153
|
}
|
|
12154
|
+
acknowledgeRecoveryQuiescence();
|
|
11396
12155
|
await publishDurableSessionEvents(
|
|
11397
12156
|
bus,
|
|
11398
12157
|
input.workspaceId,
|
|
@@ -11405,7 +12164,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11405
12164
|
} catch (recoveryError) {
|
|
11406
12165
|
console.error(
|
|
11407
12166
|
"worker-shutdown recovery checkpoint failed",
|
|
11408
|
-
safeErrorDiagnostic(recoveryError
|
|
12167
|
+
safeErrorDiagnostic(recoveryError)
|
|
11409
12168
|
);
|
|
11410
12169
|
throw recoveryError;
|
|
11411
12170
|
}
|
|
@@ -11477,11 +12236,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
11477
12236
|
await reconcileConversationTruth({ requireDurable: true });
|
|
11478
12237
|
}
|
|
11479
12238
|
checkpointDurable = true;
|
|
11480
|
-
} catch
|
|
12239
|
+
} catch {
|
|
11481
12240
|
observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
|
|
11482
|
-
|
|
11483
|
-
|
|
11484
|
-
|
|
12241
|
+
errorClass: "CodexCheckpointOperationError",
|
|
12242
|
+
errorCode: "codex_lease_loss_checkpoint_failed",
|
|
12243
|
+
origin: "worker"
|
|
11485
12244
|
});
|
|
11486
12245
|
}
|
|
11487
12246
|
const settlement = await settleCodexCredentialLeaseLoss(db, {
|
|
@@ -11556,16 +12315,16 @@ function createRunAgentTurnActivity(services) {
|
|
|
11556
12315
|
await flushRuntimeBatcher();
|
|
11557
12316
|
await reconcileConversationTruth({ requireDurable: true });
|
|
11558
12317
|
checkpointDurable = true;
|
|
11559
|
-
} catch
|
|
12318
|
+
} catch {
|
|
11560
12319
|
observability.incrementCounter({
|
|
11561
12320
|
name: "opengeni_codex_failover_checkpoints_total",
|
|
11562
12321
|
help: "Durable Codex failover checkpoint attempts by outcome.",
|
|
11563
12322
|
labels: { workspace_key: codexWorkspaceKey, outcome: "failed" }
|
|
11564
12323
|
});
|
|
11565
12324
|
observability.warn("Codex failover checkpoint failed; refusing automatic replay", {
|
|
11566
|
-
|
|
11567
|
-
|
|
11568
|
-
|
|
12325
|
+
errorClass: "CodexCheckpointOperationError",
|
|
12326
|
+
errorCode: "codex_failover_checkpoint_failed",
|
|
12327
|
+
origin: "worker"
|
|
11569
12328
|
});
|
|
11570
12329
|
}
|
|
11571
12330
|
if (checkpointDurable) {
|
|
@@ -11990,6 +12749,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11990
12749
|
return claimedResult({ status: "cancelled" });
|
|
11991
12750
|
}
|
|
11992
12751
|
if (recovery.action === "recovering") {
|
|
12752
|
+
acknowledgeRecoveryQuiescence();
|
|
11993
12753
|
await publishDurableSessionEvents(
|
|
11994
12754
|
bus,
|
|
11995
12755
|
input.workspaceId,
|
|
@@ -12002,7 +12762,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
12002
12762
|
return claimedResult({ status: "recovering" });
|
|
12003
12763
|
}
|
|
12004
12764
|
}
|
|
12005
|
-
const failure =
|
|
12765
|
+
const failure = agentRunFailurePayload(error, { isCodexTurn });
|
|
12006
12766
|
if (isSessionEventPersistenceError(error)) {
|
|
12007
12767
|
observability.error("session event persistence failed", {
|
|
12008
12768
|
accountId: input.accountId,
|
|
@@ -12023,7 +12783,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
12023
12783
|
dbColumn: error.details.database.column,
|
|
12024
12784
|
dbDataType: error.details.database.dataType,
|
|
12025
12785
|
dbConstraint: error.details.database.constraint,
|
|
12026
|
-
dbRoutine: error.details.database.routine
|
|
12786
|
+
dbRoutine: error.details.database.routine,
|
|
12787
|
+
error: error.message
|
|
12027
12788
|
});
|
|
12028
12789
|
}
|
|
12029
12790
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
@@ -12055,6 +12816,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
12055
12816
|
turnMetricOutcome = "cancelled";
|
|
12056
12817
|
return claimedResult({ status: "cancelled" });
|
|
12057
12818
|
}
|
|
12819
|
+
acknowledgeRecoveryQuiescence();
|
|
12058
12820
|
await publishDurableSessionEvents(
|
|
12059
12821
|
bus,
|
|
12060
12822
|
input.workspaceId,
|
|
@@ -12135,17 +12897,29 @@ function createRunAgentTurnActivity(services) {
|
|
|
12135
12897
|
const credentialSessionToClear = runCredentialSession;
|
|
12136
12898
|
runCredentialSession = null;
|
|
12137
12899
|
if (credentialSessionToClear) {
|
|
12138
|
-
await
|
|
12139
|
-
|
|
12140
|
-
|
|
12900
|
+
const clearAttemptCredentials = async () => await clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
12901
|
+
sessionId: input.sessionId,
|
|
12902
|
+
attemptId: input.attemptId,
|
|
12903
|
+
executionGeneration
|
|
12904
|
+
});
|
|
12905
|
+
await clearAttemptCredentialsWithSettledFence({
|
|
12906
|
+
activityStatus,
|
|
12907
|
+
runWorkspaceFencedClear: async () => await runWorkspaceMutationForSandbox(
|
|
12908
|
+
requireResolvedSandboxForMutation(
|
|
12909
|
+
"Run credential cleanup has no exact sandbox lease target"
|
|
12910
|
+
),
|
|
12911
|
+
"runCredentialAttemptClear",
|
|
12912
|
+
clearAttemptCredentials
|
|
12141
12913
|
),
|
|
12142
|
-
|
|
12143
|
-
|
|
12144
|
-
|
|
12145
|
-
|
|
12146
|
-
|
|
12147
|
-
|
|
12148
|
-
|
|
12914
|
+
clearExactAttempt: clearAttemptCredentials,
|
|
12915
|
+
onSettledAttemptFence: () => {
|
|
12916
|
+
observability.info("retrying exact run credential cleanup after turn settlement", {
|
|
12917
|
+
"opengeni.session_id": input.sessionId,
|
|
12918
|
+
"opengeni.turn_id": turnId ?? "",
|
|
12919
|
+
"opengeni.attempt_id": input.attemptId
|
|
12920
|
+
});
|
|
12921
|
+
}
|
|
12922
|
+
});
|
|
12149
12923
|
}
|
|
12150
12924
|
await drainAttemptOwnedSandboxWriters({
|
|
12151
12925
|
// Normal turn completion owns the same process boundary as
|
|
@@ -12203,18 +12977,18 @@ function createRunAgentTurnActivity(services) {
|
|
|
12203
12977
|
onReceiptFailure: (error) => {
|
|
12204
12978
|
console.error(
|
|
12205
12979
|
"agent turn quiescence receipt exhausted; signalling proof",
|
|
12206
|
-
safeErrorDiagnostic(error
|
|
12980
|
+
safeErrorDiagnostic(error)
|
|
12207
12981
|
);
|
|
12208
12982
|
},
|
|
12209
12983
|
onPublishFailure: (error) => {
|
|
12210
12984
|
console.error(
|
|
12211
12985
|
"agent turn quiescence event fanout failed",
|
|
12212
|
-
safeErrorDiagnostic(error
|
|
12986
|
+
safeErrorDiagnostic(error)
|
|
12213
12987
|
);
|
|
12214
12988
|
},
|
|
12215
12989
|
onSignalFailure: (error, attempt, retryMs) => {
|
|
12216
12990
|
console.error("agent turn quiescence proof signal failed; retrying", {
|
|
12217
|
-
error: safeErrorDiagnostic(error
|
|
12991
|
+
error: safeErrorDiagnostic(error),
|
|
12218
12992
|
attempt,
|
|
12219
12993
|
retryMs
|
|
12220
12994
|
});
|
|
@@ -12400,7 +13174,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
12400
13174
|
sandboxToRelease.release().catch((releaseError) => {
|
|
12401
13175
|
console.error(
|
|
12402
13176
|
"sandbox lease release failed (turn outcome unaffected)",
|
|
12403
|
-
safeErrorDiagnostic(releaseError
|
|
13177
|
+
safeErrorDiagnostic(releaseError)
|
|
12404
13178
|
);
|
|
12405
13179
|
}),
|
|
12406
13180
|
finalizerSignal
|
|
@@ -12410,7 +13184,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
12410
13184
|
finalizationError ??= error;
|
|
12411
13185
|
console.error(
|
|
12412
13186
|
"agent turn finalization failed (turn outcome unaffected)",
|
|
12413
|
-
safeErrorDiagnostic(error
|
|
13187
|
+
safeErrorDiagnostic(error)
|
|
12414
13188
|
);
|
|
12415
13189
|
} finally {
|
|
12416
13190
|
if (!sandboxResumeController.signal.aborted) {
|
|
@@ -12465,10 +13239,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
12465
13239
|
"opengeni.duration_ms": Math.round(durationSeconds * 1e3),
|
|
12466
13240
|
"opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1e3)
|
|
12467
13241
|
},
|
|
12468
|
-
error: finalizationError || activityError ? safeErrorForTelemetry(
|
|
12469
|
-
finalizationError ?? activityError,
|
|
12470
|
-
(value) => String(redact(value))
|
|
12471
|
-
) : void 0
|
|
13242
|
+
error: finalizationError || activityError ? safeErrorForTelemetry(finalizationError ?? activityError) : void 0
|
|
12472
13243
|
});
|
|
12473
13244
|
assertPhysicalToolQuiescenceForCancellation({
|
|
12474
13245
|
acknowledgeQuiescence,
|
|
@@ -12531,6 +13302,15 @@ function isTransientProviderError(error) {
|
|
|
12531
13302
|
);
|
|
12532
13303
|
}
|
|
12533
13304
|
function agentRunFailurePayload(error, options = {}) {
|
|
13305
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
13306
|
+
const underlying = isSessionEventPersistenceError(error.cause) ? agentRunFailurePayload(error.cause, options) : {
|
|
13307
|
+
error: error.cause instanceof Error ? error.cause.message : String(error.cause)
|
|
13308
|
+
};
|
|
13309
|
+
return {
|
|
13310
|
+
...underlying,
|
|
13311
|
+
historyPersistenceStage: error.stage
|
|
13312
|
+
};
|
|
13313
|
+
}
|
|
12534
13314
|
const message = error instanceof Error ? error.message : String(error);
|
|
12535
13315
|
const status = typeof error === "object" && error !== null ? Number(
|
|
12536
13316
|
error.status ?? error.statusCode
|
|
@@ -12545,10 +13325,8 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
12545
13325
|
}
|
|
12546
13326
|
if (isSessionEventPersistenceError(error)) {
|
|
12547
13327
|
const { details } = error;
|
|
12548
|
-
const eventLabel = details.eventTypes.join(", ") || "session events";
|
|
12549
|
-
const failureLabel = details.code === "db_deadlock" ? "Database deadlock" : details.code === "db_serialization_failure" ? "Database serialization failure" : "Database failure";
|
|
12550
13328
|
return {
|
|
12551
|
-
error:
|
|
13329
|
+
error: error.message,
|
|
12552
13330
|
code: details.code,
|
|
12553
13331
|
detail: details.retryOutcome === "exhausted" ? `The idempotent persistence transaction failed after ${details.attempts} attempts.` : "The database rejected the idempotent persistence transaction.",
|
|
12554
13332
|
correlationId: details.correlationId,
|
|
@@ -12590,7 +13368,8 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
12590
13368
|
return {
|
|
12591
13369
|
error: "A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
|
|
12592
13370
|
code: "mcp_transport_unavailable",
|
|
12593
|
-
retryable: true
|
|
13371
|
+
retryable: true,
|
|
13372
|
+
detail: message
|
|
12594
13373
|
};
|
|
12595
13374
|
}
|
|
12596
13375
|
if (status === 429 || code === "rate_limit_exceeded" || /(?:too many requests|rate.?limit|\b429\b)/i.test(message)) {
|
|
@@ -12719,7 +13498,13 @@ function pendingToolCallFromSdkEvent(event) {
|
|
|
12719
13498
|
if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
|
|
12720
13499
|
return null;
|
|
12721
13500
|
}
|
|
12722
|
-
|
|
13501
|
+
const callItem = normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]');
|
|
13502
|
+
return {
|
|
13503
|
+
callId,
|
|
13504
|
+
callType,
|
|
13505
|
+
callName: typeof raw.name === "string" ? raw.name : null,
|
|
13506
|
+
callItem
|
|
13507
|
+
};
|
|
12723
13508
|
}
|
|
12724
13509
|
function completedToolCallFromSdkEvent(event) {
|
|
12725
13510
|
if (!event || typeof event !== "object") return null;
|
|
@@ -12730,7 +13515,11 @@ function completedToolCallFromSdkEvent(event) {
|
|
|
12730
13515
|
}
|
|
12731
13516
|
const raw = item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem) ? item.rawItem : {};
|
|
12732
13517
|
const callId = raw.callId ?? raw.call_id ?? item.id;
|
|
12733
|
-
|
|
13518
|
+
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
13519
|
+
return {
|
|
13520
|
+
callId,
|
|
13521
|
+
resultItem: normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]')
|
|
13522
|
+
};
|
|
12734
13523
|
}
|
|
12735
13524
|
var BudgetExhaustedError = class extends Error {
|
|
12736
13525
|
constructor(message, serializedRunState) {
|
|
@@ -12925,12 +13714,7 @@ async function recordAuthoritativeModelCallFact(input) {
|
|
|
12925
13714
|
});
|
|
12926
13715
|
} catch (error) {
|
|
12927
13716
|
input.observability.warn("model call fact persist failed", {
|
|
12928
|
-
|
|
12929
|
-
workspaceId: input.workspaceId,
|
|
12930
|
-
sessionId: input.sessionId,
|
|
12931
|
-
turnId: input.turnId,
|
|
12932
|
-
sourceKey: input.sourceKey,
|
|
12933
|
-
error: error instanceof Error ? error.message : String(error)
|
|
13717
|
+
...safeErrorDiagnostic(error)
|
|
12934
13718
|
});
|
|
12935
13719
|
}
|
|
12936
13720
|
}
|
|
@@ -13108,21 +13892,69 @@ function startOfUtcMonth3(date = /* @__PURE__ */ new Date()) {
|
|
|
13108
13892
|
}
|
|
13109
13893
|
|
|
13110
13894
|
// src/activities/file-upload-reaper.ts
|
|
13111
|
-
import {
|
|
13895
|
+
import {
|
|
13896
|
+
claimDueTranscriptionRecordingObjectCleanup,
|
|
13897
|
+
claimExpiredFileUploadCleanup,
|
|
13898
|
+
completeDueTranscriptionRecordingObjectCleanup,
|
|
13899
|
+
completeExpiredFileUploadCleanup,
|
|
13900
|
+
purgeExpiredTranscriptionRecordings
|
|
13901
|
+
} from "@opengeni/db";
|
|
13112
13902
|
var FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1e3;
|
|
13113
13903
|
var FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
13114
13904
|
var FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
|
|
13905
|
+
var SAFE_CLEANUP_ERROR_CATEGORIES = /* @__PURE__ */ new Set([
|
|
13906
|
+
"aborted",
|
|
13907
|
+
"conflict",
|
|
13908
|
+
"invalid_response",
|
|
13909
|
+
"network",
|
|
13910
|
+
"not_found",
|
|
13911
|
+
"permission_denied",
|
|
13912
|
+
"provider",
|
|
13913
|
+
"timeout",
|
|
13914
|
+
"unavailable"
|
|
13915
|
+
]);
|
|
13916
|
+
function cleanupErrorAttributes(error) {
|
|
13917
|
+
let errorCategory = "provider_error";
|
|
13918
|
+
let errorStatus;
|
|
13919
|
+
if (error && typeof error === "object") {
|
|
13920
|
+
const rawCode = error.code;
|
|
13921
|
+
if (typeof rawCode === "string" && SAFE_CLEANUP_ERROR_CATEGORIES.has(rawCode)) {
|
|
13922
|
+
errorCategory = rawCode;
|
|
13923
|
+
} else if (rawCode !== void 0) {
|
|
13924
|
+
errorCategory = "unknown";
|
|
13925
|
+
}
|
|
13926
|
+
const rawStatus = Number(
|
|
13927
|
+
error.status ?? error.statusCode
|
|
13928
|
+
);
|
|
13929
|
+
if (Number.isInteger(rawStatus) && rawStatus >= 100 && rawStatus <= 599) {
|
|
13930
|
+
errorStatus = rawStatus;
|
|
13931
|
+
}
|
|
13932
|
+
}
|
|
13933
|
+
return { errorCategory, ...errorStatus === void 0 ? {} : { errorStatus } };
|
|
13934
|
+
}
|
|
13115
13935
|
function createFileUploadReaperActivities(services, options = {}) {
|
|
13116
13936
|
const graceMs = options.graceMs ?? FILE_UPLOAD_CLEANUP_GRACE_MS;
|
|
13117
13937
|
const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
|
|
13118
13938
|
const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
|
|
13119
13939
|
const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
|
|
13940
|
+
const claimFileUploads = options.claimFileUploads ?? claimExpiredFileUploadCleanup;
|
|
13941
|
+
const completeFileUpload = options.completeFileUpload ?? completeExpiredFileUploadCleanup;
|
|
13942
|
+
const claimTranscriptionObjects = options.claimTranscriptionObjects ?? claimDueTranscriptionRecordingObjectCleanup;
|
|
13943
|
+
const completeTranscriptionObject = options.completeTranscriptionObject ?? completeDueTranscriptionRecordingObjectCleanup;
|
|
13944
|
+
const purgeTranscriptionRecordings = options.purgeTranscriptionRecordings ?? purgeExpiredTranscriptionRecordings;
|
|
13120
13945
|
async function reapExpiredFileUploads2() {
|
|
13121
13946
|
const { db, objectStorage, observability } = await services();
|
|
13122
13947
|
if (!objectStorage) {
|
|
13948
|
+
try {
|
|
13949
|
+
await purgeTranscriptionRecordings(db, { graceMs, limit: batchSize });
|
|
13950
|
+
} catch (error) {
|
|
13951
|
+
observability.warn("expired transcription recording metadata purge failed", {
|
|
13952
|
+
...cleanupErrorAttributes(error)
|
|
13953
|
+
});
|
|
13954
|
+
}
|
|
13123
13955
|
return { claimed: 0, deleted: 0, failed: 0 };
|
|
13124
13956
|
}
|
|
13125
|
-
const claims = await
|
|
13957
|
+
const claims = await claimFileUploads(db, {
|
|
13126
13958
|
graceMs,
|
|
13127
13959
|
claimTimeoutMs,
|
|
13128
13960
|
limit: batchSize
|
|
@@ -13132,7 +13964,7 @@ function createFileUploadReaperActivities(services, options = {}) {
|
|
|
13132
13964
|
for (const claim of claims) {
|
|
13133
13965
|
try {
|
|
13134
13966
|
await deleteObject(objectStorage, claim.objectKey);
|
|
13135
|
-
const settled = await
|
|
13967
|
+
const settled = await completeFileUpload(db, claim);
|
|
13136
13968
|
if (!settled) {
|
|
13137
13969
|
throw new Error("cleanup claim no longer owns a reclaimable upload");
|
|
13138
13970
|
}
|
|
@@ -13143,18 +13975,58 @@ function createFileUploadReaperActivities(services, options = {}) {
|
|
|
13143
13975
|
workspaceId: claim.workspaceId,
|
|
13144
13976
|
uploadId: claim.uploadId,
|
|
13145
13977
|
fileId: claim.fileId,
|
|
13146
|
-
|
|
13978
|
+
...cleanupErrorAttributes(error)
|
|
13147
13979
|
});
|
|
13148
13980
|
}
|
|
13149
13981
|
}
|
|
13150
|
-
|
|
13982
|
+
const recordingClaims = await claimTranscriptionObjects(db, {
|
|
13983
|
+
graceMs,
|
|
13984
|
+
claimTimeoutMs,
|
|
13985
|
+
limit: batchSize
|
|
13986
|
+
});
|
|
13987
|
+
for (const claim of recordingClaims) {
|
|
13988
|
+
try {
|
|
13989
|
+
await deleteObject(objectStorage, claim.objectKey);
|
|
13990
|
+
const settled = await completeTranscriptionObject(db, claim);
|
|
13991
|
+
if (!settled) {
|
|
13992
|
+
throw new Error("cleanup claim no longer owns a transcription recording object");
|
|
13993
|
+
}
|
|
13994
|
+
deleted += 1;
|
|
13995
|
+
} catch (error) {
|
|
13996
|
+
failed += 1;
|
|
13997
|
+
observability.warn(
|
|
13998
|
+
"expired transcription recording object cleanup failed; claim remains reclaimable",
|
|
13999
|
+
{
|
|
14000
|
+
workspaceId: claim.workspaceId,
|
|
14001
|
+
recordingId: claim.recordingId,
|
|
14002
|
+
...cleanupErrorAttributes(error)
|
|
14003
|
+
}
|
|
14004
|
+
);
|
|
14005
|
+
}
|
|
14006
|
+
}
|
|
14007
|
+
let transcriptionRecordingsPurged = 0;
|
|
14008
|
+
try {
|
|
14009
|
+
transcriptionRecordingsPurged = await purgeTranscriptionRecordings(db, {
|
|
14010
|
+
graceMs,
|
|
14011
|
+
limit: batchSize
|
|
14012
|
+
});
|
|
14013
|
+
} catch (error) {
|
|
14014
|
+
observability.warn("expired transcription recording metadata purge failed", {
|
|
14015
|
+
...cleanupErrorAttributes(error)
|
|
14016
|
+
});
|
|
14017
|
+
}
|
|
14018
|
+
const claimed = claims.length + recordingClaims.length;
|
|
14019
|
+
if (claimed > 0 || transcriptionRecordingsPurged > 0) {
|
|
13151
14020
|
observability.info("expired file upload cleanup swept", {
|
|
13152
|
-
claimed
|
|
14021
|
+
claimed,
|
|
13153
14022
|
deleted,
|
|
13154
|
-
failed
|
|
14023
|
+
failed,
|
|
14024
|
+
fileUploads: claims.length,
|
|
14025
|
+
transcriptionRecordingObjects: recordingClaims.length,
|
|
14026
|
+
transcriptionRecordingsPurged
|
|
13155
14027
|
});
|
|
13156
14028
|
}
|
|
13157
|
-
return { claimed
|
|
14029
|
+
return { claimed, deleted, failed };
|
|
13158
14030
|
}
|
|
13159
14031
|
return { reapExpiredFileUploads: reapExpiredFileUploads2 };
|
|
13160
14032
|
}
|
|
@@ -13215,6 +14087,7 @@ import {
|
|
|
13215
14087
|
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
13216
14088
|
resolveModalCheckpointProviderBinding,
|
|
13217
14089
|
resolveModalCheckpointProviderBindingForSession as resolveModalCheckpointProviderBindingForSession3,
|
|
14090
|
+
sandboxBackendForSdkBackendId as sandboxBackendForSdkBackendId2,
|
|
13218
14091
|
sandboxCommandExitCode,
|
|
13219
14092
|
sandboxCommandStillRunning,
|
|
13220
14093
|
sweepModalOrphanSandboxes,
|
|
@@ -14132,7 +15005,8 @@ async function probeDrainableProviderReadiness(settings, lease) {
|
|
|
14132
15005
|
if (!lease.instanceId || !lease.resumeState) {
|
|
14133
15006
|
throw new Error("Expired workspace capture has no resumable provider identity");
|
|
14134
15007
|
}
|
|
14135
|
-
const
|
|
15008
|
+
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
15009
|
+
const backend = sandboxBackendForSdkBackendId2(durableBackendId) ?? durableBackendId;
|
|
14136
15010
|
let established;
|
|
14137
15011
|
try {
|
|
14138
15012
|
established = await establishSandboxSessionFromEnvelope3(settings, lease.resumeState, {
|
|
@@ -14362,7 +15236,8 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
14362
15236
|
}
|
|
14363
15237
|
}
|
|
14364
15238
|
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend) {
|
|
14365
|
-
const
|
|
15239
|
+
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
15240
|
+
const backend = sandboxBackendForSdkBackendId2(durableBackendId) ?? durableBackendId;
|
|
14366
15241
|
if (!backend || backend === "none") {
|
|
14367
15242
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
14368
15243
|
}
|
|
@@ -14541,6 +15416,7 @@ import {
|
|
|
14541
15416
|
isCodexBilledTurn as isCodexBilledTurn2,
|
|
14542
15417
|
markScheduledTaskRunFailedIfQueued,
|
|
14543
15418
|
recordUsageEvent as recordUsageEvent3,
|
|
15419
|
+
requireScheduledTaskTargetInTransaction,
|
|
14544
15420
|
requireSession as requireSession3,
|
|
14545
15421
|
setTemporalWorkflowId,
|
|
14546
15422
|
settleScheduledTaskRunInTransaction,
|
|
@@ -14663,7 +15539,11 @@ function createScheduledTaskActivities(services) {
|
|
|
14663
15539
|
const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
14664
15540
|
const goalSpec = task.agentConfig.goal ?? null;
|
|
14665
15541
|
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
14666
|
-
|
|
15542
|
+
const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
15543
|
+
if (task.runMode === "existing_session" && !existingSessionId) {
|
|
15544
|
+
throw new Error("scheduled task target session is unavailable");
|
|
15545
|
+
}
|
|
15546
|
+
if (task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && !existingSessionId) {
|
|
14667
15547
|
const variableSet = task.variableSetId ? await getVariableSet(db, task.workspaceId, task.variableSetId) : null;
|
|
14668
15548
|
if (task.variableSetId && !variableSet) {
|
|
14669
15549
|
throw new Error(`variable set not found: ${task.variableSetId}`);
|
|
@@ -14830,7 +15710,7 @@ function createScheduledTaskActivities(services) {
|
|
|
14830
15710
|
workflowWakeRevision: scheduledUpdate.workflowWakeRevision
|
|
14831
15711
|
};
|
|
14832
15712
|
} else {
|
|
14833
|
-
const session = await requireSession3(db, task.workspaceId,
|
|
15713
|
+
const session = await requireSession3(db, task.workspaceId, existingSessionId);
|
|
14834
15714
|
assertReusableSessionRevivable(session.status);
|
|
14835
15715
|
if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
|
|
14836
15716
|
throw new Error(
|
|
@@ -14842,7 +15722,7 @@ function createScheduledTaskActivities(services) {
|
|
|
14842
15722
|
"scheduled task OpenGeni Slack bot binding does not match its reusable session"
|
|
14843
15723
|
);
|
|
14844
15724
|
}
|
|
14845
|
-
const reusableGoal = goalSpec && run.status === "queued" ? await upsertSessionGoal(db, {
|
|
15725
|
+
const reusableGoal = task.runMode === "reusable_session" && goalSpec && run.status === "queued" ? await upsertSessionGoal(db, {
|
|
14846
15726
|
accountId: task.accountId,
|
|
14847
15727
|
workspaceId: task.workspaceId,
|
|
14848
15728
|
sessionId: session.id,
|
|
@@ -14899,6 +15779,13 @@ function createScheduledTaskActivities(services) {
|
|
|
14899
15779
|
personalConnectionDelegations: taskPersonalConnectionDelegations
|
|
14900
15780
|
},
|
|
14901
15781
|
async (tx, wakeEventId) => {
|
|
15782
|
+
if (task.runMode === "existing_session") {
|
|
15783
|
+
await requireScheduledTaskTargetInTransaction(tx, {
|
|
15784
|
+
workspaceId: task.workspaceId,
|
|
15785
|
+
taskId: task.id,
|
|
15786
|
+
targetSessionId: session.id
|
|
15787
|
+
});
|
|
15788
|
+
}
|
|
14902
15789
|
if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
|
|
14903
15790
|
await settleScheduledTaskRunInTransaction(tx, {
|
|
14904
15791
|
workspaceId: task.workspaceId,
|
|
@@ -15329,9 +16216,6 @@ import {
|
|
|
15329
16216
|
markWarmLeaseInstanceLost as markWarmLeaseInstanceLost3,
|
|
15330
16217
|
recordWarmingSandboxCreated as recordWarmingSandboxCreated2,
|
|
15331
16218
|
releaseLeaseHolder as releaseLeaseHolder2,
|
|
15332
|
-
sanitizeEventPayload,
|
|
15333
|
-
sanitizeEventString,
|
|
15334
|
-
sanitizeMemoryText,
|
|
15335
16219
|
touchLeaseHolder as touchLeaseHolder2,
|
|
15336
16220
|
updateRigChangeStatus
|
|
15337
16221
|
} from "@opengeni/db";
|
|
@@ -15344,7 +16228,6 @@ import {
|
|
|
15344
16228
|
serializeEstablishedSandboxEnvelope,
|
|
15345
16229
|
tagModalSandbox as tagModalSandbox2
|
|
15346
16230
|
} from "@opengeni/runtime";
|
|
15347
|
-
var OUTPUT_TAIL_LIMIT = 64 * 1024;
|
|
15348
16231
|
var RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS = 2 * 6e4;
|
|
15349
16232
|
var RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 6e4;
|
|
15350
16233
|
var RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 1e4;
|
|
@@ -15479,15 +16362,6 @@ function cleanupWaitMs(lifecycle) {
|
|
|
15479
16362
|
Math.min(RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS, lifecycle.cleanupDeadlineAtMs - Date.now())
|
|
15480
16363
|
);
|
|
15481
16364
|
}
|
|
15482
|
-
function tail(value, limit = OUTPUT_TAIL_LIMIT) {
|
|
15483
|
-
return value.length > limit ? value.slice(-limit) : value;
|
|
15484
|
-
}
|
|
15485
|
-
function scrubVerificationOutput(value) {
|
|
15486
|
-
return sanitizeMemoryText(sanitizeEventString(value)).text;
|
|
15487
|
-
}
|
|
15488
|
-
function scrubVerificationPayload(value) {
|
|
15489
|
-
return sanitizeEventPayload(value);
|
|
15490
|
-
}
|
|
15491
16365
|
function systemGrant(rig) {
|
|
15492
16366
|
return {
|
|
15493
16367
|
accountId: rig.accountId,
|
|
@@ -15834,7 +16708,7 @@ async function runCommand(session, command, timeoutMs, commandRunner) {
|
|
|
15834
16708
|
const result = await commandRunner(session, args);
|
|
15835
16709
|
return {
|
|
15836
16710
|
exitCode: sandboxCommandExitCode2(result),
|
|
15837
|
-
output:
|
|
16711
|
+
output: sandboxCommandOutput(result)
|
|
15838
16712
|
};
|
|
15839
16713
|
}
|
|
15840
16714
|
function setupAppendCommand(change) {
|
|
@@ -15965,7 +16839,7 @@ function createRigVerificationActivities(services) {
|
|
|
15965
16839
|
verification.passed = false;
|
|
15966
16840
|
const updated2 = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
15967
16841
|
status: "rejected",
|
|
15968
|
-
verification
|
|
16842
|
+
verification
|
|
15969
16843
|
});
|
|
15970
16844
|
await recordRigAuditEvent(db, {
|
|
15971
16845
|
grant,
|
|
@@ -15992,7 +16866,7 @@ function createRigVerificationActivities(services) {
|
|
|
15992
16866
|
);
|
|
15993
16867
|
checkResults.push({
|
|
15994
16868
|
name: check.name,
|
|
15995
|
-
command:
|
|
16869
|
+
command: check.command,
|
|
15996
16870
|
...result
|
|
15997
16871
|
});
|
|
15998
16872
|
}
|
|
@@ -16004,7 +16878,7 @@ function createRigVerificationActivities(services) {
|
|
|
16004
16878
|
if (classified.action === "auto_promote") {
|
|
16005
16879
|
await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
16006
16880
|
status: "verifying",
|
|
16007
|
-
verification
|
|
16881
|
+
verification
|
|
16008
16882
|
});
|
|
16009
16883
|
const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
|
|
16010
16884
|
...change,
|
|
@@ -16020,7 +16894,7 @@ function createRigVerificationActivities(services) {
|
|
|
16020
16894
|
}
|
|
16021
16895
|
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
16022
16896
|
status: classified.status,
|
|
16023
|
-
verification
|
|
16897
|
+
verification
|
|
16024
16898
|
});
|
|
16025
16899
|
await recordRigAuditEvent(db, {
|
|
16026
16900
|
grant,
|
|
@@ -16042,12 +16916,10 @@ function createRigVerificationActivities(services) {
|
|
|
16042
16916
|
} catch (error) {
|
|
16043
16917
|
verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
16044
16918
|
verification.passed = false;
|
|
16045
|
-
verification.error =
|
|
16046
|
-
error instanceof Error ? error.message : String(error)
|
|
16047
|
-
);
|
|
16919
|
+
verification.error = error instanceof Error ? error.message : String(error);
|
|
16048
16920
|
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
16049
16921
|
status: "failed",
|
|
16050
|
-
verification
|
|
16922
|
+
verification
|
|
16051
16923
|
});
|
|
16052
16924
|
await recordRigAuditEvent(db, {
|
|
16053
16925
|
grant,
|
|
@@ -16110,7 +16982,7 @@ function createRigVerificationActivities(services) {
|
|
|
16110
16982
|
for (const check of version.checks) {
|
|
16111
16983
|
checkResults.push({
|
|
16112
16984
|
name: check.name,
|
|
16113
|
-
command:
|
|
16985
|
+
command: check.command,
|
|
16114
16986
|
...await runCommand(
|
|
16115
16987
|
established.session,
|
|
16116
16988
|
check.command,
|
|
@@ -16124,33 +16996,30 @@ function createRigVerificationActivities(services) {
|
|
|
16124
16996
|
grant,
|
|
16125
16997
|
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
16126
16998
|
rigId: rig.id,
|
|
16127
|
-
metadata:
|
|
16999
|
+
metadata: {
|
|
16128
17000
|
versionId: version.id,
|
|
16129
17001
|
startedAt,
|
|
16130
17002
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
16131
17003
|
passed,
|
|
16132
17004
|
checkResults
|
|
16133
|
-
}
|
|
17005
|
+
}
|
|
16134
17006
|
});
|
|
16135
17007
|
return { versionId: version.id, passed, checkResults };
|
|
16136
17008
|
}
|
|
16137
17009
|
);
|
|
16138
17010
|
} catch (error) {
|
|
16139
|
-
const detail =
|
|
16140
|
-
scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
|
|
16141
|
-
4096
|
|
16142
|
-
);
|
|
17011
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
16143
17012
|
await recordRigAuditEvent(db, {
|
|
16144
17013
|
grant,
|
|
16145
17014
|
action: "rig.verification.failed",
|
|
16146
17015
|
rigId: rig.id,
|
|
16147
|
-
metadata:
|
|
17016
|
+
metadata: {
|
|
16148
17017
|
versionId: version.id,
|
|
16149
17018
|
startedAt,
|
|
16150
17019
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
16151
17020
|
passed: false,
|
|
16152
17021
|
error: detail
|
|
16153
|
-
}
|
|
17022
|
+
}
|
|
16154
17023
|
});
|
|
16155
17024
|
throw error;
|
|
16156
17025
|
}
|
|
@@ -16158,6 +17027,144 @@ function createRigVerificationActivities(services) {
|
|
|
16158
17027
|
};
|
|
16159
17028
|
}
|
|
16160
17029
|
|
|
17030
|
+
// src/activities/retained-screenshot-reaper.ts
|
|
17031
|
+
import {
|
|
17032
|
+
claimRetainedScreenshotMaintenance,
|
|
17033
|
+
completeRetainedScreenshotMaintenance,
|
|
17034
|
+
promoteRetainedScreenshotMaintenanceCleanup
|
|
17035
|
+
} from "@opengeni/db";
|
|
17036
|
+
var RETAINED_SCREENSHOT_PENDING_GRACE_MS = 15 * 60 * 1e3;
|
|
17037
|
+
var RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
17038
|
+
var RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE = 100;
|
|
17039
|
+
function createRetainedScreenshotMaintenanceActivities(services, options = {}) {
|
|
17040
|
+
const pendingGraceMs = options.pendingGraceMs ?? RETAINED_SCREENSHOT_PENDING_GRACE_MS;
|
|
17041
|
+
const claimTimeoutMs = options.claimTimeoutMs ?? RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS;
|
|
17042
|
+
const batchSize = options.batchSize ?? RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE;
|
|
17043
|
+
const fileExists = options.fileExists ?? ((storage, file) => storage.fileExists(file));
|
|
17044
|
+
const headFile = options.headFile ?? ((storage, file) => storage.headFile(file));
|
|
17045
|
+
const deleteObject = options.deleteObject ?? ((storage, key) => storage.deleteObject(key));
|
|
17046
|
+
async function maintainRetainedScreenshots2() {
|
|
17047
|
+
const { db, objectStorage, observability } = await services();
|
|
17048
|
+
if (!objectStorage) {
|
|
17049
|
+
return { claimed: 0, ready: 0, deleted: 0, failed: 0, retryable: 0 };
|
|
17050
|
+
}
|
|
17051
|
+
const claims = await claimRetainedScreenshotMaintenance(db, {
|
|
17052
|
+
pendingGraceMs,
|
|
17053
|
+
claimTimeoutMs,
|
|
17054
|
+
limit: batchSize
|
|
17055
|
+
});
|
|
17056
|
+
const result = {
|
|
17057
|
+
claimed: claims.length,
|
|
17058
|
+
ready: 0,
|
|
17059
|
+
deleted: 0,
|
|
17060
|
+
failed: 0,
|
|
17061
|
+
retryable: 0
|
|
17062
|
+
};
|
|
17063
|
+
for (const claim of claims) {
|
|
17064
|
+
try {
|
|
17065
|
+
if (claim.action === "reconcile") {
|
|
17066
|
+
const outcome2 = await reconcileClaim(db, objectStorage, claim, {
|
|
17067
|
+
fileExists,
|
|
17068
|
+
headFile,
|
|
17069
|
+
deleteObject
|
|
17070
|
+
});
|
|
17071
|
+
if (outcome2 === "ready") {
|
|
17072
|
+
result.ready += 1;
|
|
17073
|
+
} else {
|
|
17074
|
+
result.failed += 1;
|
|
17075
|
+
}
|
|
17076
|
+
continue;
|
|
17077
|
+
}
|
|
17078
|
+
await deleteObject(objectStorage, claim.objectKey);
|
|
17079
|
+
const outcome = terminalDeleteOutcome(claim.cleanupReason);
|
|
17080
|
+
const settled = await completeRetainedScreenshotMaintenance(db, {
|
|
17081
|
+
accountId: claim.accountId,
|
|
17082
|
+
workspaceId: claim.workspaceId,
|
|
17083
|
+
artifactId: claim.artifactId,
|
|
17084
|
+
claimId: claim.claimId,
|
|
17085
|
+
outcome
|
|
17086
|
+
});
|
|
17087
|
+
if (!settled) throw new Error("retained screenshot cleanup claim was superseded");
|
|
17088
|
+
result.deleted += 1;
|
|
17089
|
+
} catch (error) {
|
|
17090
|
+
result.retryable += 1;
|
|
17091
|
+
observability.warn("retained screenshot maintenance failed; claim remains reclaimable", {
|
|
17092
|
+
workspaceId: claim.workspaceId,
|
|
17093
|
+
sessionId: claim.sessionId,
|
|
17094
|
+
artifactId: claim.artifactId,
|
|
17095
|
+
action: claim.action,
|
|
17096
|
+
error: error instanceof Error ? error.message : String(error)
|
|
17097
|
+
});
|
|
17098
|
+
}
|
|
17099
|
+
}
|
|
17100
|
+
if (claims.length > 0) {
|
|
17101
|
+
observability.info("retained screenshot maintenance swept", result);
|
|
17102
|
+
}
|
|
17103
|
+
return result;
|
|
17104
|
+
}
|
|
17105
|
+
return { maintainRetainedScreenshots: maintainRetainedScreenshots2 };
|
|
17106
|
+
}
|
|
17107
|
+
async function reconcileClaim(db, storage, claim, provider) {
|
|
17108
|
+
const observation = await observeClaimObject(storage, claim, provider);
|
|
17109
|
+
if (observation === "matches") {
|
|
17110
|
+
const settled2 = await completeRetainedScreenshotMaintenance(db, {
|
|
17111
|
+
accountId: claim.accountId,
|
|
17112
|
+
workspaceId: claim.workspaceId,
|
|
17113
|
+
artifactId: claim.artifactId,
|
|
17114
|
+
claimId: claim.claimId,
|
|
17115
|
+
outcome: "ready"
|
|
17116
|
+
});
|
|
17117
|
+
if (!settled2) throw new Error("retained screenshot reconcile claim was superseded");
|
|
17118
|
+
return "ready";
|
|
17119
|
+
}
|
|
17120
|
+
const promoted = await promoteRetainedScreenshotMaintenanceCleanup(db, {
|
|
17121
|
+
accountId: claim.accountId,
|
|
17122
|
+
workspaceId: claim.workspaceId,
|
|
17123
|
+
artifactId: claim.artifactId,
|
|
17124
|
+
claimId: claim.claimId,
|
|
17125
|
+
cleanupReason: "failed"
|
|
17126
|
+
});
|
|
17127
|
+
if (!promoted) throw new Error("retained screenshot reconcile claim was superseded");
|
|
17128
|
+
await provider.deleteObject(storage, claim.objectKey);
|
|
17129
|
+
const settled = await completeRetainedScreenshotMaintenance(db, {
|
|
17130
|
+
accountId: claim.accountId,
|
|
17131
|
+
workspaceId: claim.workspaceId,
|
|
17132
|
+
artifactId: claim.artifactId,
|
|
17133
|
+
claimId: claim.claimId,
|
|
17134
|
+
outcome: "failed"
|
|
17135
|
+
});
|
|
17136
|
+
if (!settled) throw new Error("retained screenshot reconcile claim was superseded");
|
|
17137
|
+
return "failed";
|
|
17138
|
+
}
|
|
17139
|
+
async function observeClaimObject(storage, claim, provider) {
|
|
17140
|
+
const file = claimFile(storage, claim);
|
|
17141
|
+
if (!await provider.fileExists(storage, file)) return "missing";
|
|
17142
|
+
const head = await provider.headFile(storage, file);
|
|
17143
|
+
return head.ContentLength === claim.sizeBytes && head.ContentType === claim.mediaType && head.Metadata?.sha256 === claim.sha256 ? "matches" : "mismatch";
|
|
17144
|
+
}
|
|
17145
|
+
function claimFile(storage, claim) {
|
|
17146
|
+
const timestamp = (/* @__PURE__ */ new Date(0)).toISOString();
|
|
17147
|
+
return {
|
|
17148
|
+
id: claim.artifactId,
|
|
17149
|
+
workspaceId: claim.workspaceId,
|
|
17150
|
+
status: "pending_upload",
|
|
17151
|
+
filename: `computer-screenshot-${claim.artifactId}.png`,
|
|
17152
|
+
safeFilename: `computer-screenshot-${claim.artifactId}.png`,
|
|
17153
|
+
contentType: claim.mediaType,
|
|
17154
|
+
sizeBytes: claim.sizeBytes,
|
|
17155
|
+
sha256: claim.sha256,
|
|
17156
|
+
bucket: storage.bucket,
|
|
17157
|
+
objectKey: claim.objectKey,
|
|
17158
|
+
createdAt: timestamp,
|
|
17159
|
+
updatedAt: timestamp
|
|
17160
|
+
};
|
|
17161
|
+
}
|
|
17162
|
+
function terminalDeleteOutcome(reason) {
|
|
17163
|
+
if (reason === "expired") return "expired";
|
|
17164
|
+
if (reason === "failed") return "failed";
|
|
17165
|
+
return "deleted";
|
|
17166
|
+
}
|
|
17167
|
+
|
|
16161
17168
|
// src/activities.ts
|
|
16162
17169
|
function createActivityServices(dependencies) {
|
|
16163
17170
|
let servicesPromise = null;
|
|
@@ -16228,6 +17235,7 @@ function controlActivities(services) {
|
|
|
16228
17235
|
...createCodexCapacityActivities(services),
|
|
16229
17236
|
...createRigVerificationActivities(services),
|
|
16230
17237
|
...createFileUploadReaperActivities(services),
|
|
17238
|
+
...createRetainedScreenshotMaintenanceActivities(services),
|
|
16231
17239
|
...createWorkflowWakeActivities(services),
|
|
16232
17240
|
// P1.3: the SOLE liveness/GC/cost-stop driver. Only reapSandboxLeases — no
|
|
16233
17241
|
// *ForViewer activities, no ownerHeartbeat, no resolveOwnerTaskQueue.
|
|
@@ -16238,7 +17246,9 @@ function createControlActivities(dependencies = {}) {
|
|
|
16238
17246
|
return controlActivities(createActivityServices(dependencies));
|
|
16239
17247
|
}
|
|
16240
17248
|
function createTurnActivities(dependencies = {}) {
|
|
16241
|
-
return {
|
|
17249
|
+
return {
|
|
17250
|
+
runAgentTurn: createRunAgentTurnActivity(createActivityServices(dependencies))
|
|
17251
|
+
};
|
|
16242
17252
|
}
|
|
16243
17253
|
var defaultControlActivities = createControlActivities();
|
|
16244
17254
|
var defaultTurnActivities = createTurnActivities();
|
|
@@ -16260,6 +17270,7 @@ var getCodexCapacityWait = defaultControlActivities.getCodexCapacityWait;
|
|
|
16260
17270
|
var reconcileCodexCapacityWait = defaultControlActivities.reconcileCodexCapacityWait;
|
|
16261
17271
|
var reapSandboxLeases = defaultControlActivities.reapSandboxLeases;
|
|
16262
17272
|
var reapExpiredFileUploads = defaultControlActivities.reapExpiredFileUploads;
|
|
17273
|
+
var maintainRetainedScreenshots = defaultControlActivities.maintainRetainedScreenshots;
|
|
16263
17274
|
var dispatchSessionWorkflowWakes = defaultControlActivities.dispatchSessionWorkflowWakes;
|
|
16264
17275
|
var verifyRigChange = defaultControlActivities.verifyRigChange;
|
|
16265
17276
|
var verifyRigVersion = defaultControlActivities.verifyRigVersion;
|
|
@@ -16341,7 +17352,7 @@ function createWorkerHttpHandler(input) {
|
|
|
16341
17352
|
checks: Object.fromEntries(
|
|
16342
17353
|
Object.keys(checks).map((name) => [
|
|
16343
17354
|
name,
|
|
16344
|
-
{ ok: false, error:
|
|
17355
|
+
{ ok: false, error: "worker_not_ready" }
|
|
16345
17356
|
])
|
|
16346
17357
|
)
|
|
16347
17358
|
} : { ...await runReadinessChecks(checks, timeoutMs), ...state ? { state } : {} };
|
|
@@ -16364,11 +17375,8 @@ async function runReadinessChecks(checks, timeoutMs = 2e3) {
|
|
|
16364
17375
|
try {
|
|
16365
17376
|
await withTimeout(Promise.resolve().then(check), timeoutMs);
|
|
16366
17377
|
return [name, { ok: true }];
|
|
16367
|
-
} catch
|
|
16368
|
-
return [
|
|
16369
|
-
name,
|
|
16370
|
-
{ ok: false, error: error instanceof Error ? error.message : String(error) }
|
|
16371
|
-
];
|
|
17378
|
+
} catch {
|
|
17379
|
+
return [name, { ok: false, error: "dependency_unavailable" }];
|
|
16372
17380
|
}
|
|
16373
17381
|
}
|
|
16374
17382
|
)
|
|
@@ -16510,20 +17518,24 @@ function createWorkerServiceLifecycle(input) {
|
|
|
16510
17518
|
})();
|
|
16511
17519
|
return runPromise;
|
|
16512
17520
|
},
|
|
16513
|
-
drain: (
|
|
17521
|
+
drain: (_reason = "host request") => {
|
|
16514
17522
|
if (state === "draining" || state === "stopped" || state === "failed") {
|
|
16515
17523
|
return;
|
|
16516
17524
|
}
|
|
16517
17525
|
state = "draining";
|
|
16518
17526
|
input.observability.info("OpenGeni worker draining (graceful shutdown)", {
|
|
16519
17527
|
role: input.role,
|
|
16520
|
-
|
|
17528
|
+
errorClass: "WorkerLifecycleOperation",
|
|
17529
|
+
errorCode: "worker_draining",
|
|
17530
|
+
origin: "worker-lifecycle"
|
|
16521
17531
|
});
|
|
16522
17532
|
try {
|
|
16523
17533
|
input.worker.shutdown();
|
|
16524
|
-
} catch
|
|
17534
|
+
} catch {
|
|
16525
17535
|
input.observability.warn("worker shutdown request failed", {
|
|
16526
|
-
|
|
17536
|
+
errorClass: "WorkerLifecycleOperationError",
|
|
17537
|
+
errorCode: "worker_shutdown_request_failed",
|
|
17538
|
+
origin: "worker-lifecycle"
|
|
16527
17539
|
});
|
|
16528
17540
|
}
|
|
16529
17541
|
},
|