@opengeni/worker-bundle 0.16.21 → 0.16.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +61 -14
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/environment.d.ts +1 -0
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +93 -0
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +2 -0
- package/dist/activities/toolspace-token-renewal.d.ts +1 -1
- package/dist/activities/workspace-capture.d.ts +1 -0
- package/dist/activities.d.ts +3 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/host-export.js +1 -1
- package/dist/index.js +1317 -390
- package/dist/index.js.map +1 -1
- package/dist/sandbox-resume.d.ts +6 -0
- package/dist/sandbox-routing.d.ts +11 -6
- package/dist/workflow-bundle.js +2 -1
- package/package.json +12 -12
- package/src/activities/agent-turn.ts +482 -360
- package/src/activities/codex-capacity.ts +10 -1
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +9 -3
- package/src/activities/environment.ts +13 -24
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +892 -0
- package/src/activities/rig-verification.ts +13 -34
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +18 -4
- package/src/activities/scheduled-tasks.ts +19 -3
- package/src/activities/toolspace-token-renewal.ts +9 -2
- package/src/activities/workspace-capture.ts +33 -3
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/observability-metrics.ts +3 -3
- package/src/sandbox-resume.ts +45 -12
- package/src/sandbox-routing.ts +30 -15
- package/src/worker-service-lifecycle.ts +8 -4
- package/src/workflows/file-upload-reaper.ts +4 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createHostExportPump
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-RAMNYSAW.js";
|
|
4
4
|
|
|
5
5
|
// src/index.ts
|
|
6
6
|
import {
|
|
@@ -72,6 +72,7 @@ import {
|
|
|
72
72
|
getCodexRotationSettings,
|
|
73
73
|
getWorkspaceModelPolicy as getWorkspaceModelPolicy2,
|
|
74
74
|
getWorkspaceGrant,
|
|
75
|
+
getWorkspace as getWorkspace2,
|
|
75
76
|
listCodexAccountStatuses as listCodexAccountStatuses2,
|
|
76
77
|
fetchCodexUsageForAccount as fetchCodexUsageForAccount2,
|
|
77
78
|
getSessionCodexState,
|
|
@@ -110,7 +111,7 @@ import {
|
|
|
110
111
|
SandboxLeaseSupersededError as SandboxLeaseSupersededError2,
|
|
111
112
|
SandboxImageConflictError,
|
|
112
113
|
isSessionEventPersistenceError,
|
|
113
|
-
getEnrollment,
|
|
114
|
+
getEnrollment as getEnrollment2,
|
|
114
115
|
abandonRecordingForTurnAttempt,
|
|
115
116
|
markSessionAttemptQuiesced,
|
|
116
117
|
getOrCreatePreferenceRegistrySnapshot,
|
|
@@ -130,6 +131,7 @@ import {
|
|
|
130
131
|
modelTerminalResponseFromSdkEvent,
|
|
131
132
|
normalizeModelCallUsage,
|
|
132
133
|
normalizeSdkEvent,
|
|
134
|
+
normalizeProtocolJsonValue,
|
|
133
135
|
projectHistoryForProvider as projectHistoryForProvider2,
|
|
134
136
|
sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2,
|
|
135
137
|
projectModelInputForCapabilities,
|
|
@@ -591,10 +593,21 @@ function selectCodexCredentialLeaseForTurn(args) {
|
|
|
591
593
|
};
|
|
592
594
|
}
|
|
593
595
|
if (!rotationEnabled) {
|
|
594
|
-
const
|
|
596
|
+
const active = activeCredentialId ? accounts.find((account) => account.id === activeCredentialId) : void 0;
|
|
597
|
+
if (!active?.allocatorEnabled) {
|
|
598
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
599
|
+
}
|
|
600
|
+
if (!isCodexCredentialHealthy(active, args.now)) {
|
|
601
|
+
return {
|
|
602
|
+
credentialId: null,
|
|
603
|
+
decision: { kind: "allCapped", earliestResetAt: availableAt(active, args.now) },
|
|
604
|
+
advanceActivePointer: false
|
|
605
|
+
};
|
|
606
|
+
}
|
|
595
607
|
return {
|
|
596
|
-
credentialId,
|
|
597
|
-
decision:
|
|
608
|
+
credentialId: active.id,
|
|
609
|
+
decision: { kind: "active", credentialId: active.id, moved: false },
|
|
610
|
+
advanceActivePointer: false
|
|
598
611
|
};
|
|
599
612
|
}
|
|
600
613
|
if (!leasingEnabled && pinDisposition !== "clearStale" && args.sessionPinnedCredentialId) {
|
|
@@ -872,7 +885,9 @@ function codexCapacityDecision(context, settings) {
|
|
|
872
885
|
sessionLastCredentialId: context.sessionLastCredentialId,
|
|
873
886
|
now
|
|
874
887
|
});
|
|
875
|
-
|
|
888
|
+
const selectedAccount = selected.credentialId ? context.accounts.find((account) => account.id === selected.credentialId) : void 0;
|
|
889
|
+
const selectedIsAvailable = selectedAccount !== void 0 && (selectedAccount.id === context.existingCredentialId ? isCodexCredentialHealthy(selectedAccount, now) : isCodexCredentialEligible(selectedAccount, now));
|
|
890
|
+
if (selected.credentialId && selectedIsAvailable) {
|
|
876
891
|
return {
|
|
877
892
|
kind: "available",
|
|
878
893
|
credentialId: selected.credentialId,
|
|
@@ -1327,6 +1342,7 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
1327
1342
|
joinInternalContext(internalContext, attachmentContext),
|
|
1328
1343
|
fileAttachments.map((attachment) => attachment.resource),
|
|
1329
1344
|
options.providerApi,
|
|
1345
|
+
options.materializeModelHistory,
|
|
1330
1346
|
options.projectModelHistory
|
|
1331
1347
|
);
|
|
1332
1348
|
}
|
|
@@ -1343,6 +1359,7 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
1343
1359
|
internalContext,
|
|
1344
1360
|
[],
|
|
1345
1361
|
options.providerApi,
|
|
1362
|
+
options.materializeModelHistory,
|
|
1346
1363
|
options.projectModelHistory
|
|
1347
1364
|
);
|
|
1348
1365
|
}
|
|
@@ -1352,9 +1369,10 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
1352
1369
|
if (!state) {
|
|
1353
1370
|
throw new Error("No saved run state is available for approval decision");
|
|
1354
1371
|
}
|
|
1372
|
+
const serializedRunState = resumeRunState(state);
|
|
1355
1373
|
const prepared = await runtime.prepareInput(agent, {
|
|
1356
1374
|
kind: "approval",
|
|
1357
|
-
serializedRunState:
|
|
1375
|
+
serializedRunState: options.materializeSerializedRunState ? await options.materializeSerializedRunState(serializedRunState) : serializedRunState,
|
|
1358
1376
|
approvalId: String(payload.approvalId ?? ""),
|
|
1359
1377
|
decision: payload.decision === "approve" ? "approve" : "reject",
|
|
1360
1378
|
...typeof payload.message === "string" ? { message: payload.message } : {}
|
|
@@ -1380,9 +1398,10 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
1380
1398
|
if (!resume) {
|
|
1381
1399
|
throw new Error("Human-input response does not resolve to a durable request");
|
|
1382
1400
|
}
|
|
1401
|
+
const serializedRunState = resumeRunState(state);
|
|
1383
1402
|
const prepared = await runtime.prepareInput(agent, {
|
|
1384
1403
|
kind: "human_input",
|
|
1385
|
-
serializedRunState:
|
|
1404
|
+
serializedRunState: options.materializeSerializedRunState ? await options.materializeSerializedRunState(serializedRunState) : serializedRunState,
|
|
1386
1405
|
toolCallId: resume.toolCallId
|
|
1387
1406
|
});
|
|
1388
1407
|
return {
|
|
@@ -1401,13 +1420,14 @@ function joinInternalContext(...parts) {
|
|
|
1401
1420
|
const content = parts.map((part) => part?.trim()).filter((part) => Boolean(part));
|
|
1402
1421
|
return content.length > 0 ? content.join("\n\n") : void 0;
|
|
1403
1422
|
}
|
|
1404
|
-
async function messageInput(db, runtime, agent, trigger, text, internalContext, currentAttachmentRefs = [], providerApi = "responses", projectModelHistory) {
|
|
1423
|
+
async function messageInput(db, runtime, agent, trigger, text, internalContext, currentAttachmentRefs = [], providerApi = "responses", materializeModelHistory, projectModelHistory) {
|
|
1405
1424
|
const stored = await getActiveSessionHistoryItems(db, trigger.workspaceId, trigger.sessionId);
|
|
1406
1425
|
const envelope = await getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId);
|
|
1407
1426
|
const canonicalView = projectRejectedProviderArtifacts(stored);
|
|
1408
1427
|
const providerView = projectHistoryForProvider(canonicalView, providerApi);
|
|
1409
1428
|
const referencedHistory = withCurrentUserAttachmentRefs(providerView, currentAttachmentRefs);
|
|
1410
|
-
const
|
|
1429
|
+
const materializedHistory = materializeModelHistory ? await materializeModelHistory(referencedHistory) : referencedHistory;
|
|
1430
|
+
const historyItems = projectModelHistory ? await projectModelHistory(materializedHistory) : materializedHistory;
|
|
1411
1431
|
const prepared = await runtime.prepareInput(agent, {
|
|
1412
1432
|
kind: "message",
|
|
1413
1433
|
...text ? { text } : {},
|
|
@@ -1491,10 +1511,13 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
|
|
|
1491
1511
|
};
|
|
1492
1512
|
}
|
|
1493
1513
|
const canonicalItems = projectRejectedProviderArtifacts(active);
|
|
1494
|
-
const projectForWire = async (input) =>
|
|
1495
|
-
options.
|
|
1496
|
-
|
|
1497
|
-
|
|
1514
|
+
const projectForWire = async (input) => {
|
|
1515
|
+
const materialized = options.materializeHistory ? await options.materializeHistory(input) : input;
|
|
1516
|
+
return sanitizeHistoryItemsForModel(
|
|
1517
|
+
options.projectModelInput ? await options.projectModelInput(materialized) : materialized,
|
|
1518
|
+
settings.modelToolOutputTruncationTokens
|
|
1519
|
+
);
|
|
1520
|
+
};
|
|
1498
1521
|
const items = await projectForWire(canonicalItems);
|
|
1499
1522
|
const decision = decideCompaction({
|
|
1500
1523
|
items,
|
|
@@ -1744,6 +1767,7 @@ import {
|
|
|
1744
1767
|
import {
|
|
1745
1768
|
gitCredentialBindingIdForRepository,
|
|
1746
1769
|
gitCredentialProviderForRepository,
|
|
1770
|
+
gitRemoteIdentity,
|
|
1747
1771
|
signDelegatedAccessToken
|
|
1748
1772
|
} from "@opengeni/contracts";
|
|
1749
1773
|
import {
|
|
@@ -1809,8 +1833,13 @@ function assertWorkspaceEcho(kind, scope, echoedWorkspaceId) {
|
|
|
1809
1833
|
async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironment = {}, options = {}) {
|
|
1810
1834
|
const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
|
|
1811
1835
|
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
|
|
1836
|
+
if (options.skipToolspace) {
|
|
1837
|
+
delete environment.OPENGENI_TOOLSPACE_URL;
|
|
1838
|
+
delete environment.OPENGENI_TOOLSPACE_TOKEN_FILE;
|
|
1839
|
+
delete environment.OPENGENI_OGTOOL_PACKAGE_SPEC;
|
|
1840
|
+
}
|
|
1812
1841
|
const toolspaceScope = options.scope;
|
|
1813
|
-
const toolspaceToken = toolspaceScope && options.toolspaceAuthority ? await mintSandboxToolspaceToken(settings, toolspaceScope, options.toolspaceAuthority) : void 0;
|
|
1842
|
+
const toolspaceToken = !options.skipToolspace && toolspaceScope && options.toolspaceAuthority ? await mintSandboxToolspaceToken(settings, toolspaceScope, options.toolspaceAuthority) : void 0;
|
|
1814
1843
|
if (toolspaceToken && toolspaceScope) {
|
|
1815
1844
|
environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(
|
|
1816
1845
|
settings,
|
|
@@ -2136,7 +2165,7 @@ function gitCredentialSelections(resources) {
|
|
|
2136
2165
|
);
|
|
2137
2166
|
}
|
|
2138
2167
|
bindingProviders.set(credentialBindingId, provider);
|
|
2139
|
-
const normalizedRemote =
|
|
2168
|
+
const normalizedRemote = gitRemoteIdentity(resource.uri, provider);
|
|
2140
2169
|
const claimedBinding = remoteBindings.get(normalizedRemote);
|
|
2141
2170
|
if (claimedBinding && claimedBinding !== bindingKey) {
|
|
2142
2171
|
throw new Error(
|
|
@@ -2192,14 +2221,6 @@ function normalizedGitHost(uri) {
|
|
|
2192
2221
|
return "";
|
|
2193
2222
|
}
|
|
2194
2223
|
}
|
|
2195
|
-
function normalizedGitRemote(uri) {
|
|
2196
|
-
try {
|
|
2197
|
-
const url = new URL(uri);
|
|
2198
|
-
return `${url.protocol.toLowerCase()}//${url.host.toLowerCase()}${url.pathname.replace(/\/+$/, "").replace(/\.git$/, "")}`;
|
|
2199
|
-
} catch {
|
|
2200
|
-
return uri.trim().replace(/\/+$/, "").replace(/\.git$/, "").toLowerCase();
|
|
2201
|
-
}
|
|
2202
|
-
}
|
|
2203
2224
|
function repositoryCredentialProvider(resource) {
|
|
2204
2225
|
return gitCredentialProviderForRepository(resource);
|
|
2205
2226
|
}
|
|
@@ -2302,7 +2323,7 @@ function startGitCredentialRenewalLoop(options) {
|
|
|
2302
2323
|
} catch {
|
|
2303
2324
|
}
|
|
2304
2325
|
scheduleRefresh(nextDelayMs);
|
|
2305
|
-
} catch
|
|
2326
|
+
} catch {
|
|
2306
2327
|
if (stopped) return;
|
|
2307
2328
|
const scheduledRetryMs = retryDelayMs;
|
|
2308
2329
|
retryDelayMs = Math.min(retryDelayMs * 2, GIT_CREDENTIAL_MAX_RETRY_MS);
|
|
@@ -2310,7 +2331,7 @@ function startGitCredentialRenewalLoop(options) {
|
|
|
2310
2331
|
options.onFailure?.({
|
|
2311
2332
|
providers,
|
|
2312
2333
|
retryDelayMs: scheduledRetryMs,
|
|
2313
|
-
errorClass:
|
|
2334
|
+
errorClass: "GitCredentialRenewalOperationError"
|
|
2314
2335
|
});
|
|
2315
2336
|
} catch {
|
|
2316
2337
|
}
|
|
@@ -2401,14 +2422,14 @@ function startExpiringMaterialRenewalLoop(options) {
|
|
|
2401
2422
|
} catch {
|
|
2402
2423
|
}
|
|
2403
2424
|
scheduleRefresh(nextDelayMs);
|
|
2404
|
-
} catch
|
|
2425
|
+
} catch {
|
|
2405
2426
|
if (stopped) return;
|
|
2406
2427
|
const scheduledRetryMs = retryDelayMs;
|
|
2407
2428
|
retryDelayMs = Math.min(retryDelayMs * 2, options.policy.maxRetryMs);
|
|
2408
2429
|
try {
|
|
2409
2430
|
options.onFailure?.({
|
|
2410
2431
|
retryDelayMs: scheduledRetryMs,
|
|
2411
|
-
errorClass:
|
|
2432
|
+
errorClass: options.publicErrorClass
|
|
2412
2433
|
});
|
|
2413
2434
|
} catch {
|
|
2414
2435
|
}
|
|
@@ -2450,6 +2471,7 @@ function startRunCredentialRenewalLoop(options) {
|
|
|
2450
2471
|
resolve: options.resolve,
|
|
2451
2472
|
write: options.write,
|
|
2452
2473
|
expiresAt: (material) => material?.expiresAt ?? null,
|
|
2474
|
+
publicErrorClass: "RunCredentialRenewalOperationError",
|
|
2453
2475
|
policy: {
|
|
2454
2476
|
defaultRefreshMs: RUN_CREDENTIAL_DEFAULT_REFRESH_MS,
|
|
2455
2477
|
expiryLeadMs: RUN_CREDENTIAL_EXPIRY_LEAD_MS,
|
|
@@ -2489,6 +2511,7 @@ function startToolspaceTokenRenewalLoop(options) {
|
|
|
2489
2511
|
},
|
|
2490
2512
|
write: options.write,
|
|
2491
2513
|
expiresAt: (material) => material.expiresAt,
|
|
2514
|
+
publicErrorClass: "ToolspaceTokenRenewalOperationError",
|
|
2492
2515
|
policy: {
|
|
2493
2516
|
defaultRefreshMs: TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS,
|
|
2494
2517
|
expiryLeadMs: TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
|
|
@@ -2891,10 +2914,6 @@ function workspacePackRuntimeFromPacks(packs) {
|
|
|
2891
2914
|
skills
|
|
2892
2915
|
};
|
|
2893
2916
|
}
|
|
2894
|
-
async function resolveWorkspaceAgentInstructions(db, workspaceId) {
|
|
2895
|
-
const workspace = await getWorkspace(db, workspaceId);
|
|
2896
|
-
return workspace?.agentInstructions ?? null;
|
|
2897
|
-
}
|
|
2898
2917
|
function settingsWithPackSandboxImage(settings, sandboxImage, sandboxProviderImages = null) {
|
|
2899
2918
|
if (!sandboxImage) {
|
|
2900
2919
|
return settings;
|
|
@@ -3149,16 +3168,6 @@ function childCompletionSummary(child, goal, terminalStatus) {
|
|
|
3149
3168
|
return lines.join("\n");
|
|
3150
3169
|
}
|
|
3151
3170
|
|
|
3152
|
-
// src/activities/redaction.ts
|
|
3153
|
-
import {
|
|
3154
|
-
createSecretRedactor,
|
|
3155
|
-
identityRedactor,
|
|
3156
|
-
isCredentialHeaderName,
|
|
3157
|
-
redactSensitiveData,
|
|
3158
|
-
redactSensitiveText,
|
|
3159
|
-
redactSerializedJson
|
|
3160
|
-
} from "@opengeni/contracts";
|
|
3161
|
-
|
|
3162
3171
|
// src/activities/streaming.ts
|
|
3163
3172
|
import { Context } from "@temporalio/activity";
|
|
3164
3173
|
var TRAILING_FLUSH_MS = 33;
|
|
@@ -3309,7 +3318,6 @@ var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery
|
|
|
3309
3318
|
|
|
3310
3319
|
// src/sandbox-resume.ts
|
|
3311
3320
|
import { sandboxArchiveCaptureTimeoutMs } from "@opengeni/config";
|
|
3312
|
-
import { redactSensitiveText as redactSensitiveText2 } from "@opengeni/contracts";
|
|
3313
3321
|
import { randomUUID } from "crypto";
|
|
3314
3322
|
import {
|
|
3315
3323
|
acquireLease,
|
|
@@ -3397,14 +3405,18 @@ var SnapshotTimeoutError = class extends Error {
|
|
|
3397
3405
|
}
|
|
3398
3406
|
};
|
|
3399
3407
|
function safeSnapshotError(error) {
|
|
3400
|
-
const
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
|
|
3404
|
-
0,
|
|
3405
|
-
2048
|
|
3406
|
-
)
|
|
3408
|
+
const fields = {
|
|
3409
|
+
errorClass: "SnapshotOperationError",
|
|
3410
|
+
errorCode: "snapshot_operation_failed",
|
|
3411
|
+
origin: "sandbox-resume"
|
|
3407
3412
|
};
|
|
3413
|
+
try {
|
|
3414
|
+
const rawStatus = error && typeof error === "object" ? error.status ?? error.statusCode : void 0;
|
|
3415
|
+
const status = Number(rawStatus);
|
|
3416
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
|
|
3417
|
+
} catch {
|
|
3418
|
+
}
|
|
3419
|
+
return fields;
|
|
3408
3420
|
}
|
|
3409
3421
|
async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
|
|
3410
3422
|
if (signal?.aborted) return false;
|
|
@@ -3571,6 +3583,10 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
|
|
|
3571
3583
|
if (typeof persistable.persistWorkspace !== "function") {
|
|
3572
3584
|
return false;
|
|
3573
3585
|
}
|
|
3586
|
+
const workspacePersistence = persistable.state?.workspacePersistence ?? persistable.state?.providerState?.workspacePersistence;
|
|
3587
|
+
if (!force && workspacePersistence === "snapshot_filesystem") {
|
|
3588
|
+
return false;
|
|
3589
|
+
}
|
|
3574
3590
|
if (signal?.aborted) {
|
|
3575
3591
|
return false;
|
|
3576
3592
|
}
|
|
@@ -3580,7 +3596,7 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
|
|
|
3580
3596
|
return false;
|
|
3581
3597
|
}
|
|
3582
3598
|
const instanceId = lease.instanceId;
|
|
3583
|
-
const nativeModalPersistence =
|
|
3599
|
+
const nativeModalPersistence = workspacePersistence === "snapshot_filesystem" || workspacePersistence === "snapshot_directory";
|
|
3584
3600
|
const checkpointBinding = nativeModalPersistence ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal) : null;
|
|
3585
3601
|
if (signal?.aborted) return false;
|
|
3586
3602
|
const captureId = randomUUID();
|
|
@@ -4198,6 +4214,7 @@ import {
|
|
|
4198
4214
|
verifyRetainedProcessMutationSettlement,
|
|
4199
4215
|
verifyWorkspaceMutationSettlement,
|
|
4200
4216
|
getSandbox,
|
|
4217
|
+
getEnrollment,
|
|
4201
4218
|
markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
|
|
4202
4219
|
readLease as readLease2,
|
|
4203
4220
|
readActiveSandbox,
|
|
@@ -4541,6 +4558,11 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
4541
4558
|
} : null;
|
|
4542
4559
|
},
|
|
4543
4560
|
controlRpcFactory: controlRpcFactory(bus),
|
|
4561
|
+
resolveSelfhostedOpStream: async (sandbox) => {
|
|
4562
|
+
if (!sandbox.enrollmentId) return void 0;
|
|
4563
|
+
const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
|
|
4564
|
+
return opStreamDepsFor(services, enrollment?.opStream === true);
|
|
4565
|
+
},
|
|
4544
4566
|
relay: relayConfigFromSettings(settings),
|
|
4545
4567
|
// A selfhosted swap target runs real commands too, so give it the same split
|
|
4546
4568
|
// deadlines the machine-primary establish path uses (short control, long exec).
|
|
@@ -4676,6 +4698,11 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
4676
4698
|
} : null;
|
|
4677
4699
|
},
|
|
4678
4700
|
controlRpcFactory: controlRpcFactory(bus),
|
|
4701
|
+
resolveSelfhostedOpStream: async (sandbox) => {
|
|
4702
|
+
if (!sandbox.enrollmentId) return void 0;
|
|
4703
|
+
const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
|
|
4704
|
+
return opStreamDepsFor(services, enrollment?.opStream === true);
|
|
4705
|
+
},
|
|
4679
4706
|
relay: relayConfigFromSettings(settings),
|
|
4680
4707
|
...selfhostedResolverTimeouts(settings),
|
|
4681
4708
|
...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
|
|
@@ -4780,7 +4807,7 @@ function opStreamDepsFor(services, machineAdvertisesOpStream) {
|
|
|
4780
4807
|
async function establishSelfhostedTurnSession(services, args) {
|
|
4781
4808
|
const { settings, bus, onOp } = services;
|
|
4782
4809
|
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
4783
|
-
const opStream = opStreamDepsFor(services,
|
|
4810
|
+
const opStream = opStreamDepsFor(services, args.opStream);
|
|
4784
4811
|
const { client, session } = await buildSelfhostedBackendSession({
|
|
4785
4812
|
workspaceId: args.workspaceId,
|
|
4786
4813
|
agentId: args.agentId,
|
|
@@ -4796,8 +4823,8 @@ async function establishSelfhostedTurnSession(services, args) {
|
|
|
4796
4823
|
// Meter every control op (out-of-band telemetry) — no-op when unwired.
|
|
4797
4824
|
...onOp !== void 0 ? { onOp } : {},
|
|
4798
4825
|
// The streaming exec transport — present iff the machine advertised the
|
|
4799
|
-
// capability AND the server flag is on
|
|
4800
|
-
// legacy
|
|
4826
|
+
// capability AND the server flag is on. It is required when execTimeoutMs=0;
|
|
4827
|
+
// legacy request/reply remains available only for explicitly bounded exec.
|
|
4801
4828
|
...opStream !== void 0 ? { opStream } : {}
|
|
4802
4829
|
});
|
|
4803
4830
|
return {
|
|
@@ -4848,11 +4875,11 @@ function eventAttributes(attributes) {
|
|
|
4848
4875
|
if (!attributes) {
|
|
4849
4876
|
return void 0;
|
|
4850
4877
|
}
|
|
4851
|
-
const
|
|
4878
|
+
const projected = {};
|
|
4852
4879
|
for (const [key, value] of Object.entries(attributes)) {
|
|
4853
|
-
|
|
4880
|
+
projected[key] = eventAttributeValue(value);
|
|
4854
4881
|
}
|
|
4855
|
-
return
|
|
4882
|
+
return projected;
|
|
4856
4883
|
}
|
|
4857
4884
|
function eventAttributeValue(value) {
|
|
4858
4885
|
if (value === null || value === void 0 || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
@@ -5345,6 +5372,22 @@ function recordBatchFlush(observability, input) {
|
|
|
5345
5372
|
value: input.durationSeconds
|
|
5346
5373
|
});
|
|
5347
5374
|
}
|
|
5375
|
+
function recordSessionEventAppendLatency(observability, input) {
|
|
5376
|
+
observability.observeHistogram({
|
|
5377
|
+
name: "opengeni_session_event_append_seconds",
|
|
5378
|
+
help: "Duration in seconds of an appendSessionEvents DB write (the durable write path).",
|
|
5379
|
+
buckets: STREAM_IO_BUCKETS,
|
|
5380
|
+
value: input.durationSeconds
|
|
5381
|
+
});
|
|
5382
|
+
}
|
|
5383
|
+
function recordSessionEventPublishLatency(observability, input) {
|
|
5384
|
+
observability.observeHistogram({
|
|
5385
|
+
name: "opengeni_session_event_publish_seconds",
|
|
5386
|
+
help: "Duration in seconds of the best-effort NATS live fan-out publish (the delivery path).",
|
|
5387
|
+
buckets: STREAM_IO_BUCKETS,
|
|
5388
|
+
value: input.durationSeconds
|
|
5389
|
+
});
|
|
5390
|
+
}
|
|
5348
5391
|
var MODEL_INPUT_TOKENS_BUCKETS = [
|
|
5349
5392
|
1e3,
|
|
5350
5393
|
5e3,
|
|
@@ -5630,12 +5673,667 @@ async function discardUnpublishedRecording(args) {
|
|
|
5630
5673
|
}
|
|
5631
5674
|
function scrubFreeText(value) {
|
|
5632
5675
|
if (value === null || value === void 0) return null;
|
|
5633
|
-
const
|
|
5634
|
-
return
|
|
5676
|
+
const exact = String(value);
|
|
5677
|
+
return exact.length === 0 ? null : exact;
|
|
5635
5678
|
}
|
|
5636
5679
|
|
|
5637
|
-
// src/activities/
|
|
5680
|
+
// src/activities/retained-screenshots.ts
|
|
5681
|
+
import {
|
|
5682
|
+
COMPUTER_SCREENSHOT_MAX_BYTES,
|
|
5683
|
+
COMPUTER_SCREENSHOT_MAX_DIMENSION,
|
|
5684
|
+
COMPUTER_SCREENSHOT_MAX_PIXELS,
|
|
5685
|
+
COMPUTER_SCREENSHOT_RETENTION_MS,
|
|
5686
|
+
COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES,
|
|
5687
|
+
RetainedArtifactMetadataSchema,
|
|
5688
|
+
retainedScreenshotReferenceFromFile
|
|
5689
|
+
} from "@opengeni/contracts";
|
|
5690
|
+
import {
|
|
5691
|
+
RetainedScreenshotQuotaExceededError,
|
|
5692
|
+
getRetainedScreenshotArtifact,
|
|
5693
|
+
prepareRetainedScreenshotArtifact,
|
|
5694
|
+
recordRetainedScreenshotArtifactError,
|
|
5695
|
+
settleRetainedScreenshotArtifactReady
|
|
5696
|
+
} from "@opengeni/db";
|
|
5638
5697
|
import { createHash as createHash3 } from "crypto";
|
|
5698
|
+
var PNG_SIGNATURE = Uint8Array.of(137, 80, 78, 71, 13, 10, 26, 10);
|
|
5699
|
+
var PNG_ALLOWED_CRITICAL_CHUNKS = /* @__PURE__ */ new Set(["IHDR", "PLTE", "IDAT", "IEND"]);
|
|
5700
|
+
var PNG_MAX_CHUNKS = 16384;
|
|
5701
|
+
var RETAINED_IMAGE_MARKER = "retained_artifact";
|
|
5702
|
+
var ScreenshotValidationError = class extends Error {
|
|
5703
|
+
constructor(reason, message) {
|
|
5704
|
+
super(message);
|
|
5705
|
+
this.reason = reason;
|
|
5706
|
+
this.name = "ScreenshotValidationError";
|
|
5707
|
+
}
|
|
5708
|
+
};
|
|
5709
|
+
var RetainedScreenshotUnavailableError = class extends Error {
|
|
5710
|
+
constructor(artifactId, reason) {
|
|
5711
|
+
super(`Retained screenshot ${artifactId} is unavailable: ${reason}`);
|
|
5712
|
+
this.artifactId = artifactId;
|
|
5713
|
+
this.reason = reason;
|
|
5714
|
+
this.name = "RetainedScreenshotUnavailableError";
|
|
5715
|
+
}
|
|
5716
|
+
};
|
|
5717
|
+
function typedScreenshotFromSdkEvent(event) {
|
|
5718
|
+
if (!event || typeof event !== "object") return null;
|
|
5719
|
+
const streamEvent = event;
|
|
5720
|
+
if (streamEvent.type !== "run_item_stream_event") return null;
|
|
5721
|
+
const item = streamEvent.item;
|
|
5722
|
+
if (!item || item.type !== "tool_call_output_item") return null;
|
|
5723
|
+
const output = item.output;
|
|
5724
|
+
if (!output || typeof output !== "object" || Array.isArray(output)) return null;
|
|
5725
|
+
const outputRecord = output;
|
|
5726
|
+
if (outputRecord.type !== "image" || !outputRecord.image || typeof outputRecord.image !== "object") {
|
|
5727
|
+
return null;
|
|
5728
|
+
}
|
|
5729
|
+
const image = outputRecord.image;
|
|
5730
|
+
if (!(image.data instanceof Uint8Array) || typeof image.mediaType !== "string") return null;
|
|
5731
|
+
const raw = item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem) ? item.rawItem : {};
|
|
5732
|
+
const callId = raw.callId ?? raw.call_id;
|
|
5733
|
+
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
5734
|
+
const toolOutputId = raw.id ?? item.id ?? callId;
|
|
5735
|
+
if (typeof toolOutputId !== "string" || toolOutputId.length === 0) return null;
|
|
5736
|
+
return {
|
|
5737
|
+
callId,
|
|
5738
|
+
toolOutputId,
|
|
5739
|
+
bytes: image.data,
|
|
5740
|
+
mediaType: image.mediaType
|
|
5741
|
+
};
|
|
5742
|
+
}
|
|
5743
|
+
function validateComputerScreenshot(input, limits = {}) {
|
|
5744
|
+
const maxBytes = limits.maxBytes ?? COMPUTER_SCREENSHOT_MAX_BYTES;
|
|
5745
|
+
const maxDimension = limits.maxDimension ?? COMPUTER_SCREENSHOT_MAX_DIMENSION;
|
|
5746
|
+
const maxPixels = limits.maxPixels ?? COMPUTER_SCREENSHOT_MAX_PIXELS;
|
|
5747
|
+
if (input.mediaType !== "image/png") {
|
|
5748
|
+
throw new ScreenshotValidationError(
|
|
5749
|
+
"unsupported",
|
|
5750
|
+
"computer screenshot MIME must be image/png"
|
|
5751
|
+
);
|
|
5752
|
+
}
|
|
5753
|
+
if (input.bytes.byteLength === 0) {
|
|
5754
|
+
throw new ScreenshotValidationError("invalid_content", "computer screenshot is empty");
|
|
5755
|
+
}
|
|
5756
|
+
if (input.bytes.byteLength > maxBytes) {
|
|
5757
|
+
throw new ScreenshotValidationError(
|
|
5758
|
+
"oversized",
|
|
5759
|
+
`computer screenshot exceeds ${maxBytes} bytes`
|
|
5760
|
+
);
|
|
5761
|
+
}
|
|
5762
|
+
if (input.bytes.byteLength < PNG_SIGNATURE.byteLength + 25) {
|
|
5763
|
+
throw new ScreenshotValidationError("invalid_content", "computer screenshot PNG is truncated");
|
|
5764
|
+
}
|
|
5765
|
+
for (let index = 0; index < PNG_SIGNATURE.byteLength; index += 1) {
|
|
5766
|
+
if (input.bytes[index] !== PNG_SIGNATURE[index]) {
|
|
5767
|
+
throw new ScreenshotValidationError(
|
|
5768
|
+
"invalid_content",
|
|
5769
|
+
"computer screenshot PNG signature is invalid"
|
|
5770
|
+
);
|
|
5771
|
+
}
|
|
5772
|
+
}
|
|
5773
|
+
let offset = PNG_SIGNATURE.byteLength;
|
|
5774
|
+
let chunks = 0;
|
|
5775
|
+
let width = 0;
|
|
5776
|
+
let height = 0;
|
|
5777
|
+
let sawIhdr = false;
|
|
5778
|
+
let sawIdat = false;
|
|
5779
|
+
let endedIdat = false;
|
|
5780
|
+
let sawIend = false;
|
|
5781
|
+
while (offset < input.bytes.byteLength) {
|
|
5782
|
+
chunks += 1;
|
|
5783
|
+
if (chunks > PNG_MAX_CHUNKS || offset + 12 > input.bytes.byteLength) {
|
|
5784
|
+
throw new ScreenshotValidationError(
|
|
5785
|
+
"invalid_content",
|
|
5786
|
+
"computer screenshot PNG chunk table is invalid"
|
|
5787
|
+
);
|
|
5788
|
+
}
|
|
5789
|
+
const length = readUint32(input.bytes, offset);
|
|
5790
|
+
const typeOffset = offset + 4;
|
|
5791
|
+
const dataOffset = typeOffset + 4;
|
|
5792
|
+
const dataEnd = dataOffset + length;
|
|
5793
|
+
const crcOffset = dataEnd;
|
|
5794
|
+
const nextOffset = crcOffset + 4;
|
|
5795
|
+
if (!Number.isSafeInteger(nextOffset) || nextOffset > input.bytes.byteLength) {
|
|
5796
|
+
throw new ScreenshotValidationError(
|
|
5797
|
+
"invalid_content",
|
|
5798
|
+
"computer screenshot PNG chunk is truncated"
|
|
5799
|
+
);
|
|
5800
|
+
}
|
|
5801
|
+
const type = asciiChunkType(input.bytes, typeOffset);
|
|
5802
|
+
const expectedCrc = readUint32(input.bytes, crcOffset);
|
|
5803
|
+
const actualCrc = crc32(input.bytes.subarray(typeOffset, dataEnd));
|
|
5804
|
+
if (expectedCrc !== actualCrc) {
|
|
5805
|
+
throw new ScreenshotValidationError(
|
|
5806
|
+
"invalid_content",
|
|
5807
|
+
`computer screenshot PNG ${type} CRC is invalid`
|
|
5808
|
+
);
|
|
5809
|
+
}
|
|
5810
|
+
if (chunks === 1 && type !== "IHDR") {
|
|
5811
|
+
throw new ScreenshotValidationError(
|
|
5812
|
+
"invalid_content",
|
|
5813
|
+
"computer screenshot PNG must begin with IHDR"
|
|
5814
|
+
);
|
|
5815
|
+
}
|
|
5816
|
+
if (isCriticalChunk(type) && !PNG_ALLOWED_CRITICAL_CHUNKS.has(type)) {
|
|
5817
|
+
throw new ScreenshotValidationError(
|
|
5818
|
+
"invalid_content",
|
|
5819
|
+
`computer screenshot PNG has unsupported critical chunk ${type}`
|
|
5820
|
+
);
|
|
5821
|
+
}
|
|
5822
|
+
if (type === "IHDR") {
|
|
5823
|
+
if (sawIhdr || length !== 13) {
|
|
5824
|
+
throw new ScreenshotValidationError(
|
|
5825
|
+
"invalid_content",
|
|
5826
|
+
"computer screenshot PNG IHDR is invalid"
|
|
5827
|
+
);
|
|
5828
|
+
}
|
|
5829
|
+
sawIhdr = true;
|
|
5830
|
+
width = readUint32(input.bytes, dataOffset);
|
|
5831
|
+
height = readUint32(input.bytes, dataOffset + 4);
|
|
5832
|
+
const bitDepth = input.bytes[dataOffset + 8];
|
|
5833
|
+
const colorType = input.bytes[dataOffset + 9];
|
|
5834
|
+
const compression = input.bytes[dataOffset + 10];
|
|
5835
|
+
const filter = input.bytes[dataOffset + 11];
|
|
5836
|
+
const interlace = input.bytes[dataOffset + 12];
|
|
5837
|
+
if (!validPngColorMode(bitDepth, colorType) || compression !== 0 || filter !== 0 || interlace !== 0 && interlace !== 1) {
|
|
5838
|
+
throw new ScreenshotValidationError(
|
|
5839
|
+
"invalid_content",
|
|
5840
|
+
"computer screenshot PNG IHDR fields are invalid"
|
|
5841
|
+
);
|
|
5842
|
+
}
|
|
5843
|
+
if (width <= 0 || height <= 0 || width > maxDimension || height > maxDimension || width * height > maxPixels) {
|
|
5844
|
+
throw new ScreenshotValidationError(
|
|
5845
|
+
"oversized",
|
|
5846
|
+
"computer screenshot dimensions exceed policy"
|
|
5847
|
+
);
|
|
5848
|
+
}
|
|
5849
|
+
} else if (type === "IDAT") {
|
|
5850
|
+
if (!sawIhdr || sawIend || endedIdat || length === 0) {
|
|
5851
|
+
throw new ScreenshotValidationError(
|
|
5852
|
+
"invalid_content",
|
|
5853
|
+
"computer screenshot PNG IDAT ordering is invalid"
|
|
5854
|
+
);
|
|
5855
|
+
}
|
|
5856
|
+
sawIdat = true;
|
|
5857
|
+
} else if (sawIdat && type !== "IEND") {
|
|
5858
|
+
endedIdat = true;
|
|
5859
|
+
}
|
|
5860
|
+
if (type === "IEND") {
|
|
5861
|
+
if (!sawIhdr || !sawIdat || sawIend || length !== 0) {
|
|
5862
|
+
throw new ScreenshotValidationError(
|
|
5863
|
+
"invalid_content",
|
|
5864
|
+
"computer screenshot PNG IEND is invalid"
|
|
5865
|
+
);
|
|
5866
|
+
}
|
|
5867
|
+
sawIend = true;
|
|
5868
|
+
if (nextOffset !== input.bytes.byteLength) {
|
|
5869
|
+
throw new ScreenshotValidationError(
|
|
5870
|
+
"invalid_content",
|
|
5871
|
+
"computer screenshot PNG has trailing/polyglot bytes"
|
|
5872
|
+
);
|
|
5873
|
+
}
|
|
5874
|
+
}
|
|
5875
|
+
offset = nextOffset;
|
|
5876
|
+
}
|
|
5877
|
+
if (!sawIhdr || !sawIdat || !sawIend || offset !== input.bytes.byteLength) {
|
|
5878
|
+
throw new ScreenshotValidationError("invalid_content", "computer screenshot PNG is incomplete");
|
|
5879
|
+
}
|
|
5880
|
+
return {
|
|
5881
|
+
bytes: input.bytes,
|
|
5882
|
+
mediaType: "image/png",
|
|
5883
|
+
sizeBytes: input.bytes.byteLength,
|
|
5884
|
+
sha256: createHash3("sha256").update(input.bytes).digest("hex"),
|
|
5885
|
+
width,
|
|
5886
|
+
height
|
|
5887
|
+
};
|
|
5888
|
+
}
|
|
5889
|
+
function retainedScreenshotIdentity(input) {
|
|
5890
|
+
const settlementKey = createHash3("sha256").update("opengeni:computer-screenshot:v1\0").update(input.sessionId).update("\0").update(input.turnId).update("\0").update(input.attemptId).update("\0").update(input.toolCallId).update("\0").update(input.toolOutputId).digest("hex");
|
|
5891
|
+
const uuidBytes = Uint8Array.from(Buffer.from(settlementKey.slice(0, 32), "hex"));
|
|
5892
|
+
uuidBytes[6] = uuidBytes[6] & 15 | 128;
|
|
5893
|
+
uuidBytes[8] = uuidBytes[8] & 63 | 128;
|
|
5894
|
+
const hex = Buffer.from(uuidBytes).toString("hex");
|
|
5895
|
+
return {
|
|
5896
|
+
artifactId: `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`,
|
|
5897
|
+
settlementKey
|
|
5898
|
+
};
|
|
5899
|
+
}
|
|
5900
|
+
async function retainComputerScreenshot(input) {
|
|
5901
|
+
const identity = retainedScreenshotIdentity({
|
|
5902
|
+
sessionId: input.sessionId,
|
|
5903
|
+
turnId: input.turnId,
|
|
5904
|
+
attemptId: input.attemptId,
|
|
5905
|
+
toolCallId: input.output.callId,
|
|
5906
|
+
toolOutputId: input.output.toolOutputId
|
|
5907
|
+
});
|
|
5908
|
+
let screenshot;
|
|
5909
|
+
try {
|
|
5910
|
+
screenshot = validateComputerScreenshot(input.output);
|
|
5911
|
+
} catch (error) {
|
|
5912
|
+
if (error instanceof ScreenshotValidationError) {
|
|
5913
|
+
return unavailable(identity.artifactId, error.reason);
|
|
5914
|
+
}
|
|
5915
|
+
throw error;
|
|
5916
|
+
}
|
|
5917
|
+
if (!input.objectStorage) return unavailable(identity.artifactId, "missing_storage");
|
|
5918
|
+
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
5919
|
+
const retentionExpiresAt = new Date(
|
|
5920
|
+
now.getTime() + (input.retentionMs ?? COMPUTER_SCREENSHOT_RETENTION_MS)
|
|
5921
|
+
);
|
|
5922
|
+
const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/computer-screenshot.png`;
|
|
5923
|
+
let prepared;
|
|
5924
|
+
try {
|
|
5925
|
+
prepared = (await prepareRetainedScreenshotArtifact(input.db, {
|
|
5926
|
+
artifactId: identity.artifactId,
|
|
5927
|
+
accountId: input.accountId,
|
|
5928
|
+
workspaceId: input.workspaceId,
|
|
5929
|
+
sessionId: input.sessionId,
|
|
5930
|
+
turnId: input.turnId,
|
|
5931
|
+
attemptId: input.attemptId,
|
|
5932
|
+
settlementKey: identity.settlementKey,
|
|
5933
|
+
toolCallId: input.output.callId,
|
|
5934
|
+
toolOutputId: input.output.toolOutputId,
|
|
5935
|
+
mediaType: screenshot.mediaType,
|
|
5936
|
+
sizeBytes: screenshot.sizeBytes,
|
|
5937
|
+
sha256: screenshot.sha256,
|
|
5938
|
+
width: screenshot.width,
|
|
5939
|
+
height: screenshot.height,
|
|
5940
|
+
retentionExpiresAt,
|
|
5941
|
+
bucket: input.objectStorage.bucket,
|
|
5942
|
+
objectKey,
|
|
5943
|
+
workspaceQuotaBytes: input.workspaceQuotaBytes ?? COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES
|
|
5944
|
+
})).artifact;
|
|
5945
|
+
} catch (error) {
|
|
5946
|
+
if (error instanceof RetainedScreenshotQuotaExceededError) {
|
|
5947
|
+
return unavailable(identity.artifactId, "quota_exceeded");
|
|
5948
|
+
}
|
|
5949
|
+
throw error;
|
|
5950
|
+
}
|
|
5951
|
+
if (prepared.status === "ready") {
|
|
5952
|
+
return await verifyReadyArtifact(input.objectStorage, prepared) ? reference(prepared) : unavailable(identity.artifactId, "missing_storage");
|
|
5953
|
+
}
|
|
5954
|
+
if (prepared.status !== "pending" && prepared.status !== "reconciling") {
|
|
5955
|
+
return unavailable(identity.artifactId, unavailableReasonForStatus(prepared.status));
|
|
5956
|
+
}
|
|
5957
|
+
try {
|
|
5958
|
+
await input.objectStorage.putObject({
|
|
5959
|
+
key: objectKey,
|
|
5960
|
+
contentType: screenshot.mediaType,
|
|
5961
|
+
body: screenshot.bytes,
|
|
5962
|
+
sha256: screenshot.sha256
|
|
5963
|
+
});
|
|
5964
|
+
const head = await input.objectStorage.headFile(prepared.file);
|
|
5965
|
+
assertStoredScreenshotHead(head, screenshot);
|
|
5966
|
+
} catch (error) {
|
|
5967
|
+
await recordRetainedScreenshotArtifactError(input.db, {
|
|
5968
|
+
accountId: input.accountId,
|
|
5969
|
+
workspaceId: input.workspaceId,
|
|
5970
|
+
artifactId: identity.artifactId,
|
|
5971
|
+
settlementKey: identity.settlementKey,
|
|
5972
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5973
|
+
}).catch(() => void 0);
|
|
5974
|
+
return unavailable(identity.artifactId, "storage_write_failed");
|
|
5975
|
+
}
|
|
5976
|
+
try {
|
|
5977
|
+
const settled = await settleRetainedScreenshotArtifactReady(input.db, {
|
|
5978
|
+
accountId: input.accountId,
|
|
5979
|
+
workspaceId: input.workspaceId,
|
|
5980
|
+
artifactId: identity.artifactId,
|
|
5981
|
+
settlementKey: identity.settlementKey
|
|
5982
|
+
});
|
|
5983
|
+
return reference(settled);
|
|
5984
|
+
} catch (error) {
|
|
5985
|
+
const stillReferenced = await getRetainedScreenshotArtifact(
|
|
5986
|
+
input.db,
|
|
5987
|
+
input.workspaceId,
|
|
5988
|
+
input.sessionId,
|
|
5989
|
+
identity.artifactId
|
|
5990
|
+
).catch(() => void 0);
|
|
5991
|
+
if (stillReferenced?.status === "ready") {
|
|
5992
|
+
return await verifyReadyArtifact(input.objectStorage, stillReferenced) ? reference(stillReferenced) : unavailable(identity.artifactId, "missing_storage");
|
|
5993
|
+
}
|
|
5994
|
+
if (stillReferenced === null || stillReferenced?.status === "cleanup_queued" || stillReferenced?.status === "cleanup_pending" || stillReferenced?.status === "failed" || stillReferenced?.status === "expired" || stillReferenced?.status === "deleted") {
|
|
5995
|
+
await input.objectStorage.deleteObject(objectKey).catch(() => void 0);
|
|
5996
|
+
} else if (stillReferenced) {
|
|
5997
|
+
await recordRetainedScreenshotArtifactError(input.db, {
|
|
5998
|
+
accountId: input.accountId,
|
|
5999
|
+
workspaceId: input.workspaceId,
|
|
6000
|
+
artifactId: identity.artifactId,
|
|
6001
|
+
settlementKey: identity.settlementKey,
|
|
6002
|
+
error: error instanceof Error ? error.message : String(error)
|
|
6003
|
+
}).catch(() => void 0);
|
|
6004
|
+
}
|
|
6005
|
+
return unavailable(identity.artifactId, "pending");
|
|
6006
|
+
}
|
|
6007
|
+
}
|
|
6008
|
+
function compactRetainedScreenshotHistory(history, receiptsByCallId) {
|
|
6009
|
+
if (receiptsByCallId.size === 0) return history;
|
|
6010
|
+
return history.map((item) => {
|
|
6011
|
+
const callId = historyCallId(item);
|
|
6012
|
+
const receipt = callId ? receiptsByCallId.get(callId) : void 0;
|
|
6013
|
+
if (!receipt || !Array.isArray(item.output)) return item;
|
|
6014
|
+
let changed = false;
|
|
6015
|
+
const output = item.output.map((entry) => {
|
|
6016
|
+
if (!isInlineImageContent(entry)) return entry;
|
|
6017
|
+
changed = true;
|
|
6018
|
+
return {
|
|
6019
|
+
...entry,
|
|
6020
|
+
image: { type: RETAINED_IMAGE_MARKER, artifact: receipt }
|
|
6021
|
+
};
|
|
6022
|
+
});
|
|
6023
|
+
return changed ? { ...item, output } : item;
|
|
6024
|
+
});
|
|
6025
|
+
}
|
|
6026
|
+
function collectRetainedScreenshotReceipts(history, target = /* @__PURE__ */ new Map()) {
|
|
6027
|
+
for (const item of history) {
|
|
6028
|
+
const callId = historyCallId(item);
|
|
6029
|
+
if (!callId || !Array.isArray(item.output)) continue;
|
|
6030
|
+
for (const entry of item.output) {
|
|
6031
|
+
const receipt = retainedReceiptFromImageContent(entry);
|
|
6032
|
+
if (receipt) {
|
|
6033
|
+
target.set(callId, receipt);
|
|
6034
|
+
break;
|
|
6035
|
+
}
|
|
6036
|
+
}
|
|
6037
|
+
}
|
|
6038
|
+
return target;
|
|
6039
|
+
}
|
|
6040
|
+
function compactRetainedScreenshotRunState(serialized, receiptsByCallId) {
|
|
6041
|
+
if (receiptsByCallId.size === 0) return serialized;
|
|
6042
|
+
const parsed = parseSerializedRunState(serialized);
|
|
6043
|
+
if (!parsed) return serialized;
|
|
6044
|
+
let changed = false;
|
|
6045
|
+
visitSerializedRunStateItemArrays(parsed, (items) => {
|
|
6046
|
+
const compacted = compactRetainedScreenshotHistory(items, receiptsByCallId);
|
|
6047
|
+
if (compacted !== items && compacted.some((item, index) => item !== items[index])) {
|
|
6048
|
+
items.splice(0, items.length, ...compacted);
|
|
6049
|
+
changed = true;
|
|
6050
|
+
}
|
|
6051
|
+
});
|
|
6052
|
+
return changed ? JSON.stringify(parsed) : serialized;
|
|
6053
|
+
}
|
|
6054
|
+
function collectRetainedScreenshotRunStateReceipts(serialized, target = /* @__PURE__ */ new Map()) {
|
|
6055
|
+
const parsed = parseSerializedRunState(serialized);
|
|
6056
|
+
if (!parsed) return target;
|
|
6057
|
+
visitSerializedRunStateItemArrays(parsed, (items) => {
|
|
6058
|
+
collectRetainedScreenshotReceipts(items, target);
|
|
6059
|
+
});
|
|
6060
|
+
return target;
|
|
6061
|
+
}
|
|
6062
|
+
async function materializeRetainedScreenshotHistory(input) {
|
|
6063
|
+
return await materializeRetainedScreenshotHistoryWithCache(input, /* @__PURE__ */ new Map());
|
|
6064
|
+
}
|
|
6065
|
+
async function materializeRetainedScreenshotRunState(input) {
|
|
6066
|
+
const parsed = parseSerializedRunState(input.serialized);
|
|
6067
|
+
if (!parsed) return input.serialized;
|
|
6068
|
+
const cache = /* @__PURE__ */ new Map();
|
|
6069
|
+
let changed = false;
|
|
6070
|
+
await visitSerializedRunStateItemArraysAsync(parsed, async (items) => {
|
|
6071
|
+
const materialized = await materializeRetainedScreenshotHistoryWithCache(
|
|
6072
|
+
{ ...input, history: items },
|
|
6073
|
+
cache
|
|
6074
|
+
);
|
|
6075
|
+
if (materialized.some((item, index) => item !== items[index])) {
|
|
6076
|
+
items.splice(0, items.length, ...materialized);
|
|
6077
|
+
changed = true;
|
|
6078
|
+
}
|
|
6079
|
+
});
|
|
6080
|
+
return changed ? JSON.stringify(parsed) : input.serialized;
|
|
6081
|
+
}
|
|
6082
|
+
async function materializeRetainedScreenshotHistoryWithCache(input, cache) {
|
|
6083
|
+
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
6084
|
+
const materializeEntry = async (entry) => {
|
|
6085
|
+
const receipt = retainedReceiptFromImageContent(entry);
|
|
6086
|
+
if (!receipt) return entry;
|
|
6087
|
+
if (!receipt.available) {
|
|
6088
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, receipt.reason);
|
|
6089
|
+
}
|
|
6090
|
+
let dataUrl = cache.get(receipt.artifactId);
|
|
6091
|
+
if (!dataUrl) {
|
|
6092
|
+
if (!input.objectStorage) {
|
|
6093
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
|
|
6094
|
+
}
|
|
6095
|
+
const artifact = await getRetainedScreenshotArtifact(
|
|
6096
|
+
input.db,
|
|
6097
|
+
input.workspaceId,
|
|
6098
|
+
input.sessionId,
|
|
6099
|
+
receipt.artifactId
|
|
6100
|
+
);
|
|
6101
|
+
if (!artifact) throw new RetainedScreenshotUnavailableError(receipt.artifactId, "deleted");
|
|
6102
|
+
if (artifact.status !== "ready") {
|
|
6103
|
+
throw new RetainedScreenshotUnavailableError(
|
|
6104
|
+
receipt.artifactId,
|
|
6105
|
+
unavailableReasonForStatus(artifact.status)
|
|
6106
|
+
);
|
|
6107
|
+
}
|
|
6108
|
+
if (artifact.retentionExpiresAt.getTime() <= now.getTime()) {
|
|
6109
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "expired");
|
|
6110
|
+
}
|
|
6111
|
+
const bytes = await input.objectStorage.getFileBytes(artifact.file).catch(() => null);
|
|
6112
|
+
if (!bytes)
|
|
6113
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
|
|
6114
|
+
const validated = validateComputerScreenshot({
|
|
6115
|
+
bytes,
|
|
6116
|
+
mediaType: artifact.mediaType
|
|
6117
|
+
});
|
|
6118
|
+
if (validated.sizeBytes !== artifact.sizeBytes || validated.sha256 !== artifact.sha256 || validated.width !== artifact.width || validated.height !== artifact.height || receipt.sha256 !== artifact.sha256 || receipt.originalBytes !== artifact.sizeBytes) {
|
|
6119
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "invalid_content");
|
|
6120
|
+
}
|
|
6121
|
+
dataUrl = `data:${validated.mediaType};base64,${Buffer.from(validated.bytes).toString("base64")}`;
|
|
6122
|
+
cache.set(receipt.artifactId, dataUrl);
|
|
6123
|
+
}
|
|
6124
|
+
return { ...entry, image: dataUrl };
|
|
6125
|
+
};
|
|
6126
|
+
const materialized = [];
|
|
6127
|
+
for (const item of input.history) {
|
|
6128
|
+
if (!Array.isArray(item.output)) {
|
|
6129
|
+
materialized.push(item);
|
|
6130
|
+
continue;
|
|
6131
|
+
}
|
|
6132
|
+
let changed = false;
|
|
6133
|
+
const output = [];
|
|
6134
|
+
for (const entry of item.output) {
|
|
6135
|
+
const next = await materializeEntry(entry);
|
|
6136
|
+
changed ||= next !== entry;
|
|
6137
|
+
output.push(next);
|
|
6138
|
+
}
|
|
6139
|
+
materialized.push(changed ? { ...item, output } : item);
|
|
6140
|
+
}
|
|
6141
|
+
return materialized;
|
|
6142
|
+
}
|
|
6143
|
+
function parseSerializedRunState(serialized) {
|
|
6144
|
+
try {
|
|
6145
|
+
const parsed = JSON.parse(serialized);
|
|
6146
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
6147
|
+
} catch {
|
|
6148
|
+
return null;
|
|
6149
|
+
}
|
|
6150
|
+
}
|
|
6151
|
+
function visitSerializedRunStateItemArrays(root, visitor) {
|
|
6152
|
+
const originalInput = root.originalInput;
|
|
6153
|
+
if (Array.isArray(originalInput)) {
|
|
6154
|
+
visitor(originalInput);
|
|
6155
|
+
}
|
|
6156
|
+
if (Array.isArray(root.generatedItems)) {
|
|
6157
|
+
for (const wrapper of root.generatedItems) {
|
|
6158
|
+
if (!wrapper || typeof wrapper !== "object" || Array.isArray(wrapper)) continue;
|
|
6159
|
+
const record = wrapper;
|
|
6160
|
+
const rawItem = record.rawItem;
|
|
6161
|
+
if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
|
|
6162
|
+
const items = [rawItem];
|
|
6163
|
+
visitor(items);
|
|
6164
|
+
if (items[0] !== rawItem) record.rawItem = items[0];
|
|
6165
|
+
}
|
|
6166
|
+
}
|
|
6167
|
+
if (Array.isArray(root.modelResponses)) {
|
|
6168
|
+
for (const response of root.modelResponses) {
|
|
6169
|
+
if (!response || typeof response !== "object" || Array.isArray(response)) continue;
|
|
6170
|
+
const output = response.output;
|
|
6171
|
+
if (Array.isArray(output)) visitor(output);
|
|
6172
|
+
}
|
|
6173
|
+
}
|
|
6174
|
+
const lastModelResponse = root.lastModelResponse;
|
|
6175
|
+
if (lastModelResponse && typeof lastModelResponse === "object" && !Array.isArray(lastModelResponse)) {
|
|
6176
|
+
const output = lastModelResponse.output;
|
|
6177
|
+
if (Array.isArray(output)) visitor(output);
|
|
6178
|
+
}
|
|
6179
|
+
}
|
|
6180
|
+
async function visitSerializedRunStateItemArraysAsync(root, visitor) {
|
|
6181
|
+
const originalInput = root.originalInput;
|
|
6182
|
+
if (Array.isArray(originalInput)) {
|
|
6183
|
+
await visitor(originalInput);
|
|
6184
|
+
}
|
|
6185
|
+
if (Array.isArray(root.generatedItems)) {
|
|
6186
|
+
for (const wrapper of root.generatedItems) {
|
|
6187
|
+
if (!wrapper || typeof wrapper !== "object" || Array.isArray(wrapper)) continue;
|
|
6188
|
+
const record = wrapper;
|
|
6189
|
+
const rawItem = record.rawItem;
|
|
6190
|
+
if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
|
|
6191
|
+
const items = [rawItem];
|
|
6192
|
+
await visitor(items);
|
|
6193
|
+
if (items[0] !== rawItem) record.rawItem = items[0];
|
|
6194
|
+
}
|
|
6195
|
+
}
|
|
6196
|
+
if (Array.isArray(root.modelResponses)) {
|
|
6197
|
+
for (const response of root.modelResponses) {
|
|
6198
|
+
if (!response || typeof response !== "object" || Array.isArray(response)) continue;
|
|
6199
|
+
const output = response.output;
|
|
6200
|
+
if (Array.isArray(output)) await visitor(output);
|
|
6201
|
+
}
|
|
6202
|
+
}
|
|
6203
|
+
const lastModelResponse = root.lastModelResponse;
|
|
6204
|
+
if (lastModelResponse && typeof lastModelResponse === "object" && !Array.isArray(lastModelResponse)) {
|
|
6205
|
+
const output = lastModelResponse.output;
|
|
6206
|
+
if (Array.isArray(output)) await visitor(output);
|
|
6207
|
+
}
|
|
6208
|
+
}
|
|
6209
|
+
function reference(artifact) {
|
|
6210
|
+
if (!artifact.sessionId) {
|
|
6211
|
+
throw new Error(`Ready retained screenshot is detached: ${artifact.artifactId}`);
|
|
6212
|
+
}
|
|
6213
|
+
const value = retainedScreenshotReferenceFromFile({
|
|
6214
|
+
...artifact.file,
|
|
6215
|
+
sessionId: artifact.sessionId,
|
|
6216
|
+
width: artifact.width,
|
|
6217
|
+
height: artifact.height,
|
|
6218
|
+
expiresAt: artifact.retentionExpiresAt.toISOString()
|
|
6219
|
+
});
|
|
6220
|
+
if (!value)
|
|
6221
|
+
throw new Error(`Ready retained screenshot receipt is invalid: ${artifact.artifactId}`);
|
|
6222
|
+
return value;
|
|
6223
|
+
}
|
|
6224
|
+
function unavailable(artifactId, reason) {
|
|
6225
|
+
return RetainedArtifactMetadataSchema.parse({
|
|
6226
|
+
available: false,
|
|
6227
|
+
artifactId,
|
|
6228
|
+
reason
|
|
6229
|
+
});
|
|
6230
|
+
}
|
|
6231
|
+
function unavailableReasonForStatus(status) {
|
|
6232
|
+
switch (status) {
|
|
6233
|
+
case "pending":
|
|
6234
|
+
case "reconciling":
|
|
6235
|
+
return "pending";
|
|
6236
|
+
case "cleanup_queued":
|
|
6237
|
+
case "cleanup_pending":
|
|
6238
|
+
return "failed";
|
|
6239
|
+
case "failed":
|
|
6240
|
+
case "expired":
|
|
6241
|
+
case "deleted":
|
|
6242
|
+
return status;
|
|
6243
|
+
case "ready":
|
|
6244
|
+
return "missing_storage";
|
|
6245
|
+
}
|
|
6246
|
+
}
|
|
6247
|
+
async function verifyReadyArtifact(storage, artifact) {
|
|
6248
|
+
try {
|
|
6249
|
+
assertStoredScreenshotHead(await storage.headFile(artifact.file), {
|
|
6250
|
+
sizeBytes: artifact.sizeBytes,
|
|
6251
|
+
mediaType: "image/png",
|
|
6252
|
+
sha256: artifact.sha256
|
|
6253
|
+
});
|
|
6254
|
+
return true;
|
|
6255
|
+
} catch {
|
|
6256
|
+
return false;
|
|
6257
|
+
}
|
|
6258
|
+
}
|
|
6259
|
+
function assertStoredScreenshotHead(head, expected) {
|
|
6260
|
+
if (head.ContentLength !== expected.sizeBytes) {
|
|
6261
|
+
throw new Error("retained screenshot object size mismatch");
|
|
6262
|
+
}
|
|
6263
|
+
if (head.ContentType !== expected.mediaType) {
|
|
6264
|
+
throw new Error("retained screenshot object MIME mismatch");
|
|
6265
|
+
}
|
|
6266
|
+
if (head.Metadata?.sha256 !== expected.sha256) {
|
|
6267
|
+
throw new Error("retained screenshot object SHA-256 metadata mismatch");
|
|
6268
|
+
}
|
|
6269
|
+
}
|
|
6270
|
+
function retainedReceiptFromImageContent(entry) {
|
|
6271
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return null;
|
|
6272
|
+
const image = entry.image;
|
|
6273
|
+
if (!image || typeof image !== "object" || Array.isArray(image)) return null;
|
|
6274
|
+
const marker = image;
|
|
6275
|
+
if (marker.type !== RETAINED_IMAGE_MARKER) return null;
|
|
6276
|
+
const parsed = RetainedArtifactMetadataSchema.safeParse(marker.artifact);
|
|
6277
|
+
if (!parsed.success) return null;
|
|
6278
|
+
return parsed.data.available && parsed.data.kind !== "computer_screenshot" ? null : parsed.data;
|
|
6279
|
+
}
|
|
6280
|
+
function isInlineImageContent(entry) {
|
|
6281
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
|
|
6282
|
+
const record = entry;
|
|
6283
|
+
return record.type === "input_image" && typeof record.image === "string" && record.image.startsWith("data:image/png;base64,");
|
|
6284
|
+
}
|
|
6285
|
+
function historyCallId(item) {
|
|
6286
|
+
const value = item.callId ?? item.call_id;
|
|
6287
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
6288
|
+
}
|
|
6289
|
+
function readUint32(bytes, offset) {
|
|
6290
|
+
return (bytes[offset] << 24 | bytes[offset + 1] << 16 | bytes[offset + 2] << 8 | bytes[offset + 3]) >>> 0;
|
|
6291
|
+
}
|
|
6292
|
+
function asciiChunkType(bytes, offset) {
|
|
6293
|
+
const chars = [bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]];
|
|
6294
|
+
if (chars.some(
|
|
6295
|
+
(value) => value === void 0 || value < 65 || value > 90 && value < 97 || value > 122
|
|
6296
|
+
)) {
|
|
6297
|
+
throw new ScreenshotValidationError(
|
|
6298
|
+
"invalid_content",
|
|
6299
|
+
"computer screenshot PNG chunk type is invalid"
|
|
6300
|
+
);
|
|
6301
|
+
}
|
|
6302
|
+
return String.fromCharCode(...chars);
|
|
6303
|
+
}
|
|
6304
|
+
function isCriticalChunk(type) {
|
|
6305
|
+
return type.charCodeAt(0) >= 65 && type.charCodeAt(0) <= 90;
|
|
6306
|
+
}
|
|
6307
|
+
function validPngColorMode(bitDepth, colorType) {
|
|
6308
|
+
if (bitDepth === void 0 || colorType === void 0) return false;
|
|
6309
|
+
switch (colorType) {
|
|
6310
|
+
case 0:
|
|
6311
|
+
return [1, 2, 4, 8, 16].includes(bitDepth);
|
|
6312
|
+
case 2:
|
|
6313
|
+
case 4:
|
|
6314
|
+
case 6:
|
|
6315
|
+
return bitDepth === 8 || bitDepth === 16;
|
|
6316
|
+
case 3:
|
|
6317
|
+
return [1, 2, 4, 8].includes(bitDepth);
|
|
6318
|
+
default:
|
|
6319
|
+
return false;
|
|
6320
|
+
}
|
|
6321
|
+
}
|
|
6322
|
+
var CRC32_TABLE = Uint32Array.from({ length: 256 }, (_, value) => {
|
|
6323
|
+
let current = value;
|
|
6324
|
+
for (let bit = 0; bit < 8; bit += 1) {
|
|
6325
|
+
current = (current & 1) !== 0 ? 3988292384 ^ current >>> 1 : current >>> 1;
|
|
6326
|
+
}
|
|
6327
|
+
return current >>> 0;
|
|
6328
|
+
});
|
|
6329
|
+
function crc32(bytes) {
|
|
6330
|
+
let crc = 4294967295;
|
|
6331
|
+
for (const byte of bytes) crc = CRC32_TABLE[(crc ^ byte) & 255] ^ crc >>> 8;
|
|
6332
|
+
return (crc ^ 4294967295) >>> 0;
|
|
6333
|
+
}
|
|
6334
|
+
|
|
6335
|
+
// src/activities/workspace-capture.ts
|
|
6336
|
+
import { createHash as createHash4 } from "crypto";
|
|
5639
6337
|
import {
|
|
5640
6338
|
deleteWorkspaceCaptureRows,
|
|
5641
6339
|
insertFailedWorkspaceCapture,
|
|
@@ -5753,7 +6451,7 @@ async function readCaptureRepository(svc, root) {
|
|
|
5753
6451
|
return { complete: false, degradedReason: "repository_read_unavailable" };
|
|
5754
6452
|
}
|
|
5755
6453
|
try {
|
|
5756
|
-
const
|
|
6454
|
+
const workingRequest = svc.gitDiff({
|
|
5757
6455
|
path: root,
|
|
5758
6456
|
staged: false,
|
|
5759
6457
|
includeUntracked: true,
|
|
@@ -5762,7 +6460,24 @@ async function readCaptureRepository(svc, root) {
|
|
|
5762
6460
|
contextLines: 3,
|
|
5763
6461
|
maxBytesPerFile: PER_FILE_DIFF_GUARD_BYTES
|
|
5764
6462
|
});
|
|
5765
|
-
|
|
6463
|
+
const branchRequest = Promise.resolve().then(
|
|
6464
|
+
() => svc.gitDiff({
|
|
6465
|
+
path: root,
|
|
6466
|
+
staged: false,
|
|
6467
|
+
includeUntracked: true,
|
|
6468
|
+
fromRef: "origin/HEAD",
|
|
6469
|
+
pathspec: [],
|
|
6470
|
+
contextLines: 3,
|
|
6471
|
+
maxBytesPerFile: PER_FILE_DIFF_GUARD_BYTES
|
|
6472
|
+
})
|
|
6473
|
+
).then((value) => value).catch(() => null);
|
|
6474
|
+
const [diff, branchDiff] = await Promise.all([workingRequest, branchRequest]);
|
|
6475
|
+
return {
|
|
6476
|
+
complete: true,
|
|
6477
|
+
status,
|
|
6478
|
+
diff,
|
|
6479
|
+
...branchDiff ? { branchDiff } : {}
|
|
6480
|
+
};
|
|
5766
6481
|
} catch (error) {
|
|
5767
6482
|
const boxExiting = classifyCaptureEntryError(error);
|
|
5768
6483
|
if (boxExiting) throw boxExiting;
|
|
@@ -5915,7 +6630,7 @@ async function runCapture(input, ctx, startedAt, signal, stage) {
|
|
|
5915
6630
|
);
|
|
5916
6631
|
return;
|
|
5917
6632
|
}
|
|
5918
|
-
const { status, diff } = repository;
|
|
6633
|
+
const { status, diff, branchDiff } = repository;
|
|
5919
6634
|
const diffFiles = diff.files.filter((f) => !isUnderResidueDir(joinRepoPath(root, f.path)));
|
|
5920
6635
|
const statusFiles = status.files.filter((f) => !isUnderResidueDir(joinRepoPath(root, f.path)));
|
|
5921
6636
|
repos.push({
|
|
@@ -5926,7 +6641,12 @@ async function runCapture(input, ctx, startedAt, signal, stage) {
|
|
|
5926
6641
|
ahead: status.ahead,
|
|
5927
6642
|
behind: status.behind,
|
|
5928
6643
|
status: statusFiles,
|
|
5929
|
-
diff: diffFiles
|
|
6644
|
+
diff: diffFiles,
|
|
6645
|
+
...branchDiff ? {
|
|
6646
|
+
branchDiff: branchDiff.files.filter(
|
|
6647
|
+
(f) => !isUnderResidueDir(joinRepoPath(root, f.path))
|
|
6648
|
+
)
|
|
6649
|
+
} : {}
|
|
5930
6650
|
});
|
|
5931
6651
|
for (const f of diffFiles) {
|
|
5932
6652
|
additions += f.additions;
|
|
@@ -6337,7 +7057,7 @@ function joinRepoPath(repoRoot, repoRelPath) {
|
|
|
6337
7057
|
return `${repoRoot.replace(/\/+$/, "")}/${repoRelPath}`;
|
|
6338
7058
|
}
|
|
6339
7059
|
function sha256(bytes) {
|
|
6340
|
-
return
|
|
7060
|
+
return createHash4("sha256").update(bytes).digest("hex");
|
|
6341
7061
|
}
|
|
6342
7062
|
function utf8(s) {
|
|
6343
7063
|
return new TextEncoder().encode(s);
|
|
@@ -6417,11 +7137,21 @@ import {
|
|
|
6417
7137
|
DEFAULT_FIRST_PARTY_MCP_TOOLS,
|
|
6418
7138
|
evaluateWorkspaceModelPolicy as evaluateWorkspaceModelPolicy2,
|
|
6419
7139
|
readTurnExecutionPolicyV1,
|
|
7140
|
+
resolveWorkspaceAgentHumanInputEnabled,
|
|
6420
7141
|
resourceMountPath as resourceMountPath2
|
|
6421
7142
|
} from "@opengeni/contracts";
|
|
6422
|
-
import { createHash as
|
|
7143
|
+
import { createHash as createHash5, randomUUID as randomUUID2 } from "crypto";
|
|
6423
7144
|
var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
|
|
6424
7145
|
var PROVIDER_CONNECTIVITY_BACKOFF_MS = [2e3, 5e3, 15e3, 3e4, 6e4];
|
|
7146
|
+
var WorkspaceHumanInputDisabledError = class extends Error {
|
|
7147
|
+
constructor(kind) {
|
|
7148
|
+
super(`Workspace policy rejects structured human-input ${kind}`);
|
|
7149
|
+
this.name = "WorkspaceHumanInputDisabledError";
|
|
7150
|
+
}
|
|
7151
|
+
};
|
|
7152
|
+
function assertWorkspaceHumanInputAllowed(enabled, kind, attempted) {
|
|
7153
|
+
if (!enabled && attempted) throw new WorkspaceHumanInputDisabledError(kind);
|
|
7154
|
+
}
|
|
6425
7155
|
function credentialSubjectIdForTurnInitiator(turn) {
|
|
6426
7156
|
return directPersonalConnectionSubjectId(turn);
|
|
6427
7157
|
}
|
|
@@ -6541,10 +7271,10 @@ function filterUnmaterializedSandboxFileDownloads(downloads, materializedFileIds
|
|
|
6541
7271
|
return downloads.filter((download) => !materializedFileIds.has(download.fileId));
|
|
6542
7272
|
}
|
|
6543
7273
|
function codexWorkspaceMetricKey(workspaceId) {
|
|
6544
|
-
return
|
|
7274
|
+
return createHash5("sha256").update(workspaceId).digest("hex").slice(0, 12);
|
|
6545
7275
|
}
|
|
6546
7276
|
function stableHumanInputRequestId(sessionId, turnId, toolCallId) {
|
|
6547
|
-
const hex =
|
|
7277
|
+
const hex = createHash5("sha256").update("opengeni-human-input-v1\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
|
|
6548
7278
|
hex[12] = "5";
|
|
6549
7279
|
hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
|
|
6550
7280
|
const value = hex.join("");
|
|
@@ -6647,6 +7377,16 @@ async function waitForTurnStreamCleanup(batcherFlush, providerCompleted, signal)
|
|
|
6647
7377
|
await waitForTurnFinalizerStep(batcherFlush, signal);
|
|
6648
7378
|
await waitForTurnFinalizerStep(providerCompleted, signal);
|
|
6649
7379
|
}
|
|
7380
|
+
async function clearAttemptCredentialsWithSettledFence(input) {
|
|
7381
|
+
try {
|
|
7382
|
+
await input.runWorkspaceFencedClear();
|
|
7383
|
+
} catch (error) {
|
|
7384
|
+
const settledAttemptFence = error instanceof Error && error.name === "SandboxWorkspaceMutationFencedError" && error.code === "attempt_fenced" && (input.activityStatus === "idle" || input.activityStatus === "failed");
|
|
7385
|
+
if (!settledAttemptFence) throw error;
|
|
7386
|
+
input.onSettledAttemptFence();
|
|
7387
|
+
await input.clearExactAttempt();
|
|
7388
|
+
}
|
|
7389
|
+
}
|
|
6650
7390
|
function turnFinalizerCancellationSignal(temporalSignal, activityStatus) {
|
|
6651
7391
|
if (activityStatus !== "cancelled" || temporalSignal?.aborted) return temporalSignal;
|
|
6652
7392
|
const fenced = new AbortController();
|
|
@@ -6656,56 +7396,55 @@ function turnFinalizerCancellationSignal(temporalSignal, activityStatus) {
|
|
|
6656
7396
|
function compactionFailureReason(reason) {
|
|
6657
7397
|
return reason.startsWith("compaction summarization failed:") ? reason : `compaction summarization failed: ${reason}`;
|
|
6658
7398
|
}
|
|
6659
|
-
|
|
6660
|
-
|
|
6661
|
-
|
|
6662
|
-
|
|
7399
|
+
var MandatoryHistoryPersistenceError = class extends Error {
|
|
7400
|
+
constructor(stage, cause) {
|
|
7401
|
+
super(cause instanceof Error ? cause.message : String(cause));
|
|
7402
|
+
this.stage = stage;
|
|
7403
|
+
this.cause = cause;
|
|
7404
|
+
}
|
|
7405
|
+
name = "MandatoryHistoryPersistenceError";
|
|
7406
|
+
cause;
|
|
7407
|
+
};
|
|
7408
|
+
async function runMandatoryHistoryPersistenceStep(stage, operation) {
|
|
7409
|
+
try {
|
|
7410
|
+
return await operation();
|
|
7411
|
+
} catch (cause) {
|
|
7412
|
+
throw new MandatoryHistoryPersistenceError(stage, cause);
|
|
7413
|
+
}
|
|
7414
|
+
}
|
|
7415
|
+
function safeErrorDiagnostic(error) {
|
|
6663
7416
|
const diagnostic = {
|
|
6664
|
-
|
|
6665
|
-
|
|
7417
|
+
errorClass: "WorkerOperationError",
|
|
7418
|
+
errorCode: "worker_operation_failed",
|
|
7419
|
+
origin: "worker"
|
|
6666
7420
|
};
|
|
6667
|
-
|
|
6668
|
-
|
|
6669
|
-
|
|
6670
|
-
|
|
6671
|
-
|
|
6672
|
-
|
|
6673
|
-
|
|
6674
|
-
|
|
6675
|
-
|
|
6676
|
-
|
|
6677
|
-
if (
|
|
7421
|
+
try {
|
|
7422
|
+
let statusSource = error;
|
|
7423
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
7424
|
+
diagnostic.historyPersistenceStage = error.stage;
|
|
7425
|
+
statusSource = error.cause;
|
|
7426
|
+
}
|
|
7427
|
+
if (statusSource && typeof statusSource === "object") {
|
|
7428
|
+
const status = Number(
|
|
7429
|
+
statusSource.status ?? statusSource.statusCode
|
|
7430
|
+
);
|
|
7431
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
7432
|
+
diagnostic.status = status;
|
|
7433
|
+
}
|
|
6678
7434
|
}
|
|
7435
|
+
} catch {
|
|
6679
7436
|
}
|
|
6680
7437
|
return diagnostic;
|
|
6681
7438
|
}
|
|
6682
|
-
function safeErrorForTelemetry(error
|
|
6683
|
-
const diagnostic = safeErrorDiagnostic(error
|
|
6684
|
-
const safe = new Error(
|
|
6685
|
-
safe.name =
|
|
7439
|
+
function safeErrorForTelemetry(error) {
|
|
7440
|
+
const diagnostic = safeErrorDiagnostic(error);
|
|
7441
|
+
const safe = new Error("worker operation failed");
|
|
7442
|
+
safe.name = "WorkerOperationError";
|
|
7443
|
+
safe.code = diagnostic.errorCode;
|
|
7444
|
+
if (diagnostic.status !== void 0) safe.status = diagnostic.status;
|
|
7445
|
+
safe.origin = diagnostic.origin;
|
|
6686
7446
|
return safe;
|
|
6687
7447
|
}
|
|
6688
|
-
function headerSecretRedactions(prefix, headers, credentialHeaderNames) {
|
|
6689
|
-
const discovered = [];
|
|
6690
|
-
const explicitCredentialHeaders = credentialHeaderNames ? new Set([...credentialHeaderNames].map((name) => name.toLowerCase())) : null;
|
|
6691
|
-
for (const [headerName, value] of Object.entries(headers ?? {})) {
|
|
6692
|
-
if (!explicitCredentialHeaders?.has(headerName.toLowerCase()) && !isCredentialHeaderName(headerName)) {
|
|
6693
|
-
continue;
|
|
6694
|
-
}
|
|
6695
|
-
const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
|
|
6696
|
-
discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
|
|
6697
|
-
if (/^(?:proxy-)?authorization$/i.test(headerName)) {
|
|
6698
|
-
const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
|
|
6699
|
-
if (credential) {
|
|
6700
|
-
discovered.push({
|
|
6701
|
-
name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
|
|
6702
|
-
value: credential
|
|
6703
|
-
});
|
|
6704
|
-
}
|
|
6705
|
-
}
|
|
6706
|
-
}
|
|
6707
|
-
return discovered;
|
|
6708
|
-
}
|
|
6709
7448
|
function compactionFailureReasonFromError(error) {
|
|
6710
7449
|
if (error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError2) {
|
|
6711
7450
|
return compactionFailureReason(error.message);
|
|
@@ -7034,14 +7773,9 @@ async function emitModelCallUsage(input) {
|
|
|
7034
7773
|
if (!authoritative) return false;
|
|
7035
7774
|
try {
|
|
7036
7775
|
input.observability.info("model call usage", {
|
|
7037
|
-
accountId: input.accountId,
|
|
7038
|
-
workspaceId: input.workspaceId,
|
|
7039
|
-
sessionId: input.sessionId,
|
|
7040
|
-
turnId: input.turnId,
|
|
7041
7776
|
provider: input.provider,
|
|
7042
7777
|
providerApi: input.providerApi,
|
|
7043
7778
|
model: input.model,
|
|
7044
|
-
sourceKey: input.sourceKey,
|
|
7045
7779
|
inputTokens: telemetry.inputTokens,
|
|
7046
7780
|
outputTokens: telemetry.outputTokens,
|
|
7047
7781
|
cachedTokens: telemetry.cachedTokens,
|
|
@@ -7055,7 +7789,6 @@ async function emitModelCallUsage(input) {
|
|
|
7055
7789
|
provider: input.provider,
|
|
7056
7790
|
providerApi: input.providerApi,
|
|
7057
7791
|
model: input.model,
|
|
7058
|
-
sourceKey: input.sourceKey,
|
|
7059
7792
|
rejectedFields: normalizedUsage.rejectedFields.join(",")
|
|
7060
7793
|
});
|
|
7061
7794
|
}
|
|
@@ -7074,20 +7807,27 @@ function applyCodexCacheTelemetry(observability, provider, normalizedUsage) {
|
|
|
7074
7807
|
promptTokens: normalizedUsage.telemetry.inputTokens
|
|
7075
7808
|
});
|
|
7076
7809
|
}
|
|
7077
|
-
function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens
|
|
7078
|
-
const
|
|
7079
|
-
|
|
7080
|
-
);
|
|
7081
|
-
if (sanitized.length <= persistedHistoryCount) {
|
|
7810
|
+
function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens) {
|
|
7811
|
+
const modelReady = sanitizeHistoryItemsForModel2(rawHistory, toolOutputTruncationTokens);
|
|
7812
|
+
if (modelReady.length <= persistedHistoryCount) {
|
|
7082
7813
|
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
7083
7814
|
}
|
|
7084
|
-
const rows =
|
|
7085
|
-
|
|
7086
|
-
item
|
|
7087
|
-
|
|
7815
|
+
const rows = [];
|
|
7816
|
+
for (const [offset, item] of modelReady.slice(persistedHistoryCount).entries()) {
|
|
7817
|
+
if (item.type === "message" && item.role === "system") {
|
|
7818
|
+
continue;
|
|
7819
|
+
}
|
|
7820
|
+
rows.push({
|
|
7821
|
+
position: nextPosition + rows.length,
|
|
7822
|
+
item: normalizeProtocolJsonValue(
|
|
7823
|
+
item,
|
|
7824
|
+
`$[${persistedHistoryCount + offset}]`
|
|
7825
|
+
)
|
|
7826
|
+
});
|
|
7827
|
+
}
|
|
7088
7828
|
return {
|
|
7089
7829
|
rows,
|
|
7090
|
-
nextWatermark:
|
|
7830
|
+
nextWatermark: modelReady.length,
|
|
7091
7831
|
nextPosition: nextPosition + rows.length
|
|
7092
7832
|
};
|
|
7093
7833
|
}
|
|
@@ -7111,12 +7851,6 @@ function selectRejectedProviderArtifactHistoryIds(activeHistory, candidates, wir
|
|
|
7111
7851
|
}
|
|
7112
7852
|
return selected;
|
|
7113
7853
|
}
|
|
7114
|
-
function secretRedactionModelInputFilter(redactValue) {
|
|
7115
|
-
return ({ modelData }) => ({
|
|
7116
|
-
...modelData,
|
|
7117
|
-
input: redactValue(modelData.input)
|
|
7118
|
-
});
|
|
7119
|
-
}
|
|
7120
7854
|
function isModelOrToolProgressHistoryItem(item) {
|
|
7121
7855
|
if (item.type === "message") {
|
|
7122
7856
|
return item.role === "assistant";
|
|
@@ -7456,6 +8190,24 @@ async function refreshCappedCodexUsageRows(db, settings, workspaceId, accounts,
|
|
|
7456
8190
|
function codexCredentialLeaseDeadlineExpired(confirmedUntilMs, nowMs = performance.now()) {
|
|
7457
8191
|
return confirmedUntilMs === null || !Number.isFinite(confirmedUntilMs) || confirmedUntilMs <= nowMs;
|
|
7458
8192
|
}
|
|
8193
|
+
async function finalizeDurableTurnOpStreams(sessions, fallback) {
|
|
8194
|
+
const candidates = /* @__PURE__ */ new Set();
|
|
8195
|
+
for (const session of sessions) {
|
|
8196
|
+
const candidate = session;
|
|
8197
|
+
if (typeof candidate?.finalizeOpStreamOps === "function") {
|
|
8198
|
+
candidates.add(candidate);
|
|
8199
|
+
}
|
|
8200
|
+
}
|
|
8201
|
+
if (candidates.size === 0 && fallback) {
|
|
8202
|
+
candidates.add(fallback);
|
|
8203
|
+
}
|
|
8204
|
+
for (const candidate of candidates) {
|
|
8205
|
+
try {
|
|
8206
|
+
await candidate.finalizeOpStreamOps();
|
|
8207
|
+
} catch {
|
|
8208
|
+
}
|
|
8209
|
+
}
|
|
8210
|
+
}
|
|
7459
8211
|
function createRunAgentTurnActivity(services) {
|
|
7460
8212
|
return async function runAgentTurn2(input) {
|
|
7461
8213
|
const {
|
|
@@ -7499,6 +8251,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
7499
8251
|
acknowledgeQuiescence = true;
|
|
7500
8252
|
noteCancellationRequested();
|
|
7501
8253
|
};
|
|
8254
|
+
const acknowledgeRecoveryQuiescence = () => {
|
|
8255
|
+
acknowledgeQuiescence = true;
|
|
8256
|
+
noteCancellationRequested();
|
|
8257
|
+
};
|
|
7502
8258
|
let turnId;
|
|
7503
8259
|
let triggerEventId;
|
|
7504
8260
|
const claimedResult = (result) => {
|
|
@@ -7600,7 +8356,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7600
8356
|
workspaceId: input.workspaceId,
|
|
7601
8357
|
turnId,
|
|
7602
8358
|
reason,
|
|
7603
|
-
|
|
8359
|
+
...safeErrorDiagnostic(error)
|
|
7604
8360
|
});
|
|
7605
8361
|
observability.incrementCounter({
|
|
7606
8362
|
name: "opengeni_codex_lease_renewals_total",
|
|
@@ -7631,6 +8387,12 @@ function createRunAgentTurnActivity(services) {
|
|
|
7631
8387
|
let lazyOwnedSandbox = null;
|
|
7632
8388
|
let turnSandboxProvisioner = null;
|
|
7633
8389
|
let setupBoxSession = null;
|
|
8390
|
+
const finalizeTurnOpStreamOps = async () => {
|
|
8391
|
+
await finalizeDurableTurnOpStreams(
|
|
8392
|
+
[lazyOwnedSandbox?.session, resolvedSandbox?.established.session],
|
|
8393
|
+
machinePrimarySession
|
|
8394
|
+
);
|
|
8395
|
+
};
|
|
7634
8396
|
const onHomeSandboxRebound = (rebound) => {
|
|
7635
8397
|
const current = resolvedSandbox;
|
|
7636
8398
|
const previousSession = current?.established.session;
|
|
@@ -7861,7 +8623,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7861
8623
|
sessionId: input.sessionId,
|
|
7862
8624
|
sandboxGroupId: heartbeatGroupId,
|
|
7863
8625
|
leaseEpoch: heartbeatEpoch,
|
|
7864
|
-
|
|
8626
|
+
...safeErrorDiagnostic(error)
|
|
7865
8627
|
});
|
|
7866
8628
|
}).finally(() => {
|
|
7867
8629
|
rotationInFlight = null;
|
|
@@ -7969,6 +8731,28 @@ function createRunAgentTurnActivity(services) {
|
|
|
7969
8731
|
}
|
|
7970
8732
|
};
|
|
7971
8733
|
let persistedHistoryCount = 0;
|
|
8734
|
+
const retainedScreenshotReceiptsByCallId = /* @__PURE__ */ new Map();
|
|
8735
|
+
const computerScreenshotCallIds = /* @__PURE__ */ new Set();
|
|
8736
|
+
const materializeScreenshotHistory = async (history) => {
|
|
8737
|
+
collectRetainedScreenshotReceipts(history, retainedScreenshotReceiptsByCallId);
|
|
8738
|
+
return await materializeRetainedScreenshotHistory({
|
|
8739
|
+
db,
|
|
8740
|
+
objectStorage,
|
|
8741
|
+
workspaceId: input.workspaceId,
|
|
8742
|
+
sessionId: input.sessionId,
|
|
8743
|
+
history
|
|
8744
|
+
});
|
|
8745
|
+
};
|
|
8746
|
+
const materializeScreenshotRunState = async (serialized) => {
|
|
8747
|
+
collectRetainedScreenshotRunStateReceipts(serialized, retainedScreenshotReceiptsByCallId);
|
|
8748
|
+
return await materializeRetainedScreenshotRunState({
|
|
8749
|
+
db,
|
|
8750
|
+
objectStorage,
|
|
8751
|
+
workspaceId: input.workspaceId,
|
|
8752
|
+
sessionId: input.sessionId,
|
|
8753
|
+
serialized
|
|
8754
|
+
});
|
|
8755
|
+
};
|
|
7972
8756
|
let providerArtifactCandidates = {
|
|
7973
8757
|
knownHistoryItemIds: [],
|
|
7974
8758
|
historyItemIds: []
|
|
@@ -7978,36 +8762,42 @@ function createRunAgentTurnActivity(services) {
|
|
|
7978
8762
|
if (!stream || !turnId) {
|
|
7979
8763
|
return;
|
|
7980
8764
|
}
|
|
8765
|
+
const durableTurnId = turnId;
|
|
7981
8766
|
try {
|
|
7982
8767
|
const rawHistory = stream.state.history;
|
|
7983
8768
|
if (Array.isArray(rawHistory)) {
|
|
7984
|
-
const
|
|
8769
|
+
const durableHistory = compactRetainedScreenshotHistory(
|
|
7985
8770
|
rawHistory,
|
|
8771
|
+
retainedScreenshotReceiptsByCallId
|
|
8772
|
+
);
|
|
8773
|
+
const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
|
|
8774
|
+
durableHistory,
|
|
7986
8775
|
persistedHistoryCount,
|
|
7987
8776
|
nextHistoryPosition,
|
|
7988
|
-
modelRunSettings.modelToolOutputTruncationTokens
|
|
7989
|
-
(value) => redact(value)
|
|
8777
|
+
modelRunSettings.modelToolOutputTruncationTokens
|
|
7990
8778
|
);
|
|
7991
8779
|
const hasModelOrToolProgress = rows.some(
|
|
7992
8780
|
(row) => isModelOrToolProgressHistoryItem(row.item)
|
|
7993
8781
|
);
|
|
7994
8782
|
const shouldAppendRows = rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
|
|
7995
8783
|
if (shouldAppendRows) {
|
|
7996
|
-
|
|
7997
|
-
|
|
7998
|
-
|
|
7999
|
-
|
|
8000
|
-
|
|
8001
|
-
|
|
8002
|
-
|
|
8003
|
-
|
|
8004
|
-
|
|
8784
|
+
await runMandatoryHistoryPersistenceStep("history_append", async () => {
|
|
8785
|
+
const appended = await appendSessionHistoryItems(db, {
|
|
8786
|
+
accountId: input.accountId,
|
|
8787
|
+
workspaceId: input.workspaceId,
|
|
8788
|
+
sessionId: input.sessionId,
|
|
8789
|
+
turnId: durableTurnId,
|
|
8790
|
+
expectedExecutionGeneration: executionGeneration,
|
|
8791
|
+
expectedAttemptId: input.attemptId,
|
|
8792
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
8793
|
+
items: rows
|
|
8794
|
+
});
|
|
8795
|
+
if (!appended) {
|
|
8796
|
+
throw new TurnAttemptFencedError(
|
|
8797
|
+
"turn execution generation was fenced while saving conversation history"
|
|
8798
|
+
);
|
|
8799
|
+
}
|
|
8005
8800
|
});
|
|
8006
|
-
if (!appended) {
|
|
8007
|
-
throw new TurnAttemptFencedError(
|
|
8008
|
-
"turn execution generation was fenced while saving conversation history"
|
|
8009
|
-
);
|
|
8010
|
-
}
|
|
8011
8801
|
}
|
|
8012
8802
|
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
8013
8803
|
persistedHistoryCount = nextWatermark;
|
|
@@ -8016,32 +8806,25 @@ function createRunAgentTurnActivity(services) {
|
|
|
8016
8806
|
}
|
|
8017
8807
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
8018
8808
|
if (envelope) {
|
|
8019
|
-
await
|
|
8020
|
-
|
|
8021
|
-
|
|
8022
|
-
|
|
8023
|
-
|
|
8024
|
-
|
|
8809
|
+
await runMandatoryHistoryPersistenceStep(
|
|
8810
|
+
"sandbox_envelope",
|
|
8811
|
+
() => upsertSandboxSessionEnvelope(db, {
|
|
8812
|
+
accountId: input.accountId,
|
|
8813
|
+
workspaceId: input.workspaceId,
|
|
8814
|
+
sessionId: input.sessionId,
|
|
8815
|
+
envelope
|
|
8816
|
+
})
|
|
8817
|
+
);
|
|
8025
8818
|
}
|
|
8026
8819
|
} catch (persistError) {
|
|
8027
8820
|
console.error(
|
|
8028
8821
|
"session history dual-write failed (run unaffected)",
|
|
8029
|
-
safeErrorDiagnostic(persistError
|
|
8822
|
+
safeErrorDiagnostic(persistError)
|
|
8030
8823
|
);
|
|
8031
8824
|
if (options.requireDurable) throw persistError;
|
|
8032
8825
|
}
|
|
8033
8826
|
};
|
|
8034
|
-
let redact = createSecretRedactor([]);
|
|
8035
|
-
const secretRedactions = /* @__PURE__ */ new Map();
|
|
8036
8827
|
const publishedRunCredentialNotices = /* @__PURE__ */ new Set();
|
|
8037
|
-
const registerSecretRedactions = (secrets) => {
|
|
8038
|
-
for (const secret of secrets) {
|
|
8039
|
-
if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
|
|
8040
|
-
}
|
|
8041
|
-
redact = createSecretRedactor(
|
|
8042
|
-
[...secretRedactions].map(([value, name]) => ({ name, value }))
|
|
8043
|
-
);
|
|
8044
|
-
};
|
|
8045
8828
|
let variableSetId = "";
|
|
8046
8829
|
let rigId = "";
|
|
8047
8830
|
let rigVersionId = "";
|
|
@@ -8173,7 +8956,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8173
8956
|
publish = async (events, immediate = false) => {
|
|
8174
8957
|
const inputs = events.map((event) => ({
|
|
8175
8958
|
...event,
|
|
8176
|
-
payload:
|
|
8959
|
+
payload: event.payload,
|
|
8177
8960
|
turnId,
|
|
8178
8961
|
producerId,
|
|
8179
8962
|
producerSeq: ++producerSeq
|
|
@@ -8186,7 +8969,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
8186
8969
|
turnId,
|
|
8187
8970
|
executionGeneration,
|
|
8188
8971
|
input.attemptId,
|
|
8189
|
-
inputs
|
|
8972
|
+
inputs,
|
|
8973
|
+
{
|
|
8974
|
+
onAppend: ({ durationSeconds }) => recordSessionEventAppendLatency(observability, { durationSeconds }),
|
|
8975
|
+
onPublish: ({ durationSeconds }) => recordSessionEventPublishLatency(observability, { durationSeconds })
|
|
8976
|
+
}
|
|
8190
8977
|
);
|
|
8191
8978
|
if (inputs.length > 0 && !appended.accepted) {
|
|
8192
8979
|
throw new TurnAttemptFencedError("turn execution generation was fenced");
|
|
@@ -8231,12 +9018,18 @@ function createRunAgentTurnActivity(services) {
|
|
|
8231
9018
|
} : void 0;
|
|
8232
9019
|
const inputs = inputSettlement.events.map((event) => ({
|
|
8233
9020
|
...event,
|
|
8234
|
-
payload:
|
|
9021
|
+
payload: event.payload,
|
|
8235
9022
|
turnId,
|
|
8236
9023
|
producerId,
|
|
8237
9024
|
producerSeq: ++producerSeq
|
|
8238
9025
|
}));
|
|
8239
|
-
const
|
|
9026
|
+
const runState = inputSettlement.runState ? {
|
|
9027
|
+
...inputSettlement.runState,
|
|
9028
|
+
serializedRunState: compactRetainedScreenshotRunState(
|
|
9029
|
+
inputSettlement.runState.serializedRunState,
|
|
9030
|
+
retainedScreenshotReceiptsByCallId
|
|
9031
|
+
)
|
|
9032
|
+
} : void 0;
|
|
8240
9033
|
const result = await applySessionTurnSettlement(db, input.workspaceId, {
|
|
8241
9034
|
sessionId: input.sessionId,
|
|
8242
9035
|
turnId,
|
|
@@ -8246,7 +9039,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8246
9039
|
sessionStatus: inputSettlement.sessionStatus,
|
|
8247
9040
|
activeTurnId: inputSettlement.activeTurnId,
|
|
8248
9041
|
events: inputs,
|
|
8249
|
-
...
|
|
9042
|
+
...runState ? { runState } : {},
|
|
8250
9043
|
...recordingMutation ? { recording: recordingMutation } : {},
|
|
8251
9044
|
...compactionRequestFailure ? { compactionRequestFailure } : {}
|
|
8252
9045
|
});
|
|
@@ -8538,10 +9331,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
8538
9331
|
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult)
|
|
8539
9332
|
});
|
|
8540
9333
|
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
8541
|
-
workspaceId: input.workspaceId,
|
|
8542
9334
|
stage: shadowResult.stage,
|
|
8543
9335
|
reason: shadowResult.reason,
|
|
8544
|
-
|
|
9336
|
+
errorClass: "CodexFleetShadowOperationError",
|
|
9337
|
+
errorCode: "codex_fleet_shadow_failed",
|
|
9338
|
+
origin: "worker",
|
|
8545
9339
|
payloadBytes: shadowResult.payloadBytes
|
|
8546
9340
|
});
|
|
8547
9341
|
}
|
|
@@ -8829,14 +9623,18 @@ function createRunAgentTurnActivity(services) {
|
|
|
8829
9623
|
attemptId: input.attemptId,
|
|
8830
9624
|
executionGeneration: turn.executionGeneration
|
|
8831
9625
|
};
|
|
8832
|
-
const [
|
|
8833
|
-
|
|
9626
|
+
const [workspace, instructionPolicySnapshot, preferenceSnapshot] = await Promise.all([
|
|
9627
|
+
getWorkspace2(db, input.workspaceId),
|
|
8834
9628
|
getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
|
|
8835
9629
|
getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
|
|
8836
9630
|
if (error instanceof PreferenceRegistryInitiatorError) return null;
|
|
8837
9631
|
throw error;
|
|
8838
9632
|
})
|
|
8839
9633
|
]);
|
|
9634
|
+
if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
9635
|
+
const workspaceAgentInstructions = workspace.agentInstructions;
|
|
9636
|
+
const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
|
|
9637
|
+
assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
|
|
8840
9638
|
const workspaceGovernance = renderWorkspaceGovernanceContext({
|
|
8841
9639
|
instructionPolicy: instructionPolicySnapshot,
|
|
8842
9640
|
preferences: preferenceSnapshot
|
|
@@ -8860,31 +9658,13 @@ function createRunAgentTurnActivity(services) {
|
|
|
8860
9658
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
8861
9659
|
sandboxBackend: turn.sandboxBackend
|
|
8862
9660
|
};
|
|
8863
|
-
const sessionMcpHeaderNames = /* @__PURE__ */ new Map();
|
|
8864
9661
|
const runSettings = await settingsWithSessionMcpServersForRun(
|
|
8865
9662
|
db,
|
|
8866
9663
|
input.workspaceId,
|
|
8867
9664
|
input.sessionId,
|
|
8868
9665
|
input.attemptId,
|
|
8869
|
-
baseRunSettings
|
|
8870
|
-
{
|
|
8871
|
-
onResolvedServers: (servers) => {
|
|
8872
|
-
for (const server of servers) {
|
|
8873
|
-
sessionMcpHeaderNames.set(server.id, server.headerNames);
|
|
8874
|
-
}
|
|
8875
|
-
}
|
|
8876
|
-
}
|
|
9666
|
+
baseRunSettings
|
|
8877
9667
|
);
|
|
8878
|
-
registerSecretRedactions([
|
|
8879
|
-
...runSettings.accessKey ? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }] : [],
|
|
8880
|
-
...runSettings.mcpServers.flatMap((server) => {
|
|
8881
|
-
return headerSecretRedactions(
|
|
8882
|
-
`MCP_${server.id.toUpperCase()}_STATIC`,
|
|
8883
|
-
server.headers,
|
|
8884
|
-
sessionMcpHeaderNames.get(server.id)
|
|
8885
|
-
);
|
|
8886
|
-
})
|
|
8887
|
-
]);
|
|
8888
9668
|
const resolvedModel = runtime.resolveTurnModel(
|
|
8889
9669
|
capabilitySettings,
|
|
8890
9670
|
turnExecutionPolicy.productModelId
|
|
@@ -8900,38 +9680,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
8900
9680
|
);
|
|
8901
9681
|
const modelHistoryProjector = async (items) => projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
|
|
8902
9682
|
const compactionModelHistoryProjector = async (items) => await modelHistoryProjector(projectHistoryForProvider2(items, providerApi));
|
|
8903
|
-
const selectedProvider = resolvedModel?.provider;
|
|
8904
|
-
const publicProviderHeaders = new Set(
|
|
8905
|
-
selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? []
|
|
8906
|
-
);
|
|
8907
|
-
const publicProviderQueries = new Set(
|
|
8908
|
-
selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? []
|
|
8909
|
-
);
|
|
8910
|
-
const fallbackProviderApiKey = capabilitySettings.openaiProvider === "azure" ? capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken : capabilitySettings.openaiApiKey;
|
|
8911
|
-
registerSecretRedactions([
|
|
8912
|
-
...selectedProvider?.apiKey ? [
|
|
8913
|
-
{
|
|
8914
|
-
name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
|
|
8915
|
-
value: selectedProvider.apiKey
|
|
8916
|
-
}
|
|
8917
|
-
] : !resolvedModel && fallbackProviderApiKey ? [
|
|
8918
|
-
{
|
|
8919
|
-
name: "MODEL_PROVIDER_API_KEY",
|
|
8920
|
-
value: fallbackProviderApiKey
|
|
8921
|
-
}
|
|
8922
|
-
] : [],
|
|
8923
|
-
...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(
|
|
8924
|
-
([name, value]) => publicProviderHeaders.has(name.toLowerCase()) ? [] : [
|
|
8925
|
-
{
|
|
8926
|
-
name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`,
|
|
8927
|
-
value
|
|
8928
|
-
}
|
|
8929
|
-
]
|
|
8930
|
-
),
|
|
8931
|
-
...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(
|
|
8932
|
-
([name, value]) => publicProviderQueries.has(name.toLowerCase()) ? [] : [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }]
|
|
8933
|
-
)
|
|
8934
|
-
]);
|
|
8935
9683
|
modelRunSettings = resolvedModel ? settingsWithResolvedModelContext(runSettings, resolvedModel.configured) : runSettings;
|
|
8936
9684
|
{
|
|
8937
9685
|
const workspaceModelPolicy = await getWorkspaceModelPolicy2(db, input.workspaceId);
|
|
@@ -8957,13 +9705,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
8957
9705
|
input.workspaceId,
|
|
8958
9706
|
effectiveCodexCredentialId ?? ""
|
|
8959
9707
|
);
|
|
8960
|
-
const registerSnapshot = async (snapshotPromise) => {
|
|
8961
|
-
const snapshot = await snapshotPromise;
|
|
8962
|
-
registerSecretRedactions([
|
|
8963
|
-
{ name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken }
|
|
8964
|
-
]);
|
|
8965
|
-
return snapshot;
|
|
8966
|
-
};
|
|
8967
9708
|
return {
|
|
8968
9709
|
clientVersion: CODEX_CLIENT_VERSION,
|
|
8969
9710
|
// Backend sticky cache-routing key — the SAME id as the body's
|
|
@@ -8973,8 +9714,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
8973
9714
|
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
8974
9715
|
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
8975
9716
|
sessionId: input.sessionId,
|
|
8976
|
-
getToken: () =>
|
|
8977
|
-
refresh: () =>
|
|
9717
|
+
getToken: () => resolver.getToken(),
|
|
9718
|
+
refresh: () => resolver.refresh(),
|
|
8978
9719
|
resolveModel: buildModelResolver(
|
|
8979
9720
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
8980
9721
|
CODEX_FALLBACK_MODEL_SLUGS[0]
|
|
@@ -9174,6 +9915,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9174
9915
|
force: true,
|
|
9175
9916
|
clearRequestedCompaction: true,
|
|
9176
9917
|
trigger: "operator",
|
|
9918
|
+
materializeHistory: materializeScreenshotHistory,
|
|
9177
9919
|
projectModelInput: compactionModelHistoryProjector,
|
|
9178
9920
|
...compactionModeOptions
|
|
9179
9921
|
}
|
|
@@ -9200,7 +9942,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9200
9942
|
}
|
|
9201
9943
|
);
|
|
9202
9944
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
9203
|
-
const errorMessage = String(
|
|
9945
|
+
const errorMessage = String(compactionFailureReasonFromError(error));
|
|
9204
9946
|
if (!await settle({
|
|
9205
9947
|
events: [
|
|
9206
9948
|
{
|
|
@@ -9306,9 +10048,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9306
10048
|
rigDefaultEnvironmentValues,
|
|
9307
10049
|
workspaceVariableSet?.values ?? {}
|
|
9308
10050
|
);
|
|
9309
|
-
registerSecretRedactions(
|
|
9310
|
-
Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value }))
|
|
9311
|
-
);
|
|
9312
10051
|
const routingOn = routingEnabled(settings);
|
|
9313
10052
|
let activeSandboxPointer = routingOn ? await readActiveSandbox2(db, input.workspaceId, input.sessionId).catch(() => null) : null;
|
|
9314
10053
|
let activeSandboxRecord = null;
|
|
@@ -9365,7 +10104,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9365
10104
|
void 0
|
|
9366
10105
|
) : null;
|
|
9367
10106
|
if (initialRunCredentialMaterial) {
|
|
9368
|
-
registerSecretRedactions(initialRunCredentialMaterial.redactions);
|
|
9369
10107
|
for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
|
|
9370
10108
|
publishedRunCredentialNotices.add(JSON.stringify(payload));
|
|
9371
10109
|
await publish([{ type: "credential.auth_needed", payload }], true);
|
|
@@ -9378,7 +10116,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
9378
10116
|
cancellationSignal,
|
|
9379
10117
|
void 0
|
|
9380
10118
|
);
|
|
9381
|
-
const establishPolicy = lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none"
|
|
10119
|
+
const establishPolicy = lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none" && // Resolved run credentials must be written to one exact leased sandbox
|
|
10120
|
+
// before agent execution. A warm active pointer can bypass the lazy
|
|
10121
|
+
// provisioner entirely, which previously skipped materialization.
|
|
10122
|
+
!initialRunCredentialMaterial ? "on-demand" : "eager";
|
|
9382
10123
|
if (activeSandboxBackend !== "selfhosted") {
|
|
9383
10124
|
await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
|
|
9384
10125
|
}
|
|
@@ -9423,6 +10164,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9423
10164
|
sandboxWorkspaceEnvironmentValues,
|
|
9424
10165
|
{
|
|
9425
10166
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
10167
|
+
skipToolspace: activeSandboxBackend === "selfhosted",
|
|
9426
10168
|
deferGitHubToken: activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
|
|
9427
10169
|
scope: connectionScope,
|
|
9428
10170
|
...gitCredentialAuthority ? { authority: gitCredentialAuthority } : {},
|
|
@@ -9434,17 +10176,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9434
10176
|
cancellationSignal,
|
|
9435
10177
|
void 0
|
|
9436
10178
|
);
|
|
9437
|
-
registerSecretRedactions([
|
|
9438
|
-
...Object.entries(sandboxGitTokens ?? {}).flatMap(
|
|
9439
|
-
([provider, value]) => value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : []
|
|
9440
|
-
),
|
|
9441
|
-
...sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : [],
|
|
9442
|
-
...(sandboxGitCredentialBindings ?? []).map((binding) => ({
|
|
9443
|
-
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
9444
|
-
value: binding.token
|
|
9445
|
-
})),
|
|
9446
|
-
...sandboxToolspaceToken ? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }] : []
|
|
9447
|
-
]);
|
|
9448
10179
|
const sandboxToolspaceTokenFile = sandboxToolspaceToken ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId) : void 0;
|
|
9449
10180
|
const initialGitCredentials = sandboxGitCredentialBindings ? {
|
|
9450
10181
|
bindings: sandboxGitCredentialBindings,
|
|
@@ -9477,12 +10208,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9477
10208
|
);
|
|
9478
10209
|
if (binding) {
|
|
9479
10210
|
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
9480
|
-
registerSecretRedactions([
|
|
9481
|
-
{
|
|
9482
|
-
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
9483
|
-
value: binding.token
|
|
9484
|
-
}
|
|
9485
|
-
]);
|
|
9486
10211
|
}
|
|
9487
10212
|
pendingBinding = binding;
|
|
9488
10213
|
return binding ? {
|
|
@@ -9559,7 +10284,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9559
10284
|
toolspaceAuthority
|
|
9560
10285
|
);
|
|
9561
10286
|
if (material) {
|
|
9562
|
-
registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
|
|
9563
10287
|
}
|
|
9564
10288
|
return material;
|
|
9565
10289
|
};
|
|
@@ -9667,7 +10391,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9667
10391
|
);
|
|
9668
10392
|
return;
|
|
9669
10393
|
}
|
|
9670
|
-
registerSecretRedactions(material.redactions);
|
|
9671
10394
|
await runWorkspaceMutationForSandbox(
|
|
9672
10395
|
requireTargetSandbox(),
|
|
9673
10396
|
"runCredentialMaterialization",
|
|
@@ -9742,7 +10465,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9742
10465
|
sandboxHolderId = managedOwnership?.holderId ?? null;
|
|
9743
10466
|
sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
|
|
9744
10467
|
if (machinePrimary) {
|
|
9745
|
-
const machineOpStream = settings.agentOpStreamEnabled === true ? (await
|
|
10468
|
+
const machineOpStream = settings.agentOpStreamEnabled === true ? (await getEnrollment2(db, input.workspaceId, activeSandboxRecord.enrollmentId))?.opStream === true : false;
|
|
9746
10469
|
const established = await establishSelfhostedTurnSession(
|
|
9747
10470
|
{
|
|
9748
10471
|
db,
|
|
@@ -9774,6 +10497,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9774
10497
|
db,
|
|
9775
10498
|
settings,
|
|
9776
10499
|
bus,
|
|
10500
|
+
opJournal,
|
|
9777
10501
|
onOp: machineOpObserver.observer,
|
|
9778
10502
|
onSandboxOperation: sandboxOperationObserver,
|
|
9779
10503
|
onHomeSandboxRebound
|
|
@@ -9864,6 +10588,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9864
10588
|
db,
|
|
9865
10589
|
settings,
|
|
9866
10590
|
bus,
|
|
10591
|
+
opJournal,
|
|
9867
10592
|
onSandboxOperation: sandboxOperationObserver,
|
|
9868
10593
|
onHomeSandboxLost: publishSandboxLost,
|
|
9869
10594
|
onHomeSandboxRebound
|
|
@@ -9942,9 +10667,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9942
10667
|
const resolveCredential = async (request) => {
|
|
9943
10668
|
const result = await resolveFrozenCredential(request);
|
|
9944
10669
|
if (result.status === "ok") {
|
|
9945
|
-
registerSecretRedactions(
|
|
9946
|
-
headerSecretRedactions("MCP", result.headers, Object.keys(result.headers))
|
|
9947
|
-
);
|
|
9948
10670
|
}
|
|
9949
10671
|
return result;
|
|
9950
10672
|
};
|
|
@@ -9960,9 +10682,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9960
10682
|
clientVersion: CODEX_CLIENT_VERSION,
|
|
9961
10683
|
withAuthorization: async (use) => {
|
|
9962
10684
|
const snapshot = await resolver.getToken();
|
|
9963
|
-
registerSecretRedactions([
|
|
9964
|
-
{ name: "CODEX_APPS_ACCESS_TOKEN", value: snapshot.accessToken }
|
|
9965
|
-
]);
|
|
9966
10685
|
return await withCodexAppsRequestAuthorization(
|
|
9967
10686
|
db,
|
|
9968
10687
|
{ workspaceId: input.workspaceId, credentialId: codexAppsCredentialId },
|
|
@@ -10042,6 +10761,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10042
10761
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
10043
10762
|
...serviceTier ? { serviceTier } : {},
|
|
10044
10763
|
...humanInputResume ? { humanInputResponse: humanInputResume } : {},
|
|
10764
|
+
humanInputEnabled: agentHumanInputEnabled,
|
|
10045
10765
|
genesisTitleHint: isGenesisTurn,
|
|
10046
10766
|
persistentSessionSettings: {
|
|
10047
10767
|
titleIsSet: Boolean(session.title?.trim())
|
|
@@ -10060,14 +10780,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
10060
10780
|
...activeSandboxBackend !== "selfhosted" && sandboxGitTokens ? { gitTokenSeeds: sandboxGitTokens } : {},
|
|
10061
10781
|
...activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings ? { gitCredentialBindings: sandboxGitCredentialBindings } : {},
|
|
10062
10782
|
...activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
|
|
10063
|
-
// Toolspace is
|
|
10064
|
-
//
|
|
10065
|
-
//
|
|
10066
|
-
// machine's ONLY path to programmatic tool calling and grants no more
|
|
10067
|
-
// than toolspace:call for its own session (own-session-bound,
|
|
10068
|
-
// short-lived and renewable, per-active-turn budgeted, approval-tools
|
|
10069
|
-
// excluded). The runtime seeds it to the box's token file over the same
|
|
10070
|
-
// exec channel, off-manifest, on every backend.
|
|
10783
|
+
// Toolspace delivery is managed-sandbox-only. Connected Machines own any
|
|
10784
|
+
// manually configured API credentials and must never be contacted during
|
|
10785
|
+
// turn admission merely to seed OpenGeni tooling.
|
|
10071
10786
|
...sandboxToolspaceToken ? {
|
|
10072
10787
|
toolspaceTokenSeed: sandboxToolspaceToken,
|
|
10073
10788
|
toolspaceTokenSessionId: input.sessionId
|
|
@@ -10327,6 +11042,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10327
11042
|
db,
|
|
10328
11043
|
settings,
|
|
10329
11044
|
bus,
|
|
11045
|
+
opJournal,
|
|
10330
11046
|
onOp: machineOpObserver.observer,
|
|
10331
11047
|
onSandboxOperation: sandboxOperationObserver,
|
|
10332
11048
|
onHomeSandboxLost: publishSandboxLost,
|
|
@@ -10417,7 +11133,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10417
11133
|
}
|
|
10418
11134
|
);
|
|
10419
11135
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
10420
|
-
const errorMessage = String(
|
|
11136
|
+
const errorMessage = String(compactionFailureReasonFromError(error));
|
|
10421
11137
|
if (!await settle({
|
|
10422
11138
|
events: [
|
|
10423
11139
|
{
|
|
@@ -10502,10 +11218,12 @@ function createRunAgentTurnActivity(services) {
|
|
|
10502
11218
|
force: true,
|
|
10503
11219
|
clearRequestedCompaction: true,
|
|
10504
11220
|
trigger: "operator",
|
|
11221
|
+
materializeHistory: materializeScreenshotHistory,
|
|
10505
11222
|
projectModelInput: compactionModelHistoryProjector,
|
|
10506
11223
|
...compactionModeOptions
|
|
10507
11224
|
} : {
|
|
10508
11225
|
trigger: "auto",
|
|
11226
|
+
materializeHistory: materializeScreenshotHistory,
|
|
10509
11227
|
projectModelInput: compactionModelHistoryProjector,
|
|
10510
11228
|
...compactionModeOptions
|
|
10511
11229
|
}
|
|
@@ -10539,11 +11257,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
10539
11257
|
}
|
|
10540
11258
|
);
|
|
10541
11259
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
10542
|
-
const errorMessage = String(
|
|
11260
|
+
const errorMessage = String(compactionFailureReasonFromError(compactError));
|
|
10543
11261
|
observability.error("context compaction failed", {
|
|
10544
11262
|
sessionId: input.sessionId,
|
|
10545
11263
|
turnId,
|
|
10546
|
-
|
|
11264
|
+
...safeErrorDiagnostic(compactError)
|
|
10547
11265
|
});
|
|
10548
11266
|
if (!await settle({
|
|
10549
11267
|
events: [
|
|
@@ -10640,6 +11358,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
10640
11358
|
...unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {},
|
|
10641
11359
|
...runCredentialsNote ? { runCredentialsNote } : {},
|
|
10642
11360
|
providerApi,
|
|
11361
|
+
materializeModelHistory: materializeScreenshotHistory,
|
|
11362
|
+
materializeSerializedRunState: materializeScreenshotRunState,
|
|
10643
11363
|
projectModelHistory: modelHistoryProjector
|
|
10644
11364
|
});
|
|
10645
11365
|
runInput = prepared.input;
|
|
@@ -10675,6 +11395,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10675
11395
|
force: true,
|
|
10676
11396
|
...triggerLabel === "operator" ? { clearRequestedCompaction: true } : {},
|
|
10677
11397
|
trigger: triggerLabel,
|
|
11398
|
+
materializeHistory: materializeScreenshotHistory,
|
|
10678
11399
|
projectModelInput: compactionModelHistoryProjector,
|
|
10679
11400
|
...compactionModeOptions
|
|
10680
11401
|
}
|
|
@@ -10706,13 +11427,18 @@ function createRunAgentTurnActivity(services) {
|
|
|
10706
11427
|
const runStreamOnce = async () => {
|
|
10707
11428
|
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
10708
11429
|
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
11430
|
+
await attachRunCredentialRenewal(
|
|
11431
|
+
eagerSetupSession,
|
|
11432
|
+
resolvedSandbox
|
|
11433
|
+
);
|
|
11434
|
+
const eagerCredentialSetupSession = initialRunCredentialMaterial ? withRunCredentialsSession(eagerSetupSession, input.sessionId) : eagerSetupSession;
|
|
10709
11435
|
await runWorkspaceMutationForSandbox(
|
|
10710
11436
|
resolvedSandbox,
|
|
10711
11437
|
"eagerOwnedSandboxSetup",
|
|
10712
11438
|
async () => await runOwnedSandboxSetup(
|
|
10713
11439
|
agent,
|
|
10714
11440
|
ownedEstablished.session,
|
|
10715
|
-
|
|
11441
|
+
eagerCredentialSetupSession,
|
|
10716
11442
|
{
|
|
10717
11443
|
settings: runSettings,
|
|
10718
11444
|
environment: sandboxEnvironment,
|
|
@@ -10770,7 +11496,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10770
11496
|
} : {},
|
|
10771
11497
|
...sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox ? {
|
|
10772
11498
|
onToolspaceTokenSessionReady: async (tokenSession) => {
|
|
10773
|
-
const renewalSession =
|
|
11499
|
+
const renewalSession = setupBoxSession ?? tokenSession;
|
|
10774
11500
|
await attachToolspaceTokenRenewal(renewalSession);
|
|
10775
11501
|
}
|
|
10776
11502
|
} : {},
|
|
@@ -10785,7 +11511,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
10785
11511
|
} : {},
|
|
10786
11512
|
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
10787
11513
|
contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
10788
|
-
callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
|
|
10789
11514
|
...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
|
|
10790
11515
|
});
|
|
10791
11516
|
};
|
|
@@ -10816,6 +11541,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10816
11541
|
}
|
|
10817
11542
|
let stableToolCallIdsToClear = null;
|
|
10818
11543
|
let completedCurrentToolBatch = false;
|
|
11544
|
+
let retainedScreenshotMetadata = null;
|
|
10819
11545
|
const responseResult = await processModelResponseTerminalEvent({
|
|
10820
11546
|
event: next.value,
|
|
10821
11547
|
state: modelResponseState,
|
|
@@ -10866,7 +11592,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10866
11592
|
} catch (limitError) {
|
|
10867
11593
|
let serializedRunState = null;
|
|
10868
11594
|
try {
|
|
10869
|
-
serializedRunState = String(
|
|
11595
|
+
serializedRunState = String(stream.state.toString());
|
|
10870
11596
|
} catch {
|
|
10871
11597
|
serializedRunState = null;
|
|
10872
11598
|
}
|
|
@@ -10888,7 +11614,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10888
11614
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
10889
11615
|
callId: pendingToolCall.callId,
|
|
10890
11616
|
callType: pendingToolCall.callType,
|
|
10891
|
-
callItem:
|
|
11617
|
+
callItem: pendingToolCall.callItem
|
|
10892
11618
|
});
|
|
10893
11619
|
if (!registered.accepted) {
|
|
10894
11620
|
throw new TurnAttemptFencedError(
|
|
@@ -10896,9 +11622,33 @@ function createRunAgentTurnActivity(services) {
|
|
|
10896
11622
|
);
|
|
10897
11623
|
}
|
|
10898
11624
|
currentToolBatchCallIds.add(pendingToolCall.callId);
|
|
11625
|
+
if (pendingToolCall.callName === "computer_screenshot") {
|
|
11626
|
+
computerScreenshotCallIds.add(pendingToolCall.callId);
|
|
11627
|
+
}
|
|
10899
11628
|
}
|
|
10900
11629
|
const completedToolCall = completedToolCallFromSdkEvent(next.value);
|
|
10901
11630
|
if (completedToolCall) {
|
|
11631
|
+
const typedScreenshot = typedScreenshotFromSdkEvent(next.value);
|
|
11632
|
+
if (typedScreenshot && typedScreenshot.callId === completedToolCall.callId && computerScreenshotCallIds.has(completedToolCall.callId)) {
|
|
11633
|
+
retainedScreenshotMetadata = await retainComputerScreenshot({
|
|
11634
|
+
db,
|
|
11635
|
+
objectStorage,
|
|
11636
|
+
accountId: input.accountId,
|
|
11637
|
+
workspaceId: input.workspaceId,
|
|
11638
|
+
sessionId: input.sessionId,
|
|
11639
|
+
turnId: activeTurnId,
|
|
11640
|
+
attemptId: input.attemptId,
|
|
11641
|
+
output: typedScreenshot
|
|
11642
|
+
});
|
|
11643
|
+
retainedScreenshotReceiptsByCallId.set(
|
|
11644
|
+
completedToolCall.callId,
|
|
11645
|
+
retainedScreenshotMetadata
|
|
11646
|
+
);
|
|
11647
|
+
}
|
|
11648
|
+
const durableResultItem = retainedScreenshotMetadata ? compactRetainedScreenshotHistory(
|
|
11649
|
+
[completedToolCall.resultItem],
|
|
11650
|
+
retainedScreenshotReceiptsByCallId
|
|
11651
|
+
)[0] ?? completedToolCall.resultItem : completedToolCall.resultItem;
|
|
10902
11652
|
const recorded = await recordPendingSessionToolCallResult(db, {
|
|
10903
11653
|
accountId: input.accountId,
|
|
10904
11654
|
workspaceId: input.workspaceId,
|
|
@@ -10908,7 +11658,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10908
11658
|
attemptId: input.attemptId,
|
|
10909
11659
|
callId: completedToolCall.callId,
|
|
10910
11660
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
10911
|
-
resultItem:
|
|
11661
|
+
resultItem: durableResultItem
|
|
10912
11662
|
});
|
|
10913
11663
|
if (!recorded.accepted) {
|
|
10914
11664
|
throw new TurnAttemptFencedError(
|
|
@@ -10927,7 +11677,16 @@ function createRunAgentTurnActivity(services) {
|
|
|
10927
11677
|
completedCurrentToolBatch = currentBatchIsStable;
|
|
10928
11678
|
}
|
|
10929
11679
|
}
|
|
10930
|
-
const normalized = normalizeSdkEvent(
|
|
11680
|
+
const normalized = normalizeSdkEvent(
|
|
11681
|
+
next.value,
|
|
11682
|
+
retainedScreenshotMetadata ? {
|
|
11683
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
11684
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available ? retainedScreenshotMetadata : {
|
|
11685
|
+
available: false,
|
|
11686
|
+
reason: retainedScreenshotMetadata.reason
|
|
11687
|
+
}
|
|
11688
|
+
} : {}
|
|
11689
|
+
);
|
|
10931
11690
|
for (const event of normalized) {
|
|
10932
11691
|
streamTiming.onEvent(event.type);
|
|
10933
11692
|
await batcher.push(event);
|
|
@@ -10951,6 +11710,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
10951
11710
|
currentToolBatchCallIds = /* @__PURE__ */ new Set();
|
|
10952
11711
|
currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
|
|
10953
11712
|
}
|
|
11713
|
+
for (const callId of stableToolCallIdsToClear) {
|
|
11714
|
+
computerScreenshotCallIds.delete(callId);
|
|
11715
|
+
}
|
|
10954
11716
|
}
|
|
10955
11717
|
}
|
|
10956
11718
|
} finally {
|
|
@@ -11050,6 +11812,13 @@ function createRunAgentTurnActivity(services) {
|
|
|
11050
11812
|
await reconcileConversationTruth();
|
|
11051
11813
|
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
11052
11814
|
const humanInputInterruptions = runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
|
|
11815
|
+
const latestWorkspace = await getWorkspace2(db, input.workspaceId);
|
|
11816
|
+
if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
11817
|
+
assertWorkspaceHumanInputAllowed(
|
|
11818
|
+
resolveWorkspaceAgentHumanInputEnabled(latestWorkspace.settings),
|
|
11819
|
+
"interruption",
|
|
11820
|
+
humanInputInterruptions.length > 0
|
|
11821
|
+
);
|
|
11053
11822
|
const humanInputRequests = await Promise.all(
|
|
11054
11823
|
humanInputInterruptions.map(async (interruption) => {
|
|
11055
11824
|
const id = stableHumanInputRequestId(
|
|
@@ -11115,17 +11884,13 @@ function createRunAgentTurnActivity(services) {
|
|
|
11115
11884
|
})) {
|
|
11116
11885
|
return claimedResult({ status: "cancelled" });
|
|
11117
11886
|
}
|
|
11887
|
+
await finalizeTurnOpStreamOps();
|
|
11118
11888
|
activityStatus = "requires_action";
|
|
11119
11889
|
return claimedResult({ status: "requires_action" });
|
|
11120
11890
|
}
|
|
11121
|
-
const finalOutput = String(
|
|
11122
|
-
await reconcileConversationTruth();
|
|
11123
|
-
|
|
11124
|
-
try {
|
|
11125
|
-
await machinePrimarySession.finalizeOpStreamOps();
|
|
11126
|
-
} catch {
|
|
11127
|
-
}
|
|
11128
|
-
}
|
|
11891
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
11892
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
11893
|
+
await finalizeTurnOpStreamOps();
|
|
11129
11894
|
if (!await settle({
|
|
11130
11895
|
events: [
|
|
11131
11896
|
{
|
|
@@ -11183,8 +11948,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11183
11948
|
sessionId: input.sessionId,
|
|
11184
11949
|
turnId: activeTurnId,
|
|
11185
11950
|
reason: recoveryKind,
|
|
11186
|
-
|
|
11187
|
-
error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
|
|
11951
|
+
...safeErrorDiagnostic(attemptError),
|
|
11188
11952
|
signalTokens: compactionNeeded?.signalTokens,
|
|
11189
11953
|
thresholdTokens: compactionNeeded?.thresholdTokens
|
|
11190
11954
|
});
|
|
@@ -11226,13 +11990,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
11226
11990
|
);
|
|
11227
11991
|
compactionRequestCleared = landmark.requestConsumed;
|
|
11228
11992
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
11229
|
-
compactionFailureMessage = String(
|
|
11230
|
-
redact(compactionFailureReasonFromError(compactError))
|
|
11231
|
-
);
|
|
11993
|
+
compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
|
|
11232
11994
|
observability.warn("context compaction recovery compaction failed", {
|
|
11233
11995
|
sessionId: input.sessionId,
|
|
11234
11996
|
turnId: activeTurnId,
|
|
11235
|
-
|
|
11997
|
+
...safeErrorDiagnostic(compactError)
|
|
11236
11998
|
});
|
|
11237
11999
|
}
|
|
11238
12000
|
if (!compactionHandled) {
|
|
@@ -11305,6 +12067,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11305
12067
|
turnMetricOutcome = "cancelled";
|
|
11306
12068
|
return claimedResult({ status: "cancelled" });
|
|
11307
12069
|
}
|
|
12070
|
+
acknowledgeRecoveryQuiescence();
|
|
11308
12071
|
await publishDurableSessionEvents(
|
|
11309
12072
|
bus,
|
|
11310
12073
|
input.workspaceId,
|
|
@@ -11322,7 +12085,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11322
12085
|
} catch (recoveryError) {
|
|
11323
12086
|
console.error(
|
|
11324
12087
|
"sandbox lease supersession recovery failed",
|
|
11325
|
-
safeErrorDiagnostic(recoveryError
|
|
12088
|
+
safeErrorDiagnostic(recoveryError)
|
|
11326
12089
|
);
|
|
11327
12090
|
throw recoveryError;
|
|
11328
12091
|
}
|
|
@@ -11349,6 +12112,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11349
12112
|
turnMetricOutcome = "cancelled";
|
|
11350
12113
|
return claimedResult({ status: "cancelled" });
|
|
11351
12114
|
}
|
|
12115
|
+
acknowledgeRecoveryQuiescence();
|
|
11352
12116
|
await publishDurableSessionEvents(
|
|
11353
12117
|
bus,
|
|
11354
12118
|
input.workspaceId,
|
|
@@ -11364,7 +12128,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11364
12128
|
} catch (recoveryError) {
|
|
11365
12129
|
console.error(
|
|
11366
12130
|
"sandbox deadline rotation recovery failed",
|
|
11367
|
-
safeErrorDiagnostic(recoveryError
|
|
12131
|
+
safeErrorDiagnostic(recoveryError)
|
|
11368
12132
|
);
|
|
11369
12133
|
throw recoveryError;
|
|
11370
12134
|
}
|
|
@@ -11387,6 +12151,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11387
12151
|
turnMetricOutcome = "cancelled";
|
|
11388
12152
|
return claimedResult({ status: "cancelled" });
|
|
11389
12153
|
}
|
|
12154
|
+
acknowledgeRecoveryQuiescence();
|
|
11390
12155
|
await publishDurableSessionEvents(
|
|
11391
12156
|
bus,
|
|
11392
12157
|
input.workspaceId,
|
|
@@ -11399,7 +12164,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11399
12164
|
} catch (recoveryError) {
|
|
11400
12165
|
console.error(
|
|
11401
12166
|
"worker-shutdown recovery checkpoint failed",
|
|
11402
|
-
safeErrorDiagnostic(recoveryError
|
|
12167
|
+
safeErrorDiagnostic(recoveryError)
|
|
11403
12168
|
);
|
|
11404
12169
|
throw recoveryError;
|
|
11405
12170
|
}
|
|
@@ -11471,11 +12236,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
11471
12236
|
await reconcileConversationTruth({ requireDurable: true });
|
|
11472
12237
|
}
|
|
11473
12238
|
checkpointDurable = true;
|
|
11474
|
-
} catch
|
|
12239
|
+
} catch {
|
|
11475
12240
|
observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
|
|
11476
|
-
|
|
11477
|
-
|
|
11478
|
-
|
|
12241
|
+
errorClass: "CodexCheckpointOperationError",
|
|
12242
|
+
errorCode: "codex_lease_loss_checkpoint_failed",
|
|
12243
|
+
origin: "worker"
|
|
11479
12244
|
});
|
|
11480
12245
|
}
|
|
11481
12246
|
const settlement = await settleCodexCredentialLeaseLoss(db, {
|
|
@@ -11550,16 +12315,16 @@ function createRunAgentTurnActivity(services) {
|
|
|
11550
12315
|
await flushRuntimeBatcher();
|
|
11551
12316
|
await reconcileConversationTruth({ requireDurable: true });
|
|
11552
12317
|
checkpointDurable = true;
|
|
11553
|
-
} catch
|
|
12318
|
+
} catch {
|
|
11554
12319
|
observability.incrementCounter({
|
|
11555
12320
|
name: "opengeni_codex_failover_checkpoints_total",
|
|
11556
12321
|
help: "Durable Codex failover checkpoint attempts by outcome.",
|
|
11557
12322
|
labels: { workspace_key: codexWorkspaceKey, outcome: "failed" }
|
|
11558
12323
|
});
|
|
11559
12324
|
observability.warn("Codex failover checkpoint failed; refusing automatic replay", {
|
|
11560
|
-
|
|
11561
|
-
|
|
11562
|
-
|
|
12325
|
+
errorClass: "CodexCheckpointOperationError",
|
|
12326
|
+
errorCode: "codex_failover_checkpoint_failed",
|
|
12327
|
+
origin: "worker"
|
|
11563
12328
|
});
|
|
11564
12329
|
}
|
|
11565
12330
|
if (checkpointDurable) {
|
|
@@ -11984,6 +12749,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11984
12749
|
return claimedResult({ status: "cancelled" });
|
|
11985
12750
|
}
|
|
11986
12751
|
if (recovery.action === "recovering") {
|
|
12752
|
+
acknowledgeRecoveryQuiescence();
|
|
11987
12753
|
await publishDurableSessionEvents(
|
|
11988
12754
|
bus,
|
|
11989
12755
|
input.workspaceId,
|
|
@@ -11996,7 +12762,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
11996
12762
|
return claimedResult({ status: "recovering" });
|
|
11997
12763
|
}
|
|
11998
12764
|
}
|
|
11999
|
-
const failure =
|
|
12765
|
+
const failure = agentRunFailurePayload(error, { isCodexTurn });
|
|
12000
12766
|
if (isSessionEventPersistenceError(error)) {
|
|
12001
12767
|
observability.error("session event persistence failed", {
|
|
12002
12768
|
accountId: input.accountId,
|
|
@@ -12017,7 +12783,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
12017
12783
|
dbColumn: error.details.database.column,
|
|
12018
12784
|
dbDataType: error.details.database.dataType,
|
|
12019
12785
|
dbConstraint: error.details.database.constraint,
|
|
12020
|
-
dbRoutine: error.details.database.routine
|
|
12786
|
+
dbRoutine: error.details.database.routine,
|
|
12787
|
+
error: error.message
|
|
12021
12788
|
});
|
|
12022
12789
|
}
|
|
12023
12790
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
@@ -12049,6 +12816,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
12049
12816
|
turnMetricOutcome = "cancelled";
|
|
12050
12817
|
return claimedResult({ status: "cancelled" });
|
|
12051
12818
|
}
|
|
12819
|
+
acknowledgeRecoveryQuiescence();
|
|
12052
12820
|
await publishDurableSessionEvents(
|
|
12053
12821
|
bus,
|
|
12054
12822
|
input.workspaceId,
|
|
@@ -12129,17 +12897,29 @@ function createRunAgentTurnActivity(services) {
|
|
|
12129
12897
|
const credentialSessionToClear = runCredentialSession;
|
|
12130
12898
|
runCredentialSession = null;
|
|
12131
12899
|
if (credentialSessionToClear) {
|
|
12132
|
-
await
|
|
12133
|
-
|
|
12134
|
-
|
|
12900
|
+
const clearAttemptCredentials = async () => await clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
12901
|
+
sessionId: input.sessionId,
|
|
12902
|
+
attemptId: input.attemptId,
|
|
12903
|
+
executionGeneration
|
|
12904
|
+
});
|
|
12905
|
+
await clearAttemptCredentialsWithSettledFence({
|
|
12906
|
+
activityStatus,
|
|
12907
|
+
runWorkspaceFencedClear: async () => await runWorkspaceMutationForSandbox(
|
|
12908
|
+
requireResolvedSandboxForMutation(
|
|
12909
|
+
"Run credential cleanup has no exact sandbox lease target"
|
|
12910
|
+
),
|
|
12911
|
+
"runCredentialAttemptClear",
|
|
12912
|
+
clearAttemptCredentials
|
|
12135
12913
|
),
|
|
12136
|
-
|
|
12137
|
-
|
|
12138
|
-
|
|
12139
|
-
|
|
12140
|
-
|
|
12141
|
-
|
|
12142
|
-
|
|
12914
|
+
clearExactAttempt: clearAttemptCredentials,
|
|
12915
|
+
onSettledAttemptFence: () => {
|
|
12916
|
+
observability.info("retrying exact run credential cleanup after turn settlement", {
|
|
12917
|
+
"opengeni.session_id": input.sessionId,
|
|
12918
|
+
"opengeni.turn_id": turnId ?? "",
|
|
12919
|
+
"opengeni.attempt_id": input.attemptId
|
|
12920
|
+
});
|
|
12921
|
+
}
|
|
12922
|
+
});
|
|
12143
12923
|
}
|
|
12144
12924
|
await drainAttemptOwnedSandboxWriters({
|
|
12145
12925
|
// Normal turn completion owns the same process boundary as
|
|
@@ -12197,18 +12977,18 @@ function createRunAgentTurnActivity(services) {
|
|
|
12197
12977
|
onReceiptFailure: (error) => {
|
|
12198
12978
|
console.error(
|
|
12199
12979
|
"agent turn quiescence receipt exhausted; signalling proof",
|
|
12200
|
-
safeErrorDiagnostic(error
|
|
12980
|
+
safeErrorDiagnostic(error)
|
|
12201
12981
|
);
|
|
12202
12982
|
},
|
|
12203
12983
|
onPublishFailure: (error) => {
|
|
12204
12984
|
console.error(
|
|
12205
12985
|
"agent turn quiescence event fanout failed",
|
|
12206
|
-
safeErrorDiagnostic(error
|
|
12986
|
+
safeErrorDiagnostic(error)
|
|
12207
12987
|
);
|
|
12208
12988
|
},
|
|
12209
12989
|
onSignalFailure: (error, attempt, retryMs) => {
|
|
12210
12990
|
console.error("agent turn quiescence proof signal failed; retrying", {
|
|
12211
|
-
error: safeErrorDiagnostic(error
|
|
12991
|
+
error: safeErrorDiagnostic(error),
|
|
12212
12992
|
attempt,
|
|
12213
12993
|
retryMs
|
|
12214
12994
|
});
|
|
@@ -12394,7 +13174,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
12394
13174
|
sandboxToRelease.release().catch((releaseError) => {
|
|
12395
13175
|
console.error(
|
|
12396
13176
|
"sandbox lease release failed (turn outcome unaffected)",
|
|
12397
|
-
safeErrorDiagnostic(releaseError
|
|
13177
|
+
safeErrorDiagnostic(releaseError)
|
|
12398
13178
|
);
|
|
12399
13179
|
}),
|
|
12400
13180
|
finalizerSignal
|
|
@@ -12404,7 +13184,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
12404
13184
|
finalizationError ??= error;
|
|
12405
13185
|
console.error(
|
|
12406
13186
|
"agent turn finalization failed (turn outcome unaffected)",
|
|
12407
|
-
safeErrorDiagnostic(error
|
|
13187
|
+
safeErrorDiagnostic(error)
|
|
12408
13188
|
);
|
|
12409
13189
|
} finally {
|
|
12410
13190
|
if (!sandboxResumeController.signal.aborted) {
|
|
@@ -12459,10 +13239,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
12459
13239
|
"opengeni.duration_ms": Math.round(durationSeconds * 1e3),
|
|
12460
13240
|
"opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1e3)
|
|
12461
13241
|
},
|
|
12462
|
-
error: finalizationError || activityError ? safeErrorForTelemetry(
|
|
12463
|
-
finalizationError ?? activityError,
|
|
12464
|
-
(value) => String(redact(value))
|
|
12465
|
-
) : void 0
|
|
13242
|
+
error: finalizationError || activityError ? safeErrorForTelemetry(finalizationError ?? activityError) : void 0
|
|
12466
13243
|
});
|
|
12467
13244
|
assertPhysicalToolQuiescenceForCancellation({
|
|
12468
13245
|
acknowledgeQuiescence,
|
|
@@ -12525,6 +13302,15 @@ function isTransientProviderError(error) {
|
|
|
12525
13302
|
);
|
|
12526
13303
|
}
|
|
12527
13304
|
function agentRunFailurePayload(error, options = {}) {
|
|
13305
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
13306
|
+
const underlying = isSessionEventPersistenceError(error.cause) ? agentRunFailurePayload(error.cause, options) : {
|
|
13307
|
+
error: error.cause instanceof Error ? error.cause.message : String(error.cause)
|
|
13308
|
+
};
|
|
13309
|
+
return {
|
|
13310
|
+
...underlying,
|
|
13311
|
+
historyPersistenceStage: error.stage
|
|
13312
|
+
};
|
|
13313
|
+
}
|
|
12528
13314
|
const message = error instanceof Error ? error.message : String(error);
|
|
12529
13315
|
const status = typeof error === "object" && error !== null ? Number(
|
|
12530
13316
|
error.status ?? error.statusCode
|
|
@@ -12539,10 +13325,8 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
12539
13325
|
}
|
|
12540
13326
|
if (isSessionEventPersistenceError(error)) {
|
|
12541
13327
|
const { details } = error;
|
|
12542
|
-
const eventLabel = details.eventTypes.join(", ") || "session events";
|
|
12543
|
-
const failureLabel = details.code === "db_deadlock" ? "Database deadlock" : details.code === "db_serialization_failure" ? "Database serialization failure" : "Database failure";
|
|
12544
13328
|
return {
|
|
12545
|
-
error:
|
|
13329
|
+
error: error.message,
|
|
12546
13330
|
code: details.code,
|
|
12547
13331
|
detail: details.retryOutcome === "exhausted" ? `The idempotent persistence transaction failed after ${details.attempts} attempts.` : "The database rejected the idempotent persistence transaction.",
|
|
12548
13332
|
correlationId: details.correlationId,
|
|
@@ -12584,7 +13368,8 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
12584
13368
|
return {
|
|
12585
13369
|
error: "A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
|
|
12586
13370
|
code: "mcp_transport_unavailable",
|
|
12587
|
-
retryable: true
|
|
13371
|
+
retryable: true,
|
|
13372
|
+
detail: message
|
|
12588
13373
|
};
|
|
12589
13374
|
}
|
|
12590
13375
|
if (status === 429 || code === "rate_limit_exceeded" || /(?:too many requests|rate.?limit|\b429\b)/i.test(message)) {
|
|
@@ -12713,7 +13498,13 @@ function pendingToolCallFromSdkEvent(event) {
|
|
|
12713
13498
|
if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
|
|
12714
13499
|
return null;
|
|
12715
13500
|
}
|
|
12716
|
-
|
|
13501
|
+
const callItem = normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]');
|
|
13502
|
+
return {
|
|
13503
|
+
callId,
|
|
13504
|
+
callType,
|
|
13505
|
+
callName: typeof raw.name === "string" ? raw.name : null,
|
|
13506
|
+
callItem
|
|
13507
|
+
};
|
|
12717
13508
|
}
|
|
12718
13509
|
function completedToolCallFromSdkEvent(event) {
|
|
12719
13510
|
if (!event || typeof event !== "object") return null;
|
|
@@ -12724,7 +13515,11 @@ function completedToolCallFromSdkEvent(event) {
|
|
|
12724
13515
|
}
|
|
12725
13516
|
const raw = item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem) ? item.rawItem : {};
|
|
12726
13517
|
const callId = raw.callId ?? raw.call_id ?? item.id;
|
|
12727
|
-
|
|
13518
|
+
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
13519
|
+
return {
|
|
13520
|
+
callId,
|
|
13521
|
+
resultItem: normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]')
|
|
13522
|
+
};
|
|
12728
13523
|
}
|
|
12729
13524
|
var BudgetExhaustedError = class extends Error {
|
|
12730
13525
|
constructor(message, serializedRunState) {
|
|
@@ -12919,12 +13714,7 @@ async function recordAuthoritativeModelCallFact(input) {
|
|
|
12919
13714
|
});
|
|
12920
13715
|
} catch (error) {
|
|
12921
13716
|
input.observability.warn("model call fact persist failed", {
|
|
12922
|
-
|
|
12923
|
-
workspaceId: input.workspaceId,
|
|
12924
|
-
sessionId: input.sessionId,
|
|
12925
|
-
turnId: input.turnId,
|
|
12926
|
-
sourceKey: input.sourceKey,
|
|
12927
|
-
error: error instanceof Error ? error.message : String(error)
|
|
13717
|
+
...safeErrorDiagnostic(error)
|
|
12928
13718
|
});
|
|
12929
13719
|
}
|
|
12930
13720
|
}
|
|
@@ -14626,6 +15416,7 @@ import {
|
|
|
14626
15416
|
isCodexBilledTurn as isCodexBilledTurn2,
|
|
14627
15417
|
markScheduledTaskRunFailedIfQueued,
|
|
14628
15418
|
recordUsageEvent as recordUsageEvent3,
|
|
15419
|
+
requireScheduledTaskTargetInTransaction,
|
|
14629
15420
|
requireSession as requireSession3,
|
|
14630
15421
|
setTemporalWorkflowId,
|
|
14631
15422
|
settleScheduledTaskRunInTransaction,
|
|
@@ -14748,7 +15539,11 @@ function createScheduledTaskActivities(services) {
|
|
|
14748
15539
|
const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
14749
15540
|
const goalSpec = task.agentConfig.goal ?? null;
|
|
14750
15541
|
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
14751
|
-
|
|
15542
|
+
const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
15543
|
+
if (task.runMode === "existing_session" && !existingSessionId) {
|
|
15544
|
+
throw new Error("scheduled task target session is unavailable");
|
|
15545
|
+
}
|
|
15546
|
+
if (task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && !existingSessionId) {
|
|
14752
15547
|
const variableSet = task.variableSetId ? await getVariableSet(db, task.workspaceId, task.variableSetId) : null;
|
|
14753
15548
|
if (task.variableSetId && !variableSet) {
|
|
14754
15549
|
throw new Error(`variable set not found: ${task.variableSetId}`);
|
|
@@ -14915,7 +15710,7 @@ function createScheduledTaskActivities(services) {
|
|
|
14915
15710
|
workflowWakeRevision: scheduledUpdate.workflowWakeRevision
|
|
14916
15711
|
};
|
|
14917
15712
|
} else {
|
|
14918
|
-
const session = await requireSession3(db, task.workspaceId,
|
|
15713
|
+
const session = await requireSession3(db, task.workspaceId, existingSessionId);
|
|
14919
15714
|
assertReusableSessionRevivable(session.status);
|
|
14920
15715
|
if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
|
|
14921
15716
|
throw new Error(
|
|
@@ -14927,7 +15722,7 @@ function createScheduledTaskActivities(services) {
|
|
|
14927
15722
|
"scheduled task OpenGeni Slack bot binding does not match its reusable session"
|
|
14928
15723
|
);
|
|
14929
15724
|
}
|
|
14930
|
-
const reusableGoal = goalSpec && run.status === "queued" ? await upsertSessionGoal(db, {
|
|
15725
|
+
const reusableGoal = task.runMode === "reusable_session" && goalSpec && run.status === "queued" ? await upsertSessionGoal(db, {
|
|
14931
15726
|
accountId: task.accountId,
|
|
14932
15727
|
workspaceId: task.workspaceId,
|
|
14933
15728
|
sessionId: session.id,
|
|
@@ -14984,6 +15779,13 @@ function createScheduledTaskActivities(services) {
|
|
|
14984
15779
|
personalConnectionDelegations: taskPersonalConnectionDelegations
|
|
14985
15780
|
},
|
|
14986
15781
|
async (tx, wakeEventId) => {
|
|
15782
|
+
if (task.runMode === "existing_session") {
|
|
15783
|
+
await requireScheduledTaskTargetInTransaction(tx, {
|
|
15784
|
+
workspaceId: task.workspaceId,
|
|
15785
|
+
taskId: task.id,
|
|
15786
|
+
targetSessionId: session.id
|
|
15787
|
+
});
|
|
15788
|
+
}
|
|
14987
15789
|
if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
|
|
14988
15790
|
await settleScheduledTaskRunInTransaction(tx, {
|
|
14989
15791
|
workspaceId: task.workspaceId,
|
|
@@ -15414,9 +16216,6 @@ import {
|
|
|
15414
16216
|
markWarmLeaseInstanceLost as markWarmLeaseInstanceLost3,
|
|
15415
16217
|
recordWarmingSandboxCreated as recordWarmingSandboxCreated2,
|
|
15416
16218
|
releaseLeaseHolder as releaseLeaseHolder2,
|
|
15417
|
-
sanitizeEventPayload,
|
|
15418
|
-
sanitizeEventString,
|
|
15419
|
-
sanitizeMemoryText,
|
|
15420
16219
|
touchLeaseHolder as touchLeaseHolder2,
|
|
15421
16220
|
updateRigChangeStatus
|
|
15422
16221
|
} from "@opengeni/db";
|
|
@@ -15429,7 +16228,6 @@ import {
|
|
|
15429
16228
|
serializeEstablishedSandboxEnvelope,
|
|
15430
16229
|
tagModalSandbox as tagModalSandbox2
|
|
15431
16230
|
} from "@opengeni/runtime";
|
|
15432
|
-
var OUTPUT_TAIL_LIMIT = 64 * 1024;
|
|
15433
16231
|
var RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS = 2 * 6e4;
|
|
15434
16232
|
var RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 6e4;
|
|
15435
16233
|
var RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 1e4;
|
|
@@ -15564,15 +16362,6 @@ function cleanupWaitMs(lifecycle) {
|
|
|
15564
16362
|
Math.min(RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS, lifecycle.cleanupDeadlineAtMs - Date.now())
|
|
15565
16363
|
);
|
|
15566
16364
|
}
|
|
15567
|
-
function tail(value, limit = OUTPUT_TAIL_LIMIT) {
|
|
15568
|
-
return value.length > limit ? value.slice(-limit) : value;
|
|
15569
|
-
}
|
|
15570
|
-
function scrubVerificationOutput(value) {
|
|
15571
|
-
return sanitizeMemoryText(sanitizeEventString(value)).text;
|
|
15572
|
-
}
|
|
15573
|
-
function scrubVerificationPayload(value) {
|
|
15574
|
-
return sanitizeEventPayload(value);
|
|
15575
|
-
}
|
|
15576
16365
|
function systemGrant(rig) {
|
|
15577
16366
|
return {
|
|
15578
16367
|
accountId: rig.accountId,
|
|
@@ -15919,7 +16708,7 @@ async function runCommand(session, command, timeoutMs, commandRunner) {
|
|
|
15919
16708
|
const result = await commandRunner(session, args);
|
|
15920
16709
|
return {
|
|
15921
16710
|
exitCode: sandboxCommandExitCode2(result),
|
|
15922
|
-
output:
|
|
16711
|
+
output: sandboxCommandOutput(result)
|
|
15923
16712
|
};
|
|
15924
16713
|
}
|
|
15925
16714
|
function setupAppendCommand(change) {
|
|
@@ -16050,7 +16839,7 @@ function createRigVerificationActivities(services) {
|
|
|
16050
16839
|
verification.passed = false;
|
|
16051
16840
|
const updated2 = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
16052
16841
|
status: "rejected",
|
|
16053
|
-
verification
|
|
16842
|
+
verification
|
|
16054
16843
|
});
|
|
16055
16844
|
await recordRigAuditEvent(db, {
|
|
16056
16845
|
grant,
|
|
@@ -16077,7 +16866,7 @@ function createRigVerificationActivities(services) {
|
|
|
16077
16866
|
);
|
|
16078
16867
|
checkResults.push({
|
|
16079
16868
|
name: check.name,
|
|
16080
|
-
command:
|
|
16869
|
+
command: check.command,
|
|
16081
16870
|
...result
|
|
16082
16871
|
});
|
|
16083
16872
|
}
|
|
@@ -16089,7 +16878,7 @@ function createRigVerificationActivities(services) {
|
|
|
16089
16878
|
if (classified.action === "auto_promote") {
|
|
16090
16879
|
await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
16091
16880
|
status: "verifying",
|
|
16092
|
-
verification
|
|
16881
|
+
verification
|
|
16093
16882
|
});
|
|
16094
16883
|
const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
|
|
16095
16884
|
...change,
|
|
@@ -16105,7 +16894,7 @@ function createRigVerificationActivities(services) {
|
|
|
16105
16894
|
}
|
|
16106
16895
|
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
16107
16896
|
status: classified.status,
|
|
16108
|
-
verification
|
|
16897
|
+
verification
|
|
16109
16898
|
});
|
|
16110
16899
|
await recordRigAuditEvent(db, {
|
|
16111
16900
|
grant,
|
|
@@ -16127,12 +16916,10 @@ function createRigVerificationActivities(services) {
|
|
|
16127
16916
|
} catch (error) {
|
|
16128
16917
|
verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
16129
16918
|
verification.passed = false;
|
|
16130
|
-
verification.error =
|
|
16131
|
-
error instanceof Error ? error.message : String(error)
|
|
16132
|
-
);
|
|
16919
|
+
verification.error = error instanceof Error ? error.message : String(error);
|
|
16133
16920
|
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
16134
16921
|
status: "failed",
|
|
16135
|
-
verification
|
|
16922
|
+
verification
|
|
16136
16923
|
});
|
|
16137
16924
|
await recordRigAuditEvent(db, {
|
|
16138
16925
|
grant,
|
|
@@ -16195,7 +16982,7 @@ function createRigVerificationActivities(services) {
|
|
|
16195
16982
|
for (const check of version.checks) {
|
|
16196
16983
|
checkResults.push({
|
|
16197
16984
|
name: check.name,
|
|
16198
|
-
command:
|
|
16985
|
+
command: check.command,
|
|
16199
16986
|
...await runCommand(
|
|
16200
16987
|
established.session,
|
|
16201
16988
|
check.command,
|
|
@@ -16209,33 +16996,30 @@ function createRigVerificationActivities(services) {
|
|
|
16209
16996
|
grant,
|
|
16210
16997
|
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
16211
16998
|
rigId: rig.id,
|
|
16212
|
-
metadata:
|
|
16999
|
+
metadata: {
|
|
16213
17000
|
versionId: version.id,
|
|
16214
17001
|
startedAt,
|
|
16215
17002
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
16216
17003
|
passed,
|
|
16217
17004
|
checkResults
|
|
16218
|
-
}
|
|
17005
|
+
}
|
|
16219
17006
|
});
|
|
16220
17007
|
return { versionId: version.id, passed, checkResults };
|
|
16221
17008
|
}
|
|
16222
17009
|
);
|
|
16223
17010
|
} catch (error) {
|
|
16224
|
-
const detail =
|
|
16225
|
-
scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
|
|
16226
|
-
4096
|
|
16227
|
-
);
|
|
17011
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
16228
17012
|
await recordRigAuditEvent(db, {
|
|
16229
17013
|
grant,
|
|
16230
17014
|
action: "rig.verification.failed",
|
|
16231
17015
|
rigId: rig.id,
|
|
16232
|
-
metadata:
|
|
17016
|
+
metadata: {
|
|
16233
17017
|
versionId: version.id,
|
|
16234
17018
|
startedAt,
|
|
16235
17019
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
16236
17020
|
passed: false,
|
|
16237
17021
|
error: detail
|
|
16238
|
-
}
|
|
17022
|
+
}
|
|
16239
17023
|
});
|
|
16240
17024
|
throw error;
|
|
16241
17025
|
}
|
|
@@ -16243,6 +17027,144 @@ function createRigVerificationActivities(services) {
|
|
|
16243
17027
|
};
|
|
16244
17028
|
}
|
|
16245
17029
|
|
|
17030
|
+
// src/activities/retained-screenshot-reaper.ts
|
|
17031
|
+
import {
|
|
17032
|
+
claimRetainedScreenshotMaintenance,
|
|
17033
|
+
completeRetainedScreenshotMaintenance,
|
|
17034
|
+
promoteRetainedScreenshotMaintenanceCleanup
|
|
17035
|
+
} from "@opengeni/db";
|
|
17036
|
+
var RETAINED_SCREENSHOT_PENDING_GRACE_MS = 15 * 60 * 1e3;
|
|
17037
|
+
var RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
17038
|
+
var RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE = 100;
|
|
17039
|
+
function createRetainedScreenshotMaintenanceActivities(services, options = {}) {
|
|
17040
|
+
const pendingGraceMs = options.pendingGraceMs ?? RETAINED_SCREENSHOT_PENDING_GRACE_MS;
|
|
17041
|
+
const claimTimeoutMs = options.claimTimeoutMs ?? RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS;
|
|
17042
|
+
const batchSize = options.batchSize ?? RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE;
|
|
17043
|
+
const fileExists = options.fileExists ?? ((storage, file) => storage.fileExists(file));
|
|
17044
|
+
const headFile = options.headFile ?? ((storage, file) => storage.headFile(file));
|
|
17045
|
+
const deleteObject = options.deleteObject ?? ((storage, key) => storage.deleteObject(key));
|
|
17046
|
+
async function maintainRetainedScreenshots2() {
|
|
17047
|
+
const { db, objectStorage, observability } = await services();
|
|
17048
|
+
if (!objectStorage) {
|
|
17049
|
+
return { claimed: 0, ready: 0, deleted: 0, failed: 0, retryable: 0 };
|
|
17050
|
+
}
|
|
17051
|
+
const claims = await claimRetainedScreenshotMaintenance(db, {
|
|
17052
|
+
pendingGraceMs,
|
|
17053
|
+
claimTimeoutMs,
|
|
17054
|
+
limit: batchSize
|
|
17055
|
+
});
|
|
17056
|
+
const result = {
|
|
17057
|
+
claimed: claims.length,
|
|
17058
|
+
ready: 0,
|
|
17059
|
+
deleted: 0,
|
|
17060
|
+
failed: 0,
|
|
17061
|
+
retryable: 0
|
|
17062
|
+
};
|
|
17063
|
+
for (const claim of claims) {
|
|
17064
|
+
try {
|
|
17065
|
+
if (claim.action === "reconcile") {
|
|
17066
|
+
const outcome2 = await reconcileClaim(db, objectStorage, claim, {
|
|
17067
|
+
fileExists,
|
|
17068
|
+
headFile,
|
|
17069
|
+
deleteObject
|
|
17070
|
+
});
|
|
17071
|
+
if (outcome2 === "ready") {
|
|
17072
|
+
result.ready += 1;
|
|
17073
|
+
} else {
|
|
17074
|
+
result.failed += 1;
|
|
17075
|
+
}
|
|
17076
|
+
continue;
|
|
17077
|
+
}
|
|
17078
|
+
await deleteObject(objectStorage, claim.objectKey);
|
|
17079
|
+
const outcome = terminalDeleteOutcome(claim.cleanupReason);
|
|
17080
|
+
const settled = await completeRetainedScreenshotMaintenance(db, {
|
|
17081
|
+
accountId: claim.accountId,
|
|
17082
|
+
workspaceId: claim.workspaceId,
|
|
17083
|
+
artifactId: claim.artifactId,
|
|
17084
|
+
claimId: claim.claimId,
|
|
17085
|
+
outcome
|
|
17086
|
+
});
|
|
17087
|
+
if (!settled) throw new Error("retained screenshot cleanup claim was superseded");
|
|
17088
|
+
result.deleted += 1;
|
|
17089
|
+
} catch (error) {
|
|
17090
|
+
result.retryable += 1;
|
|
17091
|
+
observability.warn("retained screenshot maintenance failed; claim remains reclaimable", {
|
|
17092
|
+
workspaceId: claim.workspaceId,
|
|
17093
|
+
sessionId: claim.sessionId,
|
|
17094
|
+
artifactId: claim.artifactId,
|
|
17095
|
+
action: claim.action,
|
|
17096
|
+
error: error instanceof Error ? error.message : String(error)
|
|
17097
|
+
});
|
|
17098
|
+
}
|
|
17099
|
+
}
|
|
17100
|
+
if (claims.length > 0) {
|
|
17101
|
+
observability.info("retained screenshot maintenance swept", result);
|
|
17102
|
+
}
|
|
17103
|
+
return result;
|
|
17104
|
+
}
|
|
17105
|
+
return { maintainRetainedScreenshots: maintainRetainedScreenshots2 };
|
|
17106
|
+
}
|
|
17107
|
+
async function reconcileClaim(db, storage, claim, provider) {
|
|
17108
|
+
const observation = await observeClaimObject(storage, claim, provider);
|
|
17109
|
+
if (observation === "matches") {
|
|
17110
|
+
const settled2 = await completeRetainedScreenshotMaintenance(db, {
|
|
17111
|
+
accountId: claim.accountId,
|
|
17112
|
+
workspaceId: claim.workspaceId,
|
|
17113
|
+
artifactId: claim.artifactId,
|
|
17114
|
+
claimId: claim.claimId,
|
|
17115
|
+
outcome: "ready"
|
|
17116
|
+
});
|
|
17117
|
+
if (!settled2) throw new Error("retained screenshot reconcile claim was superseded");
|
|
17118
|
+
return "ready";
|
|
17119
|
+
}
|
|
17120
|
+
const promoted = await promoteRetainedScreenshotMaintenanceCleanup(db, {
|
|
17121
|
+
accountId: claim.accountId,
|
|
17122
|
+
workspaceId: claim.workspaceId,
|
|
17123
|
+
artifactId: claim.artifactId,
|
|
17124
|
+
claimId: claim.claimId,
|
|
17125
|
+
cleanupReason: "failed"
|
|
17126
|
+
});
|
|
17127
|
+
if (!promoted) throw new Error("retained screenshot reconcile claim was superseded");
|
|
17128
|
+
await provider.deleteObject(storage, claim.objectKey);
|
|
17129
|
+
const settled = await completeRetainedScreenshotMaintenance(db, {
|
|
17130
|
+
accountId: claim.accountId,
|
|
17131
|
+
workspaceId: claim.workspaceId,
|
|
17132
|
+
artifactId: claim.artifactId,
|
|
17133
|
+
claimId: claim.claimId,
|
|
17134
|
+
outcome: "failed"
|
|
17135
|
+
});
|
|
17136
|
+
if (!settled) throw new Error("retained screenshot reconcile claim was superseded");
|
|
17137
|
+
return "failed";
|
|
17138
|
+
}
|
|
17139
|
+
async function observeClaimObject(storage, claim, provider) {
|
|
17140
|
+
const file = claimFile(storage, claim);
|
|
17141
|
+
if (!await provider.fileExists(storage, file)) return "missing";
|
|
17142
|
+
const head = await provider.headFile(storage, file);
|
|
17143
|
+
return head.ContentLength === claim.sizeBytes && head.ContentType === claim.mediaType && head.Metadata?.sha256 === claim.sha256 ? "matches" : "mismatch";
|
|
17144
|
+
}
|
|
17145
|
+
function claimFile(storage, claim) {
|
|
17146
|
+
const timestamp = (/* @__PURE__ */ new Date(0)).toISOString();
|
|
17147
|
+
return {
|
|
17148
|
+
id: claim.artifactId,
|
|
17149
|
+
workspaceId: claim.workspaceId,
|
|
17150
|
+
status: "pending_upload",
|
|
17151
|
+
filename: `computer-screenshot-${claim.artifactId}.png`,
|
|
17152
|
+
safeFilename: `computer-screenshot-${claim.artifactId}.png`,
|
|
17153
|
+
contentType: claim.mediaType,
|
|
17154
|
+
sizeBytes: claim.sizeBytes,
|
|
17155
|
+
sha256: claim.sha256,
|
|
17156
|
+
bucket: storage.bucket,
|
|
17157
|
+
objectKey: claim.objectKey,
|
|
17158
|
+
createdAt: timestamp,
|
|
17159
|
+
updatedAt: timestamp
|
|
17160
|
+
};
|
|
17161
|
+
}
|
|
17162
|
+
function terminalDeleteOutcome(reason) {
|
|
17163
|
+
if (reason === "expired") return "expired";
|
|
17164
|
+
if (reason === "failed") return "failed";
|
|
17165
|
+
return "deleted";
|
|
17166
|
+
}
|
|
17167
|
+
|
|
16246
17168
|
// src/activities.ts
|
|
16247
17169
|
function createActivityServices(dependencies) {
|
|
16248
17170
|
let servicesPromise = null;
|
|
@@ -16313,6 +17235,7 @@ function controlActivities(services) {
|
|
|
16313
17235
|
...createCodexCapacityActivities(services),
|
|
16314
17236
|
...createRigVerificationActivities(services),
|
|
16315
17237
|
...createFileUploadReaperActivities(services),
|
|
17238
|
+
...createRetainedScreenshotMaintenanceActivities(services),
|
|
16316
17239
|
...createWorkflowWakeActivities(services),
|
|
16317
17240
|
// P1.3: the SOLE liveness/GC/cost-stop driver. Only reapSandboxLeases — no
|
|
16318
17241
|
// *ForViewer activities, no ownerHeartbeat, no resolveOwnerTaskQueue.
|
|
@@ -16323,7 +17246,9 @@ function createControlActivities(dependencies = {}) {
|
|
|
16323
17246
|
return controlActivities(createActivityServices(dependencies));
|
|
16324
17247
|
}
|
|
16325
17248
|
function createTurnActivities(dependencies = {}) {
|
|
16326
|
-
return {
|
|
17249
|
+
return {
|
|
17250
|
+
runAgentTurn: createRunAgentTurnActivity(createActivityServices(dependencies))
|
|
17251
|
+
};
|
|
16327
17252
|
}
|
|
16328
17253
|
var defaultControlActivities = createControlActivities();
|
|
16329
17254
|
var defaultTurnActivities = createTurnActivities();
|
|
@@ -16345,6 +17270,7 @@ var getCodexCapacityWait = defaultControlActivities.getCodexCapacityWait;
|
|
|
16345
17270
|
var reconcileCodexCapacityWait = defaultControlActivities.reconcileCodexCapacityWait;
|
|
16346
17271
|
var reapSandboxLeases = defaultControlActivities.reapSandboxLeases;
|
|
16347
17272
|
var reapExpiredFileUploads = defaultControlActivities.reapExpiredFileUploads;
|
|
17273
|
+
var maintainRetainedScreenshots = defaultControlActivities.maintainRetainedScreenshots;
|
|
16348
17274
|
var dispatchSessionWorkflowWakes = defaultControlActivities.dispatchSessionWorkflowWakes;
|
|
16349
17275
|
var verifyRigChange = defaultControlActivities.verifyRigChange;
|
|
16350
17276
|
var verifyRigVersion = defaultControlActivities.verifyRigVersion;
|
|
@@ -16426,7 +17352,7 @@ function createWorkerHttpHandler(input) {
|
|
|
16426
17352
|
checks: Object.fromEntries(
|
|
16427
17353
|
Object.keys(checks).map((name) => [
|
|
16428
17354
|
name,
|
|
16429
|
-
{ ok: false, error:
|
|
17355
|
+
{ ok: false, error: "worker_not_ready" }
|
|
16430
17356
|
])
|
|
16431
17357
|
)
|
|
16432
17358
|
} : { ...await runReadinessChecks(checks, timeoutMs), ...state ? { state } : {} };
|
|
@@ -16449,11 +17375,8 @@ async function runReadinessChecks(checks, timeoutMs = 2e3) {
|
|
|
16449
17375
|
try {
|
|
16450
17376
|
await withTimeout(Promise.resolve().then(check), timeoutMs);
|
|
16451
17377
|
return [name, { ok: true }];
|
|
16452
|
-
} catch
|
|
16453
|
-
return [
|
|
16454
|
-
name,
|
|
16455
|
-
{ ok: false, error: error instanceof Error ? error.message : String(error) }
|
|
16456
|
-
];
|
|
17378
|
+
} catch {
|
|
17379
|
+
return [name, { ok: false, error: "dependency_unavailable" }];
|
|
16457
17380
|
}
|
|
16458
17381
|
}
|
|
16459
17382
|
)
|
|
@@ -16595,20 +17518,24 @@ function createWorkerServiceLifecycle(input) {
|
|
|
16595
17518
|
})();
|
|
16596
17519
|
return runPromise;
|
|
16597
17520
|
},
|
|
16598
|
-
drain: (
|
|
17521
|
+
drain: (_reason = "host request") => {
|
|
16599
17522
|
if (state === "draining" || state === "stopped" || state === "failed") {
|
|
16600
17523
|
return;
|
|
16601
17524
|
}
|
|
16602
17525
|
state = "draining";
|
|
16603
17526
|
input.observability.info("OpenGeni worker draining (graceful shutdown)", {
|
|
16604
17527
|
role: input.role,
|
|
16605
|
-
|
|
17528
|
+
errorClass: "WorkerLifecycleOperation",
|
|
17529
|
+
errorCode: "worker_draining",
|
|
17530
|
+
origin: "worker-lifecycle"
|
|
16606
17531
|
});
|
|
16607
17532
|
try {
|
|
16608
17533
|
input.worker.shutdown();
|
|
16609
|
-
} catch
|
|
17534
|
+
} catch {
|
|
16610
17535
|
input.observability.warn("worker shutdown request failed", {
|
|
16611
|
-
|
|
17536
|
+
errorClass: "WorkerLifecycleOperationError",
|
|
17537
|
+
errorCode: "worker_shutdown_request_failed",
|
|
17538
|
+
origin: "worker-lifecycle"
|
|
16612
17539
|
});
|
|
16613
17540
|
}
|
|
16614
17541
|
},
|