@opengeni/worker-bundle 0.16.17 → 0.16.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +83 -32
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/environment.d.ts +1 -0
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +6 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +93 -0
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +2 -0
- package/dist/activities/toolspace-token-renewal.d.ts +1 -1
- package/dist/activities/workspace-capture.d.ts +1 -0
- package/dist/activities.d.ts +3 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/host-export.js +1 -1
- package/dist/index.js +1465 -453
- package/dist/index.js.map +1 -1
- package/dist/sandbox-resume.d.ts +6 -0
- package/dist/sandbox-routing.d.ts +11 -6
- package/dist/workflow-bundle.js +2 -1
- package/package.json +12 -12
- package/src/activities/agent-turn.ts +559 -434
- package/src/activities/codex-capacity.ts +10 -1
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +9 -5
- package/src/activities/environment.ts +13 -24
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +108 -7
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +892 -0
- package/src/activities/rig-verification.ts +13 -34
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +18 -4
- package/src/activities/sandbox-lease.ts +5 -2
- package/src/activities/scheduled-tasks.ts +19 -3
- package/src/activities/toolspace-token-renewal.ts +9 -2
- package/src/activities/workspace-capture.ts +33 -3
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/observability-metrics.ts +3 -3
- package/src/sandbox-resume.ts +45 -12
- package/src/sandbox-routing.ts +35 -17
- package/src/worker-service-lifecycle.ts +8 -4
- package/src/workflows/file-upload-reaper.ts +4 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
getCodexRotationSettings,
|
|
28
28
|
getWorkspaceModelPolicy,
|
|
29
29
|
getWorkspaceGrant,
|
|
30
|
+
getWorkspace,
|
|
30
31
|
listCodexAccountStatuses,
|
|
31
32
|
fetchCodexUsageForAccount,
|
|
32
33
|
getSessionCodexState,
|
|
@@ -89,9 +90,10 @@ import {
|
|
|
89
90
|
sandboxStateEntryFromRunState,
|
|
90
91
|
maxTurnsExceededRunState,
|
|
91
92
|
modelResponseServiceTierFromSdkEvent,
|
|
92
|
-
|
|
93
|
+
modelTerminalResponseFromSdkEvent,
|
|
93
94
|
normalizeModelCallUsage,
|
|
94
95
|
normalizeSdkEvent,
|
|
96
|
+
normalizeProtocolJsonValue,
|
|
95
97
|
projectHistoryForProvider,
|
|
96
98
|
sanitizeHistoryItemsForModel,
|
|
97
99
|
projectModelInputForCapabilities,
|
|
@@ -126,6 +128,7 @@ import {
|
|
|
126
128
|
runOwnedSandboxSetup,
|
|
127
129
|
RoutingMutationOutcomeUnknownError,
|
|
128
130
|
WorkspaceArchiveIntegrityError,
|
|
131
|
+
sdkBackendIdForSandboxBackend,
|
|
129
132
|
swapTargetEstablishability,
|
|
130
133
|
type SandboxFileDownload,
|
|
131
134
|
type SandboxFileDownloadFailure,
|
|
@@ -136,7 +139,6 @@ import {
|
|
|
136
139
|
type ModelCallUsageNormalization,
|
|
137
140
|
type BuildAgentOptions,
|
|
138
141
|
type ConnectorActionPolicyHooks,
|
|
139
|
-
type RunAgentStreamOptions,
|
|
140
142
|
type TurnToolCancellationFence,
|
|
141
143
|
type BackendUnresolvableCode,
|
|
142
144
|
type EstablishedSandboxSession,
|
|
@@ -262,19 +264,12 @@ import {
|
|
|
262
264
|
import { withFirstPartyTools } from "./goals";
|
|
263
265
|
import {
|
|
264
266
|
mergeRigDefaultVariableSetEnvironment,
|
|
265
|
-
resolveWorkspaceAgentInstructions,
|
|
266
267
|
resolveWorkspacePackRuntime,
|
|
267
268
|
resolveWorkspaceSkillLibraryRuntime,
|
|
268
269
|
settingsWithPackSandboxImage,
|
|
269
270
|
settingsWithRigImage,
|
|
270
271
|
} from "./packs";
|
|
271
272
|
import { deliverFailedChildTurnToParent } from "./parent-wake";
|
|
272
|
-
import {
|
|
273
|
-
createSecretRedactor,
|
|
274
|
-
isCredentialHeaderName,
|
|
275
|
-
redactSensitiveText,
|
|
276
|
-
type SecretForRedaction,
|
|
277
|
-
} from "./redaction";
|
|
278
273
|
import {
|
|
279
274
|
createModelHistoryAttachmentProjector,
|
|
280
275
|
turnInput,
|
|
@@ -322,6 +317,8 @@ import {
|
|
|
322
317
|
recordCreditMicros,
|
|
323
318
|
recordModelCacheTokens,
|
|
324
319
|
recordModelInputTokens,
|
|
320
|
+
recordSessionEventAppendLatency,
|
|
321
|
+
recordSessionEventPublishLatency,
|
|
325
322
|
runtimeMetricsHooksForObservability,
|
|
326
323
|
StreamTimingMetrics,
|
|
327
324
|
turnLifecycleMetricsFor,
|
|
@@ -333,6 +330,16 @@ import {
|
|
|
333
330
|
prepareRecordingForSettlement,
|
|
334
331
|
type ActiveRecording,
|
|
335
332
|
} from "./recording";
|
|
333
|
+
import {
|
|
334
|
+
collectRetainedScreenshotReceipts,
|
|
335
|
+
collectRetainedScreenshotRunStateReceipts,
|
|
336
|
+
compactRetainedScreenshotHistory,
|
|
337
|
+
compactRetainedScreenshotRunState,
|
|
338
|
+
materializeRetainedScreenshotHistory,
|
|
339
|
+
materializeRetainedScreenshotRunState,
|
|
340
|
+
retainComputerScreenshot,
|
|
341
|
+
typedScreenshotFromSdkEvent,
|
|
342
|
+
} from "./retained-screenshots";
|
|
336
343
|
import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "./workspace-capture";
|
|
337
344
|
import type { ChannelASession } from "@opengeni/runtime/sandbox";
|
|
338
345
|
import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
|
|
@@ -346,9 +353,11 @@ import {
|
|
|
346
353
|
DEFAULT_FIRST_PARTY_MCP_TOOLS,
|
|
347
354
|
evaluateWorkspaceModelPolicy,
|
|
348
355
|
readTurnExecutionPolicyV1,
|
|
356
|
+
resolveWorkspaceAgentHumanInputEnabled,
|
|
349
357
|
resourceMountPath,
|
|
350
358
|
type LatencyMode,
|
|
351
359
|
type ResourceRef,
|
|
360
|
+
type RetainedArtifactMetadata,
|
|
352
361
|
type SessionEvent,
|
|
353
362
|
type SessionEventType,
|
|
354
363
|
type SessionStatus,
|
|
@@ -363,6 +372,21 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
363
372
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
364
373
|
const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
|
|
365
374
|
|
|
375
|
+
export class WorkspaceHumanInputDisabledError extends Error {
|
|
376
|
+
constructor(kind: "resume" | "interruption") {
|
|
377
|
+
super(`Workspace policy rejects structured human-input ${kind}`);
|
|
378
|
+
this.name = "WorkspaceHumanInputDisabledError";
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
export function assertWorkspaceHumanInputAllowed(
|
|
383
|
+
enabled: boolean,
|
|
384
|
+
kind: "resume" | "interruption",
|
|
385
|
+
attempted: boolean,
|
|
386
|
+
): void {
|
|
387
|
+
if (!enabled && attempted) throw new WorkspaceHumanInputDisabledError(kind);
|
|
388
|
+
}
|
|
389
|
+
|
|
366
390
|
/** Broad personal lookup is allowed only for a direct human/API command. */
|
|
367
391
|
export function credentialSubjectIdForTurnInitiator(
|
|
368
392
|
turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
|
|
@@ -830,6 +854,33 @@ export async function waitForTurnStreamCleanup(
|
|
|
830
854
|
await waitForTurnFinalizerStep(providerCompleted, signal);
|
|
831
855
|
}
|
|
832
856
|
|
|
857
|
+
/**
|
|
858
|
+
* Terminal settlement closes the active attempt before turn finalization. The
|
|
859
|
+
* ordinary workspace admission therefore correctly returns `attempt_fenced`
|
|
860
|
+
* for the final attempt-qualified credential deletion. Retry only that exact
|
|
861
|
+
* settled-attempt case directly; every other fence/status and the direct
|
|
862
|
+
* deletion itself remain fail-closed.
|
|
863
|
+
*/
|
|
864
|
+
export async function clearAttemptCredentialsWithSettledFence(input: {
|
|
865
|
+
activityStatus: RunAgentTurnResult["status"] | "unknown";
|
|
866
|
+
runWorkspaceFencedClear: () => Promise<void>;
|
|
867
|
+
clearExactAttempt: () => Promise<void>;
|
|
868
|
+
onSettledAttemptFence: () => void;
|
|
869
|
+
}): Promise<void> {
|
|
870
|
+
try {
|
|
871
|
+
await input.runWorkspaceFencedClear();
|
|
872
|
+
} catch (error) {
|
|
873
|
+
const settledAttemptFence =
|
|
874
|
+
error instanceof Error &&
|
|
875
|
+
error.name === "SandboxWorkspaceMutationFencedError" &&
|
|
876
|
+
(error as Error & { code?: unknown }).code === "attempt_fenced" &&
|
|
877
|
+
(input.activityStatus === "idle" || input.activityStatus === "failed");
|
|
878
|
+
if (!settledAttemptFence) throw error;
|
|
879
|
+
input.onSettledAttemptFence();
|
|
880
|
+
await input.clearExactAttempt();
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
|
|
833
884
|
function turnFinalizerCancellationSignal(
|
|
834
885
|
temporalSignal: AbortSignal | undefined,
|
|
835
886
|
activityStatus: RunAgentTurnResult["status"] | "unknown",
|
|
@@ -846,84 +897,94 @@ function compactionFailureReason(reason: string): string {
|
|
|
846
897
|
: `compaction summarization failed: ${reason}`;
|
|
847
898
|
}
|
|
848
899
|
|
|
900
|
+
export type MandatoryHistoryPersistenceStage = "history_append" | "sandbox_envelope";
|
|
901
|
+
|
|
902
|
+
/**
|
|
903
|
+
* Preserve the exact storage failure for the permission-controlled turn event
|
|
904
|
+
* while attaching only a stable, non-secret operation token to public worker
|
|
905
|
+
* diagnostics. Mandatory history failures remain terminal and never inherit a
|
|
906
|
+
* provider-retry classification from coincidental source text or status.
|
|
907
|
+
*/
|
|
908
|
+
export class MandatoryHistoryPersistenceError extends Error {
|
|
909
|
+
readonly name = "MandatoryHistoryPersistenceError";
|
|
910
|
+
readonly cause: unknown;
|
|
911
|
+
|
|
912
|
+
constructor(
|
|
913
|
+
readonly stage: MandatoryHistoryPersistenceStage,
|
|
914
|
+
cause: unknown,
|
|
915
|
+
) {
|
|
916
|
+
super(cause instanceof Error ? cause.message : String(cause));
|
|
917
|
+
this.cause = cause;
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
export async function runMandatoryHistoryPersistenceStep<T>(
|
|
922
|
+
stage: MandatoryHistoryPersistenceStage,
|
|
923
|
+
operation: () => Promise<T>,
|
|
924
|
+
): Promise<T> {
|
|
925
|
+
try {
|
|
926
|
+
return await operation();
|
|
927
|
+
} catch (cause) {
|
|
928
|
+
throw new MandatoryHistoryPersistenceError(stage, cause);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
|
|
849
932
|
export type SafeErrorDiagnostic = {
|
|
850
|
-
|
|
851
|
-
|
|
933
|
+
errorClass: "WorkerOperationError";
|
|
934
|
+
errorCode: "worker_operation_failed";
|
|
852
935
|
status?: number;
|
|
853
|
-
|
|
936
|
+
origin: "worker";
|
|
937
|
+
historyPersistenceStage?: MandatoryHistoryPersistenceStage;
|
|
854
938
|
};
|
|
855
939
|
|
|
856
940
|
/**
|
|
857
941
|
* Produce the only exception shape allowed in worker logs. It deliberately
|
|
858
|
-
* excludes stack, cause, response/request
|
|
859
|
-
* properties
|
|
942
|
+
* excludes the arbitrary source message, stack, cause, response/request
|
|
943
|
+
* bodies, and enumerable properties. Exact failure content belongs in the
|
|
944
|
+
* permission-controlled session event, not stdout or telemetry.
|
|
860
945
|
*/
|
|
861
|
-
export function safeErrorDiagnostic(
|
|
862
|
-
error: unknown,
|
|
863
|
-
redactText: (value: string) => string = redactSensitiveText,
|
|
864
|
-
): SafeErrorDiagnostic {
|
|
865
|
-
const rawName = error instanceof Error ? error.name : "Error";
|
|
866
|
-
const name = /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error";
|
|
867
|
-
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
946
|
+
export function safeErrorDiagnostic(error: unknown): SafeErrorDiagnostic {
|
|
868
947
|
const diagnostic: SafeErrorDiagnostic = {
|
|
869
|
-
|
|
870
|
-
|
|
948
|
+
errorClass: "WorkerOperationError",
|
|
949
|
+
errorCode: "worker_operation_failed",
|
|
950
|
+
origin: "worker",
|
|
871
951
|
};
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
878
|
-
diagnostic.status = status;
|
|
952
|
+
try {
|
|
953
|
+
let statusSource = error;
|
|
954
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
955
|
+
diagnostic.historyPersistenceStage = error.stage;
|
|
956
|
+
statusSource = error.cause;
|
|
879
957
|
}
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
958
|
+
if (statusSource && typeof statusSource === "object") {
|
|
959
|
+
const status = Number(
|
|
960
|
+
(statusSource as { status?: unknown; statusCode?: unknown }).status ??
|
|
961
|
+
(statusSource as { statusCode?: unknown }).statusCode,
|
|
962
|
+
);
|
|
963
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
964
|
+
diagnostic.status = status;
|
|
965
|
+
}
|
|
884
966
|
}
|
|
967
|
+
} catch {
|
|
968
|
+
// Public diagnostics are best-effort and must never replace the exact
|
|
969
|
+
// internal worker failure.
|
|
885
970
|
}
|
|
886
971
|
return diagnostic;
|
|
887
972
|
}
|
|
888
973
|
|
|
889
|
-
function safeErrorForTelemetry(error: unknown
|
|
890
|
-
const diagnostic = safeErrorDiagnostic(error
|
|
891
|
-
const safe = new Error(
|
|
892
|
-
|
|
974
|
+
function safeErrorForTelemetry(error: unknown): Error {
|
|
975
|
+
const diagnostic = safeErrorDiagnostic(error);
|
|
976
|
+
const safe = new Error("worker operation failed") as Error & {
|
|
977
|
+
code?: string;
|
|
978
|
+
status?: number;
|
|
979
|
+
origin?: string;
|
|
980
|
+
};
|
|
981
|
+
safe.name = "WorkerOperationError";
|
|
982
|
+
safe.code = diagnostic.errorCode;
|
|
983
|
+
if (diagnostic.status !== undefined) safe.status = diagnostic.status;
|
|
984
|
+
safe.origin = diagnostic.origin;
|
|
893
985
|
return safe;
|
|
894
986
|
}
|
|
895
987
|
|
|
896
|
-
export function headerSecretRedactions(
|
|
897
|
-
prefix: string,
|
|
898
|
-
headers: Readonly<Record<string, string>> | undefined,
|
|
899
|
-
credentialHeaderNames?: Iterable<string>,
|
|
900
|
-
): SecretForRedaction[] {
|
|
901
|
-
const discovered: SecretForRedaction[] = [];
|
|
902
|
-
const explicitCredentialHeaders = credentialHeaderNames
|
|
903
|
-
? new Set([...credentialHeaderNames].map((name) => name.toLowerCase()))
|
|
904
|
-
: null;
|
|
905
|
-
for (const [headerName, value] of Object.entries(headers ?? {})) {
|
|
906
|
-
if (
|
|
907
|
-
!explicitCredentialHeaders?.has(headerName.toLowerCase()) &&
|
|
908
|
-
!isCredentialHeaderName(headerName)
|
|
909
|
-
) {
|
|
910
|
-
continue;
|
|
911
|
-
}
|
|
912
|
-
const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
|
|
913
|
-
discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
|
|
914
|
-
if (/^(?:proxy-)?authorization$/i.test(headerName)) {
|
|
915
|
-
const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
|
|
916
|
-
if (credential) {
|
|
917
|
-
discovered.push({
|
|
918
|
-
name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
|
|
919
|
-
value: credential,
|
|
920
|
-
});
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
return discovered;
|
|
925
|
-
}
|
|
926
|
-
|
|
927
988
|
function compactionFailureReasonFromError(error: unknown): string {
|
|
928
989
|
if (
|
|
929
990
|
error instanceof CompactionProviderResponseError ||
|
|
@@ -1043,14 +1104,14 @@ function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): stri
|
|
|
1043
1104
|
* an earlier slice — the orphaned tool output that 400s the Responses API and
|
|
1044
1105
|
* bricks the session on every replay.
|
|
1045
1106
|
*
|
|
1046
|
-
* Defending:
|
|
1107
|
+
* Defending: structurally repair the full current history into an API-valid sequence (the
|
|
1047
1108
|
* same pure rules the read path uses), then append only the new tail beyond the
|
|
1048
1109
|
* watermark. A trailing dangling call is dropped here and re-evaluated next
|
|
1049
1110
|
* pass once its result lands, so a call and its result are written together at
|
|
1050
1111
|
* consecutive positions and a result is never persisted without its call. The
|
|
1051
|
-
* watermark advances to the
|
|
1112
|
+
* watermark advances to the repaired length — never past anything unwritten —
|
|
1052
1113
|
* so a non-monotonic history can never desync it. When previously-persisted
|
|
1053
|
-
* rows already exceed the
|
|
1114
|
+
* rows already exceed the repaired length (e.g. legacy orphans written before
|
|
1054
1115
|
* this fix), nothing new is appended and the watermark holds steady.
|
|
1055
1116
|
*/
|
|
1056
1117
|
/**
|
|
@@ -1123,10 +1184,9 @@ type TurnEventPublisher = (
|
|
|
1123
1184
|
immediate?: boolean,
|
|
1124
1185
|
) => Promise<{ events: SessionEvent[]; accepted: boolean }>;
|
|
1125
1186
|
|
|
1126
|
-
export type
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
lastProviderContextTokensObserved: number | null;
|
|
1187
|
+
export type ModelResponseEventState = {
|
|
1188
|
+
responseCount: number;
|
|
1189
|
+
contextSignal: { revision: number; totalTokens: number } | null;
|
|
1130
1190
|
claimedSourceKeys: Set<string>;
|
|
1131
1191
|
};
|
|
1132
1192
|
|
|
@@ -1135,13 +1195,12 @@ export type CompactionModelUsageEventState = {
|
|
|
1135
1195
|
claimedSourceKeys: Set<string>;
|
|
1136
1196
|
};
|
|
1137
1197
|
|
|
1138
|
-
export function
|
|
1198
|
+
export function createModelResponseEventState(
|
|
1139
1199
|
claimedSourceKeys: Set<string> = new Set<string>(),
|
|
1140
|
-
):
|
|
1200
|
+
): ModelResponseEventState {
|
|
1141
1201
|
return {
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
lastProviderContextTokensObserved: null,
|
|
1202
|
+
responseCount: 0,
|
|
1203
|
+
contextSignal: null,
|
|
1145
1204
|
claimedSourceKeys,
|
|
1146
1205
|
};
|
|
1147
1206
|
}
|
|
@@ -1152,15 +1211,10 @@ export function createCompactionModelUsageEventState(
|
|
|
1152
1211
|
return { usageCount: 0, claimedSourceKeys };
|
|
1153
1212
|
}
|
|
1154
1213
|
|
|
1155
|
-
export function
|
|
1156
|
-
state:
|
|
1214
|
+
export function modelResponseContextSignal(
|
|
1215
|
+
state: ModelResponseEventState,
|
|
1157
1216
|
): { revision: number; totalTokens: number } | null {
|
|
1158
|
-
return state.
|
|
1159
|
-
? null
|
|
1160
|
-
: {
|
|
1161
|
-
revision: state.providerContextRevision,
|
|
1162
|
-
totalTokens: state.lastProviderContextTokensObserved,
|
|
1163
|
-
};
|
|
1217
|
+
return state.contextSignal;
|
|
1164
1218
|
}
|
|
1165
1219
|
|
|
1166
1220
|
export function assertModelResponseLatencyMode(input: {
|
|
@@ -1195,19 +1249,20 @@ export function assertModelResponseLatencyMode(input: {
|
|
|
1195
1249
|
}
|
|
1196
1250
|
|
|
1197
1251
|
/**
|
|
1198
|
-
* Process one SDK
|
|
1252
|
+
* Process one SDK terminal-response event through the production authority path.
|
|
1199
1253
|
*
|
|
1200
1254
|
* The pinned Responses SDK mirrors one provider terminal response as both a
|
|
1201
1255
|
* normalized `response_done` and a raw `model/response.completed` event. Claim
|
|
1202
|
-
* the stable response/source key before lease renewal or any
|
|
1203
|
-
*
|
|
1204
|
-
*
|
|
1205
|
-
*
|
|
1206
|
-
*
|
|
1256
|
+
* the stable response/source key before lease renewal or any side effect, and
|
|
1257
|
+
* use that one positional ordinal for both response identity and same-run
|
|
1258
|
+
* context binding. A response without usage still clears attempt-owned token
|
|
1259
|
+
* state. When usage exists, the durable `agent.model.usage` source-key fence
|
|
1260
|
+
* remains the cross-restart authority: a replay may retry the idempotent billing
|
|
1261
|
+
* write, but it cannot advance metrics, context, or attempt-owned signals.
|
|
1207
1262
|
*/
|
|
1208
|
-
export async function
|
|
1209
|
-
event: Parameters<typeof
|
|
1210
|
-
state:
|
|
1263
|
+
export async function processModelResponseTerminalEvent(input: {
|
|
1264
|
+
event: Parameters<typeof modelTerminalResponseFromSdkEvent>[0];
|
|
1265
|
+
state: ModelResponseEventState;
|
|
1211
1266
|
dispatchId: string | null;
|
|
1212
1267
|
settings: Settings;
|
|
1213
1268
|
db: ActivityServices["db"];
|
|
@@ -1230,20 +1285,25 @@ export async function processModelResponseUsageEvent(input: {
|
|
|
1230
1285
|
renewLease: () => Promise<void>;
|
|
1231
1286
|
leaseLost: () => boolean;
|
|
1232
1287
|
leaseLostMessage: string;
|
|
1233
|
-
setLastInputTokens: (tokens: number) => Promise<void>;
|
|
1288
|
+
setLastInputTokens: (tokens: number | null) => Promise<void>;
|
|
1234
1289
|
}): Promise<
|
|
1235
|
-
| { status: "
|
|
1290
|
+
| { status: "not_response" }
|
|
1236
1291
|
| { status: "duplicate"; sourceKey: string }
|
|
1237
|
-
| {
|
|
1292
|
+
| {
|
|
1293
|
+
status: "processed";
|
|
1294
|
+
sourceKey: string;
|
|
1295
|
+
authoritative: boolean;
|
|
1296
|
+
usageReported: boolean;
|
|
1297
|
+
}
|
|
1238
1298
|
> {
|
|
1239
|
-
const
|
|
1240
|
-
if (!
|
|
1241
|
-
return { status: "
|
|
1299
|
+
const terminal = modelTerminalResponseFromSdkEvent(input.event);
|
|
1300
|
+
if (!terminal) {
|
|
1301
|
+
return { status: "not_response" };
|
|
1242
1302
|
}
|
|
1243
1303
|
|
|
1244
|
-
const responseOrdinal = input.state.
|
|
1304
|
+
const responseOrdinal = input.state.responseCount + 1;
|
|
1245
1305
|
const sourceKey = modelUsageSourceKey({
|
|
1246
|
-
responseId:
|
|
1306
|
+
responseId: terminal.responseId,
|
|
1247
1307
|
dispatchId: input.dispatchId,
|
|
1248
1308
|
positionalKey: `response-${responseOrdinal}`,
|
|
1249
1309
|
});
|
|
@@ -1251,20 +1311,25 @@ export async function processModelResponseUsageEvent(input: {
|
|
|
1251
1311
|
return { status: "duplicate", sourceKey };
|
|
1252
1312
|
}
|
|
1253
1313
|
input.state.claimedSourceKeys.add(sourceKey);
|
|
1254
|
-
input.state.
|
|
1314
|
+
input.state.responseCount = responseOrdinal;
|
|
1315
|
+
|
|
1316
|
+
const responseUsage = terminal.usage;
|
|
1255
1317
|
|
|
1256
|
-
const normalizedUsage = normalizeModelCallUsage(responseUsage
|
|
1318
|
+
const normalizedUsage = normalizeModelCallUsage(responseUsage?.usage);
|
|
1257
1319
|
const accountContext = modelCallAccountContext({
|
|
1258
1320
|
servingCredentialId: input.servingCredentialId,
|
|
1259
1321
|
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
1260
1322
|
isFirstCallOfTurn: responseOrdinal === 1,
|
|
1261
1323
|
});
|
|
1262
|
-
|
|
1324
|
+
// A terminal response without usage still authoritatively replaces the
|
|
1325
|
+
// attempt-owned context signal. There is simply no billing event to claim.
|
|
1326
|
+
let authoritative = responseUsage === null;
|
|
1263
1327
|
await recordCompletedModelCallBeforeOwnershipFences({
|
|
1264
1328
|
renewLease: input.renewLease,
|
|
1265
1329
|
leaseLost: input.leaseLost,
|
|
1266
1330
|
leaseLostMessage: input.leaseLostMessage,
|
|
1267
1331
|
recordUsage: async () => {
|
|
1332
|
+
if (!responseUsage) return;
|
|
1268
1333
|
const billing = await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
1269
1334
|
accountId: input.accountId,
|
|
1270
1335
|
workspaceId: input.workspaceId,
|
|
@@ -1322,17 +1387,22 @@ export async function processModelResponseUsageEvent(input: {
|
|
|
1322
1387
|
recordAttemptSignals: async () => {
|
|
1323
1388
|
if (!authoritative) return;
|
|
1324
1389
|
const observedTotal = normalizedUsage.totalTokens;
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1390
|
+
input.state.contextSignal =
|
|
1391
|
+
observedTotal !== null && observedTotal > 0
|
|
1392
|
+
? { revision: responseOrdinal, totalTokens: observedTotal }
|
|
1393
|
+
: null;
|
|
1329
1394
|
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1395
|
+
await input.setLastInputTokens(
|
|
1396
|
+
observedInput !== null && observedInput > 0 ? observedInput : null,
|
|
1397
|
+
);
|
|
1333
1398
|
},
|
|
1334
1399
|
});
|
|
1335
|
-
return {
|
|
1400
|
+
return {
|
|
1401
|
+
status: "processed",
|
|
1402
|
+
sourceKey,
|
|
1403
|
+
authoritative,
|
|
1404
|
+
usageReported: responseUsage !== null,
|
|
1405
|
+
};
|
|
1336
1406
|
}
|
|
1337
1407
|
|
|
1338
1408
|
/**
|
|
@@ -1508,14 +1578,9 @@ export async function emitModelCallUsage(input: {
|
|
|
1508
1578
|
if (!authoritative) return false;
|
|
1509
1579
|
try {
|
|
1510
1580
|
input.observability.info("model call usage", {
|
|
1511
|
-
accountId: input.accountId,
|
|
1512
|
-
workspaceId: input.workspaceId,
|
|
1513
|
-
sessionId: input.sessionId,
|
|
1514
|
-
turnId: input.turnId,
|
|
1515
1581
|
provider: input.provider,
|
|
1516
1582
|
providerApi: input.providerApi,
|
|
1517
1583
|
model: input.model,
|
|
1518
|
-
sourceKey: input.sourceKey,
|
|
1519
1584
|
inputTokens: telemetry.inputTokens,
|
|
1520
1585
|
outputTokens: telemetry.outputTokens,
|
|
1521
1586
|
cachedTokens: telemetry.cachedTokens,
|
|
@@ -1533,7 +1598,6 @@ export async function emitModelCallUsage(input: {
|
|
|
1533
1598
|
provider: input.provider,
|
|
1534
1599
|
providerApi: input.providerApi,
|
|
1535
1600
|
model: input.model,
|
|
1536
|
-
sourceKey: input.sourceKey,
|
|
1537
1601
|
rejectedFields: normalizedUsage.rejectedFields.join(","),
|
|
1538
1602
|
});
|
|
1539
1603
|
}
|
|
@@ -1580,16 +1644,13 @@ export function historyRowsToAppend(
|
|
|
1580
1644
|
// positions from 0) when callers do not pass an explicit next position.
|
|
1581
1645
|
nextPosition: number = persistedHistoryCount,
|
|
1582
1646
|
toolOutputTruncationTokens?: number,
|
|
1583
|
-
redactValue: (value: unknown) => unknown = (value) => value,
|
|
1584
1647
|
): {
|
|
1585
1648
|
rows: Array<{ position: number; item: Record<string, unknown> }>;
|
|
1586
1649
|
nextWatermark: number;
|
|
1587
1650
|
nextPosition: number;
|
|
1588
1651
|
} {
|
|
1589
|
-
const
|
|
1590
|
-
|
|
1591
|
-
);
|
|
1592
|
-
if (sanitized.length <= persistedHistoryCount) {
|
|
1652
|
+
const modelReady = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens);
|
|
1653
|
+
if (modelReady.length <= persistedHistoryCount) {
|
|
1593
1654
|
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
1594
1655
|
}
|
|
1595
1656
|
// Canonical model-facing inputs (including machine-input batches) are
|
|
@@ -1599,16 +1660,22 @@ export function historyRowsToAppend(
|
|
|
1599
1660
|
// notes—must not accidentally become conversation memory during reconciliation.
|
|
1600
1661
|
// Advance the in-memory watermark past them, but only allocate durable
|
|
1601
1662
|
// positions to actual model/tool output.
|
|
1602
|
-
const rows =
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1663
|
+
const rows: Array<{ position: number; item: Record<string, unknown> }> = [];
|
|
1664
|
+
for (const [offset, item] of modelReady.slice(persistedHistoryCount).entries()) {
|
|
1665
|
+
if (item.type === "message" && item.role === "system") {
|
|
1666
|
+
continue;
|
|
1667
|
+
}
|
|
1668
|
+
rows.push({
|
|
1669
|
+
position: nextPosition + rows.length,
|
|
1670
|
+
item: normalizeProtocolJsonValue(
|
|
1671
|
+
item as Record<string, unknown>,
|
|
1672
|
+
`$[${persistedHistoryCount + offset}]`,
|
|
1673
|
+
),
|
|
1674
|
+
});
|
|
1675
|
+
}
|
|
1609
1676
|
return {
|
|
1610
1677
|
rows,
|
|
1611
|
-
nextWatermark:
|
|
1678
|
+
nextWatermark: modelReady.length,
|
|
1612
1679
|
nextPosition: nextPosition + rows.length,
|
|
1613
1680
|
};
|
|
1614
1681
|
}
|
|
@@ -1650,16 +1717,7 @@ export function selectRejectedProviderArtifactHistoryIds(
|
|
|
1650
1717
|
}
|
|
1651
1718
|
|
|
1652
1719
|
/** Literal final per-call boundary for replayed and current-turn model input. */
|
|
1653
|
-
export function
|
|
1654
|
-
redactValue: (value: unknown) => unknown,
|
|
1655
|
-
): NonNullable<RunAgentStreamOptions["callModelInputFilter"]> {
|
|
1656
|
-
return ({ modelData }) => ({
|
|
1657
|
-
...modelData,
|
|
1658
|
-
input: redactValue(modelData.input) as typeof modelData.input,
|
|
1659
|
-
});
|
|
1660
|
-
}
|
|
1661
|
-
|
|
1662
|
-
function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
1720
|
+
export function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
1663
1721
|
if (item.type === "message") {
|
|
1664
1722
|
return item.role === "assistant";
|
|
1665
1723
|
}
|
|
@@ -2272,10 +2330,6 @@ export function createTurnSandboxProvisioner<T>(
|
|
|
2272
2330
|
};
|
|
2273
2331
|
}
|
|
2274
2332
|
|
|
2275
|
-
function sdkBackendIdForSandboxBackend(backend: Settings["sandboxBackend"]): string {
|
|
2276
|
-
return backend === "local" ? "unix_local" : backend;
|
|
2277
|
-
}
|
|
2278
|
-
|
|
2279
2333
|
/**
|
|
2280
2334
|
* Decide whether THIS turn may send OpenAI's `prompt_cache_key` request field.
|
|
2281
2335
|
*
|
|
@@ -2365,6 +2419,34 @@ export function codexCredentialLeaseDeadlineExpired(
|
|
|
2365
2419
|
);
|
|
2366
2420
|
}
|
|
2367
2421
|
|
|
2422
|
+
type OpStreamFinalizer = {
|
|
2423
|
+
finalizeOpStreamOps(): Promise<void>;
|
|
2424
|
+
};
|
|
2425
|
+
|
|
2426
|
+
/** Release retained Connected Machine output only after the turn is durable. */
|
|
2427
|
+
export async function finalizeDurableTurnOpStreams(
|
|
2428
|
+
sessions: readonly unknown[],
|
|
2429
|
+
fallback: OpStreamFinalizer | null,
|
|
2430
|
+
): Promise<void> {
|
|
2431
|
+
const candidates = new Set<OpStreamFinalizer>();
|
|
2432
|
+
for (const session of sessions) {
|
|
2433
|
+
const candidate = session as Partial<OpStreamFinalizer> | null;
|
|
2434
|
+
if (typeof candidate?.finalizeOpStreamOps === "function") {
|
|
2435
|
+
candidates.add(candidate as OpStreamFinalizer);
|
|
2436
|
+
}
|
|
2437
|
+
}
|
|
2438
|
+
if (candidates.size === 0 && fallback) {
|
|
2439
|
+
candidates.add(fallback);
|
|
2440
|
+
}
|
|
2441
|
+
for (const candidate of candidates) {
|
|
2442
|
+
try {
|
|
2443
|
+
await candidate.finalizeOpStreamOps();
|
|
2444
|
+
} catch {
|
|
2445
|
+
// The runner's retention TTL owns the fallback.
|
|
2446
|
+
}
|
|
2447
|
+
}
|
|
2448
|
+
}
|
|
2449
|
+
|
|
2368
2450
|
export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
|
|
2369
2451
|
return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
|
|
2370
2452
|
const {
|
|
@@ -2423,6 +2505,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2423
2505
|
acknowledgeQuiescence = true;
|
|
2424
2506
|
noteCancellationRequested();
|
|
2425
2507
|
};
|
|
2508
|
+
const acknowledgeRecoveryQuiescence = (): void => {
|
|
2509
|
+
// requestSessionTurnRecovery closed this exact attempt and durably
|
|
2510
|
+
// recorded the replacement cause. The activity still owns the physical
|
|
2511
|
+
// tool/credential boundary until finally drains it and publishes the
|
|
2512
|
+
// exact quiescence receipt; a workflow result alone is never that proof.
|
|
2513
|
+
acknowledgeQuiescence = true;
|
|
2514
|
+
noteCancellationRequested();
|
|
2515
|
+
};
|
|
2426
2516
|
let turnId: string | undefined;
|
|
2427
2517
|
let triggerEventId: string | undefined;
|
|
2428
2518
|
const claimedResult = (
|
|
@@ -2454,7 +2544,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2454
2544
|
// Still required by credential-loss/capacity settlements, whose own
|
|
2455
2545
|
// recovery transactions fence against worker-death redispatches.
|
|
2456
2546
|
let redispatchesAtDispatch = 0;
|
|
2457
|
-
const setLastInputTokensFenced = async (lastInputTokens: number): Promise<void> => {
|
|
2547
|
+
const setLastInputTokensFenced = async (lastInputTokens: number | null): Promise<void> => {
|
|
2458
2548
|
if (!turnId || executionGeneration <= 0) {
|
|
2459
2549
|
throw new Error("Turn attempt was not initialized before token accounting");
|
|
2460
2550
|
}
|
|
@@ -2556,7 +2646,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2556
2646
|
workspaceId: input.workspaceId,
|
|
2557
2647
|
turnId,
|
|
2558
2648
|
reason,
|
|
2559
|
-
|
|
2649
|
+
...safeErrorDiagnostic(error),
|
|
2560
2650
|
});
|
|
2561
2651
|
observability.incrementCounter({
|
|
2562
2652
|
name: "opengeni_codex_lease_renewals_total",
|
|
@@ -2588,8 +2678,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2588
2678
|
return resolvedSandbox;
|
|
2589
2679
|
};
|
|
2590
2680
|
// The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
|
|
2591
|
-
// routing proxy)
|
|
2592
|
-
//
|
|
2681
|
+
// routing proxy). Kept as a fallback finalizer; the routing proxy normally
|
|
2682
|
+
// aggregates it together with every machine reached after a mid-turn swap.
|
|
2593
2683
|
let machinePrimarySession: import("@opengeni/runtime").SelfhostedSession | null = null;
|
|
2594
2684
|
let lazyOwnedSandbox: EstablishedSandboxSession | null = null;
|
|
2595
2685
|
let turnSandboxProvisioner: TurnSandboxProvisioner<ResumedTurnSandbox> | null = null;
|
|
@@ -2598,6 +2688,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2598
2688
|
// THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
|
|
2599
2689
|
// connected machine (the user's real computer).
|
|
2600
2690
|
let setupBoxSession: unknown = null;
|
|
2691
|
+
const finalizeTurnOpStreamOps = async (): Promise<void> => {
|
|
2692
|
+
await finalizeDurableTurnOpStreams(
|
|
2693
|
+
[lazyOwnedSandbox?.session, resolvedSandbox?.established.session],
|
|
2694
|
+
machinePrimarySession,
|
|
2695
|
+
);
|
|
2696
|
+
};
|
|
2601
2697
|
// A same-target API repair can replace the home provider while this turn is
|
|
2602
2698
|
// alive. Keep setup/snapshot persistence on the rebound raw session while
|
|
2603
2699
|
// preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
|
|
@@ -2945,7 +3041,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2945
3041
|
sessionId: input.sessionId,
|
|
2946
3042
|
sandboxGroupId: heartbeatGroupId,
|
|
2947
3043
|
leaseEpoch: heartbeatEpoch,
|
|
2948
|
-
|
|
3044
|
+
...safeErrorDiagnostic(error),
|
|
2949
3045
|
});
|
|
2950
3046
|
})
|
|
2951
3047
|
.finally(() => {
|
|
@@ -3098,6 +3194,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3098
3194
|
// stable and this scalar append watermark is valid again. The sanitizer
|
|
3099
3195
|
// remains the final call/result pairing guard for every other reconcile.
|
|
3100
3196
|
let persistedHistoryCount = 0;
|
|
3197
|
+
const retainedScreenshotReceiptsByCallId = new Map<string, RetainedArtifactMetadata>();
|
|
3198
|
+
const computerScreenshotCallIds = new Set<string>();
|
|
3199
|
+
const materializeScreenshotHistory = async (history: Array<Record<string, unknown>>) => {
|
|
3200
|
+
collectRetainedScreenshotReceipts(history, retainedScreenshotReceiptsByCallId);
|
|
3201
|
+
return await materializeRetainedScreenshotHistory({
|
|
3202
|
+
db,
|
|
3203
|
+
objectStorage,
|
|
3204
|
+
workspaceId: input.workspaceId,
|
|
3205
|
+
sessionId: input.sessionId,
|
|
3206
|
+
history,
|
|
3207
|
+
});
|
|
3208
|
+
};
|
|
3209
|
+
const materializeScreenshotRunState = async (serialized: string) => {
|
|
3210
|
+
collectRetainedScreenshotRunStateReceipts(serialized, retainedScreenshotReceiptsByCallId);
|
|
3211
|
+
return await materializeRetainedScreenshotRunState({
|
|
3212
|
+
db,
|
|
3213
|
+
objectStorage,
|
|
3214
|
+
workspaceId: input.workspaceId,
|
|
3215
|
+
sessionId: input.sessionId,
|
|
3216
|
+
serialized,
|
|
3217
|
+
});
|
|
3218
|
+
};
|
|
3101
3219
|
let providerArtifactCandidates: Awaited<
|
|
3102
3220
|
ReturnType<typeof turnInput>
|
|
3103
3221
|
>["providerArtifactCandidates"] = {
|
|
@@ -3115,15 +3233,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3115
3233
|
if (!stream || !turnId) {
|
|
3116
3234
|
return;
|
|
3117
3235
|
}
|
|
3236
|
+
const durableTurnId = turnId;
|
|
3118
3237
|
try {
|
|
3119
3238
|
const rawHistory = (stream.state as { history?: unknown[] }).history;
|
|
3120
3239
|
if (Array.isArray(rawHistory)) {
|
|
3121
|
-
const
|
|
3240
|
+
const durableHistory = compactRetainedScreenshotHistory(
|
|
3122
3241
|
rawHistory as Array<Record<string, unknown>>,
|
|
3242
|
+
retainedScreenshotReceiptsByCallId,
|
|
3243
|
+
);
|
|
3244
|
+
const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
|
|
3245
|
+
durableHistory,
|
|
3123
3246
|
persistedHistoryCount,
|
|
3124
3247
|
nextHistoryPosition,
|
|
3125
3248
|
modelRunSettings.modelToolOutputTruncationTokens,
|
|
3126
|
-
(value) => redact(value),
|
|
3127
3249
|
);
|
|
3128
3250
|
const hasModelOrToolProgress = rows.some((row) =>
|
|
3129
3251
|
isModelOrToolProgressHistoryItem(row.item),
|
|
@@ -3131,21 +3253,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3131
3253
|
const shouldAppendRows =
|
|
3132
3254
|
rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
|
|
3133
3255
|
if (shouldAppendRows) {
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3256
|
+
await runMandatoryHistoryPersistenceStep("history_append", async () => {
|
|
3257
|
+
const appended = await appendSessionHistoryItems(db, {
|
|
3258
|
+
accountId: input.accountId,
|
|
3259
|
+
workspaceId: input.workspaceId,
|
|
3260
|
+
sessionId: input.sessionId,
|
|
3261
|
+
turnId: durableTurnId,
|
|
3262
|
+
expectedExecutionGeneration: executionGeneration,
|
|
3263
|
+
expectedAttemptId: input.attemptId,
|
|
3264
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
3265
|
+
items: rows,
|
|
3266
|
+
});
|
|
3267
|
+
if (!appended) {
|
|
3268
|
+
throw new TurnAttemptFencedError(
|
|
3269
|
+
"turn execution generation was fenced while saving conversation history",
|
|
3270
|
+
);
|
|
3271
|
+
}
|
|
3143
3272
|
});
|
|
3144
|
-
if (!appended) {
|
|
3145
|
-
throw new TurnAttemptFencedError(
|
|
3146
|
-
"turn execution generation was fenced while saving conversation history",
|
|
3147
|
-
);
|
|
3148
|
-
}
|
|
3149
3273
|
}
|
|
3150
3274
|
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
3151
3275
|
persistedHistoryCount = nextWatermark;
|
|
@@ -3154,37 +3278,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3154
3278
|
}
|
|
3155
3279
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
3156
3280
|
if (envelope) {
|
|
3157
|
-
await
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3281
|
+
await runMandatoryHistoryPersistenceStep("sandbox_envelope", () =>
|
|
3282
|
+
upsertSandboxSessionEnvelope(db, {
|
|
3283
|
+
accountId: input.accountId,
|
|
3284
|
+
workspaceId: input.workspaceId,
|
|
3285
|
+
sessionId: input.sessionId,
|
|
3286
|
+
envelope,
|
|
3287
|
+
}),
|
|
3288
|
+
);
|
|
3163
3289
|
}
|
|
3164
3290
|
} catch (persistError) {
|
|
3165
3291
|
console.error(
|
|
3166
3292
|
"session history dual-write failed (run unaffected)",
|
|
3167
|
-
safeErrorDiagnostic(persistError
|
|
3293
|
+
safeErrorDiagnostic(persistError),
|
|
3168
3294
|
);
|
|
3169
3295
|
if (options.requireDurable) throw persistError;
|
|
3170
3296
|
}
|
|
3171
3297
|
};
|
|
3172
|
-
// Reassigned after the variable set loads; the publish closure is
|
|
3173
|
-
// created (and used for turn.started) before the variableSet is available.
|
|
3174
|
-
// Generic auth/cookie/signed-URL/token classification is active before any
|
|
3175
|
-
// exact run-secret provenance is available. Registered values strengthen
|
|
3176
|
-
// this boundary as credentials are resolved and renewed during the turn.
|
|
3177
|
-
let redact: (payload: unknown) => unknown = createSecretRedactor([]);
|
|
3178
|
-
const secretRedactions = new Map<string, string>();
|
|
3179
3298
|
const publishedRunCredentialNotices = new Set<string>();
|
|
3180
|
-
|
|
3181
|
-
for (const secret of secrets) {
|
|
3182
|
-
if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
|
|
3183
|
-
}
|
|
3184
|
-
redact = createSecretRedactor(
|
|
3185
|
-
[...secretRedactions].map(([value, name]) => ({ name, value })),
|
|
3186
|
-
);
|
|
3187
|
-
};
|
|
3299
|
+
|
|
3188
3300
|
let variableSetId = "";
|
|
3189
3301
|
// Rig telemetry (M3): set once the session loads; empty string for a rig-less
|
|
3190
3302
|
// turn (mirrors variableSetId). Read by the activity span's finally block.
|
|
@@ -3371,7 +3483,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3371
3483
|
) => {
|
|
3372
3484
|
const inputs = events.map((event) => ({
|
|
3373
3485
|
...event,
|
|
3374
|
-
payload:
|
|
3486
|
+
payload: event.payload,
|
|
3375
3487
|
turnId: turnId!,
|
|
3376
3488
|
producerId,
|
|
3377
3489
|
producerSeq: ++producerSeq,
|
|
@@ -3385,6 +3497,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3385
3497
|
executionGeneration,
|
|
3386
3498
|
input.attemptId,
|
|
3387
3499
|
inputs,
|
|
3500
|
+
{
|
|
3501
|
+
onAppend: ({ durationSeconds }) =>
|
|
3502
|
+
recordSessionEventAppendLatency(observability, { durationSeconds }),
|
|
3503
|
+
onPublish: ({ durationSeconds }) =>
|
|
3504
|
+
recordSessionEventPublishLatency(observability, { durationSeconds }),
|
|
3505
|
+
},
|
|
3388
3506
|
);
|
|
3389
3507
|
if (inputs.length > 0 && !appended.accepted) {
|
|
3390
3508
|
throw new TurnAttemptFencedError("turn execution generation was fenced");
|
|
@@ -3439,13 +3557,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3439
3557
|
: undefined;
|
|
3440
3558
|
const inputs = inputSettlement.events.map((event) => ({
|
|
3441
3559
|
...event,
|
|
3442
|
-
payload:
|
|
3560
|
+
payload: event.payload,
|
|
3443
3561
|
turnId: turnId!,
|
|
3444
3562
|
producerId,
|
|
3445
3563
|
producerSeq: ++producerSeq,
|
|
3446
3564
|
}));
|
|
3447
|
-
const
|
|
3448
|
-
?
|
|
3565
|
+
const runState = inputSettlement.runState
|
|
3566
|
+
? {
|
|
3567
|
+
...inputSettlement.runState,
|
|
3568
|
+
serializedRunState: compactRetainedScreenshotRunState(
|
|
3569
|
+
inputSettlement.runState.serializedRunState,
|
|
3570
|
+
retainedScreenshotReceiptsByCallId,
|
|
3571
|
+
),
|
|
3572
|
+
}
|
|
3449
3573
|
: undefined;
|
|
3450
3574
|
const result = await applySessionTurnSettlement(db, input.workspaceId, {
|
|
3451
3575
|
sessionId: input.sessionId,
|
|
@@ -3456,7 +3580,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3456
3580
|
sessionStatus: inputSettlement.sessionStatus,
|
|
3457
3581
|
activeTurnId: inputSettlement.activeTurnId,
|
|
3458
3582
|
events: inputs,
|
|
3459
|
-
...(
|
|
3583
|
+
...(runState ? { runState } : {}),
|
|
3460
3584
|
...(recordingMutation ? { recording: recordingMutation } : {}),
|
|
3461
3585
|
...(compactionRequestFailure ? { compactionRequestFailure } : {}),
|
|
3462
3586
|
});
|
|
@@ -3814,10 +3938,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3814
3938
|
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
|
|
3815
3939
|
});
|
|
3816
3940
|
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
3817
|
-
workspaceId: input.workspaceId,
|
|
3818
3941
|
stage: shadowResult.stage,
|
|
3819
3942
|
reason: shadowResult.reason,
|
|
3820
|
-
|
|
3943
|
+
errorClass: "CodexFleetShadowOperationError",
|
|
3944
|
+
errorCode: "codex_fleet_shadow_failed",
|
|
3945
|
+
origin: "worker",
|
|
3821
3946
|
payloadBytes: shadowResult.payloadBytes,
|
|
3822
3947
|
});
|
|
3823
3948
|
}
|
|
@@ -4167,15 +4292,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4167
4292
|
attemptId: input.attemptId,
|
|
4168
4293
|
executionGeneration: turn.executionGeneration,
|
|
4169
4294
|
};
|
|
4170
|
-
const [
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4295
|
+
const [workspace, instructionPolicySnapshot, preferenceSnapshot] = await Promise.all([
|
|
4296
|
+
getWorkspace(db, input.workspaceId),
|
|
4297
|
+
getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
|
|
4298
|
+
getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
|
|
4299
|
+
if (error instanceof PreferenceRegistryInitiatorError) return null;
|
|
4300
|
+
throw error;
|
|
4301
|
+
}),
|
|
4302
|
+
]);
|
|
4303
|
+
if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
4304
|
+
const workspaceAgentInstructions = workspace.agentInstructions;
|
|
4305
|
+
const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
|
|
4306
|
+
assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
|
|
4179
4307
|
const workspaceGovernance = renderWorkspaceGovernanceContext({
|
|
4180
4308
|
instructionPolicy: instructionPolicySnapshot,
|
|
4181
4309
|
preferences: preferenceSnapshot,
|
|
@@ -4200,33 +4328,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4200
4328
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
4201
4329
|
sandboxBackend: turn.sandboxBackend,
|
|
4202
4330
|
};
|
|
4203
|
-
const sessionMcpHeaderNames = new Map<string, readonly string[]>();
|
|
4204
4331
|
const runSettings = await settingsWithSessionMcpServersForRun(
|
|
4205
4332
|
db,
|
|
4206
4333
|
input.workspaceId,
|
|
4207
4334
|
input.sessionId,
|
|
4208
4335
|
input.attemptId,
|
|
4209
4336
|
baseRunSettings,
|
|
4210
|
-
{
|
|
4211
|
-
onResolvedServers: (servers) => {
|
|
4212
|
-
for (const server of servers) {
|
|
4213
|
-
sessionMcpHeaderNames.set(server.id, server.headerNames);
|
|
4214
|
-
}
|
|
4215
|
-
},
|
|
4216
|
-
},
|
|
4217
4337
|
);
|
|
4218
|
-
|
|
4219
|
-
...(runSettings.accessKey
|
|
4220
|
-
? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }]
|
|
4221
|
-
: []),
|
|
4222
|
-
...runSettings.mcpServers.flatMap((server) => {
|
|
4223
|
-
return headerSecretRedactions(
|
|
4224
|
-
`MCP_${server.id.toUpperCase()}_STATIC`,
|
|
4225
|
-
server.headers,
|
|
4226
|
-
sessionMcpHeaderNames.get(server.id),
|
|
4227
|
-
);
|
|
4228
|
-
}),
|
|
4229
|
-
]);
|
|
4338
|
+
|
|
4230
4339
|
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
4231
4340
|
// hosted web search, encrypted reasoning, context window) the agent and
|
|
4232
4341
|
// compaction summarizer must use; null falls back to the legacy global
|
|
@@ -4257,49 +4366,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4257
4366
|
projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
|
|
4258
4367
|
const compactionModelHistoryProjector = async (items: Array<Record<string, unknown>>) =>
|
|
4259
4368
|
await modelHistoryProjector(projectHistoryForProvider(items, providerApi));
|
|
4260
|
-
const selectedProvider = resolvedModel?.provider;
|
|
4261
|
-
const publicProviderHeaders = new Set(
|
|
4262
|
-
selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? [],
|
|
4263
|
-
);
|
|
4264
|
-
const publicProviderQueries = new Set(
|
|
4265
|
-
selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? [],
|
|
4266
|
-
);
|
|
4267
|
-
const fallbackProviderApiKey =
|
|
4268
|
-
capabilitySettings.openaiProvider === "azure"
|
|
4269
|
-
? (capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken)
|
|
4270
|
-
: capabilitySettings.openaiApiKey;
|
|
4271
|
-
registerSecretRedactions([
|
|
4272
|
-
...(selectedProvider?.apiKey
|
|
4273
|
-
? [
|
|
4274
|
-
{
|
|
4275
|
-
name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
|
|
4276
|
-
value: selectedProvider.apiKey,
|
|
4277
|
-
},
|
|
4278
|
-
]
|
|
4279
|
-
: !resolvedModel && fallbackProviderApiKey
|
|
4280
|
-
? [
|
|
4281
|
-
{
|
|
4282
|
-
name: "MODEL_PROVIDER_API_KEY",
|
|
4283
|
-
value: fallbackProviderApiKey,
|
|
4284
|
-
},
|
|
4285
|
-
]
|
|
4286
|
-
: []),
|
|
4287
|
-
...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(([name, value]) =>
|
|
4288
|
-
publicProviderHeaders.has(name.toLowerCase())
|
|
4289
|
-
? []
|
|
4290
|
-
: [
|
|
4291
|
-
{
|
|
4292
|
-
name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`,
|
|
4293
|
-
value,
|
|
4294
|
-
},
|
|
4295
|
-
],
|
|
4296
|
-
),
|
|
4297
|
-
...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(([name, value]) =>
|
|
4298
|
-
publicProviderQueries.has(name.toLowerCase())
|
|
4299
|
-
? []
|
|
4300
|
-
: [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }],
|
|
4301
|
-
),
|
|
4302
|
-
]);
|
|
4303
4369
|
// Bind the provider/model catalog's context policy to every model-facing
|
|
4304
4370
|
// path for this turn. In particular, Codex subscription turns must not
|
|
4305
4371
|
// inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
|
|
@@ -4354,15 +4420,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4354
4420
|
input.workspaceId,
|
|
4355
4421
|
effectiveCodexCredentialId ?? "",
|
|
4356
4422
|
);
|
|
4357
|
-
const registerSnapshot = async (
|
|
4358
|
-
snapshotPromise: ReturnType<typeof resolver.getToken>,
|
|
4359
|
-
) => {
|
|
4360
|
-
const snapshot = await snapshotPromise;
|
|
4361
|
-
registerSecretRedactions([
|
|
4362
|
-
{ name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken },
|
|
4363
|
-
]);
|
|
4364
|
-
return snapshot;
|
|
4365
|
-
};
|
|
4366
4423
|
return {
|
|
4367
4424
|
clientVersion: CODEX_CLIENT_VERSION,
|
|
4368
4425
|
// Backend sticky cache-routing key — the SAME id as the body's
|
|
@@ -4372,8 +4429,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4372
4429
|
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
4373
4430
|
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
4374
4431
|
sessionId: input.sessionId,
|
|
4375
|
-
getToken: () =>
|
|
4376
|
-
refresh: () =>
|
|
4432
|
+
getToken: () => resolver.getToken(),
|
|
4433
|
+
refresh: () => resolver.refresh(),
|
|
4377
4434
|
resolveModel: buildModelResolver(
|
|
4378
4435
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
4379
4436
|
CODEX_FALLBACK_MODEL_SLUGS[0],
|
|
@@ -4616,6 +4673,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4616
4673
|
force: true,
|
|
4617
4674
|
clearRequestedCompaction: true,
|
|
4618
4675
|
trigger: "operator",
|
|
4676
|
+
materializeHistory: materializeScreenshotHistory,
|
|
4619
4677
|
projectModelInput: compactionModelHistoryProjector,
|
|
4620
4678
|
...compactionModeOptions,
|
|
4621
4679
|
},
|
|
@@ -4648,7 +4706,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4648
4706
|
},
|
|
4649
4707
|
);
|
|
4650
4708
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
4651
|
-
const errorMessage = String(
|
|
4709
|
+
const errorMessage = String(compactionFailureReasonFromError(error));
|
|
4652
4710
|
if (
|
|
4653
4711
|
!(await settle!({
|
|
4654
4712
|
events: [
|
|
@@ -4781,11 +4839,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4781
4839
|
rigDefaultEnvironmentValues,
|
|
4782
4840
|
workspaceVariableSet?.values ?? {},
|
|
4783
4841
|
);
|
|
4784
|
-
// Redact EVERY exported secret value (rig defaults + session set) from turn
|
|
4785
|
-
// output, not just the session set's.
|
|
4786
|
-
registerSecretRedactions(
|
|
4787
|
-
Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value })),
|
|
4788
|
-
);
|
|
4789
4842
|
// EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
|
|
4790
4843
|
// D1-lite) and reused for EVERY downstream decision: the env mint (skip
|
|
4791
4844
|
// inert platform git tokens for a machine turn), the establish path (no phantom Modal
|
|
@@ -4891,7 +4944,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4891
4944
|
)
|
|
4892
4945
|
: null;
|
|
4893
4946
|
if (initialRunCredentialMaterial) {
|
|
4894
|
-
registerSecretRedactions(initialRunCredentialMaterial.redactions);
|
|
4895
4947
|
for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
|
|
4896
4948
|
publishedRunCredentialNotices.add(JSON.stringify(payload));
|
|
4897
4949
|
await publish!([{ type: "credential.auth_needed", payload }], true);
|
|
@@ -4907,7 +4959,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4907
4959
|
undefined,
|
|
4908
4960
|
);
|
|
4909
4961
|
const establishPolicy: "eager" | "on-demand" =
|
|
4910
|
-
lazyProvisionEnabled(settings) &&
|
|
4962
|
+
lazyProvisionEnabled(settings) &&
|
|
4963
|
+
!machinePrimary &&
|
|
4964
|
+
runSettings.sandboxBackend !== "none" &&
|
|
4965
|
+
// Resolved run credentials must be written to one exact leased sandbox
|
|
4966
|
+
// before agent execution. A warm active pointer can bypass the lazy
|
|
4967
|
+
// provisioner entirely, which previously skipped materialization.
|
|
4968
|
+
!initialRunCredentialMaterial
|
|
4911
4969
|
? "on-demand"
|
|
4912
4970
|
: "eager";
|
|
4913
4971
|
// Computed exactly ONCE per turn and reused for BOTH the box manifest
|
|
@@ -4981,6 +5039,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4981
5039
|
sandboxWorkspaceEnvironmentValues,
|
|
4982
5040
|
{
|
|
4983
5041
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
5042
|
+
skipToolspace: activeSandboxBackend === "selfhosted",
|
|
4984
5043
|
deferGitHubToken:
|
|
4985
5044
|
activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
|
|
4986
5045
|
scope: connectionScope,
|
|
@@ -4993,19 +5052,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4993
5052
|
cancellationSignal,
|
|
4994
5053
|
undefined,
|
|
4995
5054
|
);
|
|
4996
|
-
|
|
4997
|
-
...Object.entries(sandboxGitTokens ?? {}).flatMap(([provider, value]) =>
|
|
4998
|
-
value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : [],
|
|
4999
|
-
),
|
|
5000
|
-
...(sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : []),
|
|
5001
|
-
...((sandboxGitCredentialBindings ?? []).map((binding) => ({
|
|
5002
|
-
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
5003
|
-
value: binding.token,
|
|
5004
|
-
})) satisfies SecretForRedaction[]),
|
|
5005
|
-
...(sandboxToolspaceToken
|
|
5006
|
-
? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }]
|
|
5007
|
-
: []),
|
|
5008
|
-
]);
|
|
5055
|
+
|
|
5009
5056
|
const sandboxToolspaceTokenFile = sandboxToolspaceToken
|
|
5010
5057
|
? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
|
|
5011
5058
|
: undefined;
|
|
@@ -5051,12 +5098,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5051
5098
|
);
|
|
5052
5099
|
if (binding) {
|
|
5053
5100
|
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
5054
|
-
registerSecretRedactions([
|
|
5055
|
-
{
|
|
5056
|
-
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
5057
|
-
value: binding.token,
|
|
5058
|
-
},
|
|
5059
|
-
]);
|
|
5060
5101
|
}
|
|
5061
5102
|
pendingBinding = binding;
|
|
5062
5103
|
return binding
|
|
@@ -5144,7 +5185,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5144
5185
|
toolspaceAuthority,
|
|
5145
5186
|
);
|
|
5146
5187
|
if (material) {
|
|
5147
|
-
registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
|
|
5148
5188
|
}
|
|
5149
5189
|
return material;
|
|
5150
5190
|
};
|
|
@@ -5271,7 +5311,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5271
5311
|
);
|
|
5272
5312
|
return;
|
|
5273
5313
|
}
|
|
5274
|
-
|
|
5314
|
+
|
|
5275
5315
|
await runWorkspaceMutationForSandbox(
|
|
5276
5316
|
requireTargetSandbox(),
|
|
5277
5317
|
"runCredentialMaterialization",
|
|
@@ -5429,6 +5469,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5429
5469
|
db,
|
|
5430
5470
|
settings,
|
|
5431
5471
|
bus,
|
|
5472
|
+
opJournal,
|
|
5432
5473
|
onOp: machineOpObserver.observer,
|
|
5433
5474
|
onSandboxOperation: sandboxOperationObserver,
|
|
5434
5475
|
onHomeSandboxRebound,
|
|
@@ -5538,6 +5579,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5538
5579
|
db,
|
|
5539
5580
|
settings,
|
|
5540
5581
|
bus,
|
|
5582
|
+
opJournal,
|
|
5541
5583
|
onSandboxOperation: sandboxOperationObserver,
|
|
5542
5584
|
onHomeSandboxLost: publishSandboxLost,
|
|
5543
5585
|
onHomeSandboxRebound,
|
|
@@ -5622,9 +5664,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5622
5664
|
const resolveCredential: typeof rawResolveCredential = async (request) => {
|
|
5623
5665
|
const result = await resolveFrozenCredential(request);
|
|
5624
5666
|
if (result.status === "ok") {
|
|
5625
|
-
registerSecretRedactions(
|
|
5626
|
-
headerSecretRedactions("MCP", result.headers, Object.keys(result.headers)),
|
|
5627
|
-
);
|
|
5628
5667
|
}
|
|
5629
5668
|
return result;
|
|
5630
5669
|
};
|
|
@@ -5646,9 +5685,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5646
5685
|
}) => Promise<T>,
|
|
5647
5686
|
): Promise<T> => {
|
|
5648
5687
|
const snapshot = await resolver.getToken();
|
|
5649
|
-
|
|
5650
|
-
{ name: "CODEX_APPS_ACCESS_TOKEN", value: snapshot.accessToken },
|
|
5651
|
-
]);
|
|
5688
|
+
|
|
5652
5689
|
return await withCodexAppsRequestAuthorization(
|
|
5653
5690
|
db,
|
|
5654
5691
|
{ workspaceId: input.workspaceId, credentialId: codexAppsCredentialId },
|
|
@@ -5763,6 +5800,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5763
5800
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
5764
5801
|
...(serviceTier ? { serviceTier } : {}),
|
|
5765
5802
|
...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
|
|
5803
|
+
humanInputEnabled: agentHumanInputEnabled,
|
|
5766
5804
|
genesisTitleHint: isGenesisTurn,
|
|
5767
5805
|
persistentSessionSettings: {
|
|
5768
5806
|
titleIsSet: Boolean(session.title?.trim()),
|
|
@@ -5787,14 +5825,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5787
5825
|
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
5788
5826
|
? { gitTokenSeed: sandboxGitToken }
|
|
5789
5827
|
: {}),
|
|
5790
|
-
// Toolspace is
|
|
5791
|
-
//
|
|
5792
|
-
//
|
|
5793
|
-
// machine's ONLY path to programmatic tool calling and grants no more
|
|
5794
|
-
// than toolspace:call for its own session (own-session-bound,
|
|
5795
|
-
// short-lived and renewable, per-active-turn budgeted, approval-tools
|
|
5796
|
-
// excluded). The runtime seeds it to the box's token file over the same
|
|
5797
|
-
// exec channel, off-manifest, on every backend.
|
|
5828
|
+
// Toolspace delivery is managed-sandbox-only. Connected Machines own any
|
|
5829
|
+
// manually configured API credentials and must never be contacted during
|
|
5830
|
+
// turn admission merely to seed OpenGeni tooling.
|
|
5798
5831
|
...(sandboxToolspaceToken
|
|
5799
5832
|
? {
|
|
5800
5833
|
toolspaceTokenSeed: sandboxToolspaceToken,
|
|
@@ -6102,6 +6135,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6102
6135
|
db,
|
|
6103
6136
|
settings,
|
|
6104
6137
|
bus,
|
|
6138
|
+
opJournal,
|
|
6105
6139
|
onOp: machineOpObserver.observer,
|
|
6106
6140
|
onSandboxOperation: sandboxOperationObserver,
|
|
6107
6141
|
onHomeSandboxLost: publishSandboxLost,
|
|
@@ -6200,7 +6234,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6200
6234
|
},
|
|
6201
6235
|
);
|
|
6202
6236
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
6203
|
-
const errorMessage = String(
|
|
6237
|
+
const errorMessage = String(compactionFailureReasonFromError(error));
|
|
6204
6238
|
if (
|
|
6205
6239
|
!(await settle!({
|
|
6206
6240
|
events: [
|
|
@@ -6302,11 +6336,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6302
6336
|
force: true,
|
|
6303
6337
|
clearRequestedCompaction: true,
|
|
6304
6338
|
trigger: "operator",
|
|
6339
|
+
materializeHistory: materializeScreenshotHistory,
|
|
6305
6340
|
projectModelInput: compactionModelHistoryProjector,
|
|
6306
6341
|
...compactionModeOptions,
|
|
6307
6342
|
}
|
|
6308
6343
|
: {
|
|
6309
6344
|
trigger: "auto",
|
|
6345
|
+
materializeHistory: materializeScreenshotHistory,
|
|
6310
6346
|
projectModelInput: compactionModelHistoryProjector,
|
|
6311
6347
|
...compactionModeOptions,
|
|
6312
6348
|
},
|
|
@@ -6340,11 +6376,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6340
6376
|
},
|
|
6341
6377
|
);
|
|
6342
6378
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
6343
|
-
const errorMessage = String(
|
|
6379
|
+
const errorMessage = String(compactionFailureReasonFromError(compactError));
|
|
6344
6380
|
observability.error("context compaction failed", {
|
|
6345
6381
|
sessionId: input.sessionId,
|
|
6346
6382
|
turnId,
|
|
6347
|
-
|
|
6383
|
+
...safeErrorDiagnostic(compactError),
|
|
6348
6384
|
});
|
|
6349
6385
|
if (
|
|
6350
6386
|
!(await settle!({
|
|
@@ -6459,6 +6495,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6459
6495
|
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
6460
6496
|
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
6461
6497
|
providerApi,
|
|
6498
|
+
materializeModelHistory: materializeScreenshotHistory,
|
|
6499
|
+
materializeSerializedRunState: materializeScreenshotRunState,
|
|
6462
6500
|
projectModelHistory: modelHistoryProjector,
|
|
6463
6501
|
});
|
|
6464
6502
|
runInput = prepared.input;
|
|
@@ -6472,16 +6510,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6472
6510
|
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
6473
6511
|
// the old total-count value, so the common path is unchanged.
|
|
6474
6512
|
//
|
|
6475
|
-
// CRITICAL: seed from the
|
|
6513
|
+
// CRITICAL: seed from the structurally repaired active-row length, not the raw active
|
|
6476
6514
|
// count. `prepareRunInput` builds `state.history` from
|
|
6477
6515
|
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
6478
6516
|
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
6479
6517
|
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
6480
|
-
//
|
|
6518
|
+
// repaired `state.history` off `persistedHistoryCount`; seeding it from
|
|
6481
6519
|
// the raw count (K too high) skips K genuinely-new items, and a
|
|
6482
6520
|
// `function_call` left in that skipped region can later have its
|
|
6483
6521
|
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
6484
|
-
// and bricks the session (issue-61). The
|
|
6522
|
+
// and bricks the session (issue-61). The repaired seed is already
|
|
6485
6523
|
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
6486
6524
|
// slice begins exactly at the first genuinely-new item.
|
|
6487
6525
|
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
@@ -6509,16 +6547,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6509
6547
|
attemptId: input.attemptId,
|
|
6510
6548
|
},
|
|
6511
6549
|
// Never reuse the persisted prior-turn signal for recovery. Proactive
|
|
6512
|
-
// guards provide their exact current signal; provider
|
|
6513
|
-
//
|
|
6514
|
-
// recovery proves progress separately by comparing the
|
|
6515
|
-
// with the current active-history estimate
|
|
6550
|
+
// guards provide their exact current provider-based signal; provider
|
|
6551
|
+
// overflows do not, so their diagnostic signal stays unknown/zero.
|
|
6552
|
+
// Forced recovery proves progress separately by comparing the
|
|
6553
|
+
// replacement with the current active-history estimate.
|
|
6516
6554
|
recoverySignalTokens,
|
|
6517
6555
|
compactSummarizer,
|
|
6518
6556
|
{
|
|
6519
6557
|
force: true,
|
|
6520
6558
|
...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
|
|
6521
6559
|
trigger: triggerLabel,
|
|
6560
|
+
materializeHistory: materializeScreenshotHistory,
|
|
6522
6561
|
projectModelInput: compactionModelHistoryProjector,
|
|
6523
6562
|
...compactionModeOptions,
|
|
6524
6563
|
},
|
|
@@ -6535,13 +6574,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6535
6574
|
return outcome;
|
|
6536
6575
|
};
|
|
6537
6576
|
|
|
6538
|
-
// Keep response
|
|
6577
|
+
// Keep response identity across every stream attempt in this
|
|
6539
6578
|
// activity. Context compaction retries the same logical dispatch by
|
|
6540
6579
|
// calling runStreamAttempt again; resetting this state there would reuse
|
|
6541
6580
|
// the first no-response-ID fallback key and suppress a real model call.
|
|
6542
|
-
const
|
|
6543
|
-
claimedModelUsageSourceKeys,
|
|
6544
|
-
);
|
|
6581
|
+
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
6545
6582
|
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
6546
6583
|
if (!runInput) {
|
|
6547
6584
|
throw new Error("Run input was not prepared");
|
|
@@ -6555,6 +6592,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6555
6592
|
// schema or compatibility state.
|
|
6556
6593
|
let currentToolBatchCallIds = new Set<string>();
|
|
6557
6594
|
let currentToolBatchCompletedCallIds = new Set<string>();
|
|
6595
|
+
let streamSawPerResponseUsage = false;
|
|
6558
6596
|
// Actual input tokens of the most recent model response this turn; the
|
|
6559
6597
|
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
6560
6598
|
throwIfWorkerShuttingDown();
|
|
@@ -6568,6 +6606,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6568
6606
|
// its first-operation provisioner above.
|
|
6569
6607
|
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
6570
6608
|
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
6609
|
+
// `deferredSetup: true` below tells the runtime that the worker owns
|
|
6610
|
+
// platform setup, so the runtime intentionally skips its credential
|
|
6611
|
+
// session callback. Materialize host-managed run credentials here
|
|
6612
|
+
// before any setup command (including provider login hooks), then
|
|
6613
|
+
// decorate setup commands so they source the active generation.
|
|
6614
|
+
await attachRunCredentialRenewal(
|
|
6615
|
+
eagerSetupSession as RunCredentialCommandSession,
|
|
6616
|
+
resolvedSandbox,
|
|
6617
|
+
);
|
|
6618
|
+
const eagerCredentialSetupSession = initialRunCredentialMaterial
|
|
6619
|
+
? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
|
|
6620
|
+
: eagerSetupSession;
|
|
6571
6621
|
await runWorkspaceMutationForSandbox(
|
|
6572
6622
|
resolvedSandbox,
|
|
6573
6623
|
"eagerOwnedSandboxSetup",
|
|
@@ -6575,7 +6625,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6575
6625
|
await runOwnedSandboxSetup(
|
|
6576
6626
|
agent,
|
|
6577
6627
|
ownedEstablished.session as never,
|
|
6578
|
-
|
|
6628
|
+
eagerCredentialSetupSession as never,
|
|
6579
6629
|
{
|
|
6580
6630
|
settings: runSettings,
|
|
6581
6631
|
environment: sandboxEnvironment,
|
|
@@ -6647,9 +6697,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6647
6697
|
tokenSession: ToolspaceTokenWriterSession,
|
|
6648
6698
|
) => {
|
|
6649
6699
|
const renewalSession =
|
|
6650
|
-
|
|
6651
|
-
? tokenSession
|
|
6652
|
-
: ((setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession);
|
|
6700
|
+
(setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession;
|
|
6653
6701
|
await attachToolspaceTokenRenewal(renewalSession);
|
|
6654
6702
|
},
|
|
6655
6703
|
}
|
|
@@ -6671,10 +6719,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6671
6719
|
: {}),
|
|
6672
6720
|
}
|
|
6673
6721
|
: {}),
|
|
6674
|
-
contextCompactionSignal: () =>
|
|
6722
|
+
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
6675
6723
|
contextCompactionRequested: () =>
|
|
6676
6724
|
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
6677
|
-
callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
|
|
6678
6725
|
...(toolCancellationFenceRef.current
|
|
6679
6726
|
? { turnToolCancellationFence: toolCancellationFenceRef.current }
|
|
6680
6727
|
: {}),
|
|
@@ -6712,9 +6759,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6712
6759
|
}
|
|
6713
6760
|
let stableToolCallIdsToClear: string[] | null = null;
|
|
6714
6761
|
let completedCurrentToolBatch = false;
|
|
6715
|
-
|
|
6762
|
+
let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
|
|
6763
|
+
const responseResult = await processModelResponseTerminalEvent({
|
|
6716
6764
|
event: next.value,
|
|
6717
|
-
state:
|
|
6765
|
+
state: modelResponseState,
|
|
6718
6766
|
dispatchId: modelUsageDispatchId,
|
|
6719
6767
|
settings,
|
|
6720
6768
|
db,
|
|
@@ -6745,7 +6793,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6745
6793
|
model: turn.model,
|
|
6746
6794
|
...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
|
|
6747
6795
|
});
|
|
6748
|
-
if (
|
|
6796
|
+
if (responseResult.status === "processed") {
|
|
6797
|
+
streamSawPerResponseUsage ||= responseResult.usageReported;
|
|
6749
6798
|
currentToolBatchCallIds = new Set<string>();
|
|
6750
6799
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
6751
6800
|
await reconcileConversationTruth();
|
|
@@ -6764,7 +6813,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6764
6813
|
// conversation context preserved for the post-top-up resume.
|
|
6765
6814
|
let serializedRunState: string | null = null;
|
|
6766
6815
|
try {
|
|
6767
|
-
serializedRunState = String(
|
|
6816
|
+
serializedRunState = String(stream.state.toString());
|
|
6768
6817
|
} catch {
|
|
6769
6818
|
serializedRunState = null;
|
|
6770
6819
|
}
|
|
@@ -6786,7 +6835,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6786
6835
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
6787
6836
|
callId: pendingToolCall.callId,
|
|
6788
6837
|
callType: pendingToolCall.callType,
|
|
6789
|
-
callItem:
|
|
6838
|
+
callItem: pendingToolCall.callItem as Record<string, unknown>,
|
|
6790
6839
|
});
|
|
6791
6840
|
if (!registered.accepted) {
|
|
6792
6841
|
throw new TurnAttemptFencedError(
|
|
@@ -6794,9 +6843,39 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6794
6843
|
);
|
|
6795
6844
|
}
|
|
6796
6845
|
currentToolBatchCallIds.add(pendingToolCall.callId);
|
|
6846
|
+
if (pendingToolCall.callName === "computer_screenshot") {
|
|
6847
|
+
computerScreenshotCallIds.add(pendingToolCall.callId);
|
|
6848
|
+
}
|
|
6797
6849
|
}
|
|
6798
6850
|
const completedToolCall = completedToolCallFromSdkEvent(next.value);
|
|
6799
6851
|
if (completedToolCall) {
|
|
6852
|
+
const typedScreenshot = typedScreenshotFromSdkEvent(next.value);
|
|
6853
|
+
if (
|
|
6854
|
+
typedScreenshot &&
|
|
6855
|
+
typedScreenshot.callId === completedToolCall.callId &&
|
|
6856
|
+
computerScreenshotCallIds.has(completedToolCall.callId)
|
|
6857
|
+
) {
|
|
6858
|
+
retainedScreenshotMetadata = await retainComputerScreenshot({
|
|
6859
|
+
db,
|
|
6860
|
+
objectStorage,
|
|
6861
|
+
accountId: input.accountId,
|
|
6862
|
+
workspaceId: input.workspaceId,
|
|
6863
|
+
sessionId: input.sessionId,
|
|
6864
|
+
turnId: activeTurnId,
|
|
6865
|
+
attemptId: input.attemptId,
|
|
6866
|
+
output: typedScreenshot,
|
|
6867
|
+
});
|
|
6868
|
+
retainedScreenshotReceiptsByCallId.set(
|
|
6869
|
+
completedToolCall.callId,
|
|
6870
|
+
retainedScreenshotMetadata,
|
|
6871
|
+
);
|
|
6872
|
+
}
|
|
6873
|
+
const durableResultItem = retainedScreenshotMetadata
|
|
6874
|
+
? (compactRetainedScreenshotHistory(
|
|
6875
|
+
[completedToolCall.resultItem],
|
|
6876
|
+
retainedScreenshotReceiptsByCallId,
|
|
6877
|
+
)[0] ?? completedToolCall.resultItem)
|
|
6878
|
+
: completedToolCall.resultItem;
|
|
6800
6879
|
// Keep every parallel result in the attempt ledger until the full
|
|
6801
6880
|
// call batch has settled. The SDK's computed history is
|
|
6802
6881
|
// non-monotonic while parallel calls complete (a later call may
|
|
@@ -6811,7 +6890,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6811
6890
|
attemptId: input.attemptId,
|
|
6812
6891
|
callId: completedToolCall.callId,
|
|
6813
6892
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
6814
|
-
resultItem:
|
|
6893
|
+
resultItem: durableResultItem as Record<string, unknown>,
|
|
6815
6894
|
});
|
|
6816
6895
|
if (!recorded.accepted) {
|
|
6817
6896
|
throw new TurnAttemptFencedError(
|
|
@@ -6840,7 +6919,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6840
6919
|
completedCurrentToolBatch = currentBatchIsStable;
|
|
6841
6920
|
}
|
|
6842
6921
|
}
|
|
6843
|
-
const normalized = normalizeSdkEvent(
|
|
6922
|
+
const normalized = normalizeSdkEvent(
|
|
6923
|
+
next.value,
|
|
6924
|
+
retainedScreenshotMetadata
|
|
6925
|
+
? {
|
|
6926
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
6927
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
6928
|
+
? retainedScreenshotMetadata
|
|
6929
|
+
: {
|
|
6930
|
+
available: false,
|
|
6931
|
+
reason: retainedScreenshotMetadata.reason,
|
|
6932
|
+
},
|
|
6933
|
+
}
|
|
6934
|
+
: {},
|
|
6935
|
+
);
|
|
6844
6936
|
for (const event of normalized) {
|
|
6845
6937
|
streamTiming.onEvent(event.type);
|
|
6846
6938
|
await batcher.push(event);
|
|
@@ -6864,6 +6956,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6864
6956
|
currentToolBatchCallIds = new Set<string>();
|
|
6865
6957
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
6866
6958
|
}
|
|
6959
|
+
for (const callId of stableToolCallIdsToClear) {
|
|
6960
|
+
computerScreenshotCallIds.delete(callId);
|
|
6961
|
+
}
|
|
6867
6962
|
}
|
|
6868
6963
|
}
|
|
6869
6964
|
} finally {
|
|
@@ -6884,7 +6979,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6884
6979
|
stream.completed.catch(() => undefined),
|
|
6885
6980
|
cancellationSignal,
|
|
6886
6981
|
);
|
|
6887
|
-
if (
|
|
6982
|
+
if (!streamSawPerResponseUsage) {
|
|
6888
6983
|
const aggregateUsage = stream.state.usage;
|
|
6889
6984
|
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
6890
6985
|
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
@@ -6895,8 +6990,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6895
6990
|
});
|
|
6896
6991
|
if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
|
|
6897
6992
|
claimedModelUsageSourceKeys.add(aggregateSourceKey);
|
|
6898
|
-
// The
|
|
6899
|
-
//
|
|
6993
|
+
// The aggregate frame is only a billing fallback when no terminal
|
|
6994
|
+
// response exposed usage. It is not final-request context authority.
|
|
6900
6995
|
const aggregateAccountCtx = modelCallAccountContext({
|
|
6901
6996
|
servingCredentialId: effectiveCodexCredentialId,
|
|
6902
6997
|
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
@@ -6963,14 +7058,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6963
7058
|
},
|
|
6964
7059
|
recordAttemptSignals: async () => {
|
|
6965
7060
|
if (!aggregateAuthoritative) return;
|
|
6966
|
-
|
|
6967
|
-
|
|
6968
|
-
|
|
6969
|
-
|
|
6970
|
-
|
|
6971
|
-
if (aggregateInput !== null && aggregateInput > 0) {
|
|
6972
|
-
await setLastInputTokensFenced(aggregateInput);
|
|
6973
|
-
}
|
|
7061
|
+
// Stream aggregates can cover multiple requests and therefore
|
|
7062
|
+
// cannot identify the final request that compaction must bind.
|
|
7063
|
+
// They remain billing/telemetry fallback data only.
|
|
7064
|
+
modelResponseState.contextSignal = null;
|
|
7065
|
+
await setLastInputTokensFenced(null);
|
|
6974
7066
|
},
|
|
6975
7067
|
});
|
|
6976
7068
|
}
|
|
@@ -6980,6 +7072,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6980
7072
|
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
6981
7073
|
const humanInputInterruptions =
|
|
6982
7074
|
runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
|
|
7075
|
+
const latestWorkspace = await getWorkspace(db, input.workspaceId);
|
|
7076
|
+
if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
7077
|
+
assertWorkspaceHumanInputAllowed(
|
|
7078
|
+
resolveWorkspaceAgentHumanInputEnabled(latestWorkspace.settings),
|
|
7079
|
+
"interruption",
|
|
7080
|
+
humanInputInterruptions.length > 0,
|
|
7081
|
+
);
|
|
6983
7082
|
const humanInputRequests = await Promise.all(
|
|
6984
7083
|
humanInputInterruptions.map(async (interruption) => {
|
|
6985
7084
|
const id = stableHumanInputRequestId(
|
|
@@ -7055,25 +7154,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7055
7154
|
) {
|
|
7056
7155
|
return claimedResult({ status: "cancelled" });
|
|
7057
7156
|
}
|
|
7157
|
+
// The interruption and its preceding tool results are now durable.
|
|
7158
|
+
await finalizeTurnOpStreamOps();
|
|
7058
7159
|
activityStatus = "requires_action";
|
|
7059
7160
|
return claimedResult({ status: "requires_action" });
|
|
7060
7161
|
}
|
|
7061
7162
|
|
|
7062
|
-
const finalOutput = String(
|
|
7063
|
-
await reconcileConversationTruth();
|
|
7163
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
7164
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
7064
7165
|
// Op-stream durability fence: the tool outputs are now durably in the
|
|
7065
7166
|
// history store (a redispatch would NOT re-execute them), so this
|
|
7066
7167
|
// turn's settled ops may advance their acked frontier — journal persist
|
|
7067
7168
|
// then wire final ack (licensing the runner to GC its retained
|
|
7068
7169
|
// frames). Best-effort: a miss leaves the runner's retention TTL to
|
|
7069
7170
|
// reap, never fails a completed turn.
|
|
7070
|
-
|
|
7071
|
-
try {
|
|
7072
|
-
await machinePrimarySession.finalizeOpStreamOps();
|
|
7073
|
-
} catch {
|
|
7074
|
-
// The runner's retention TTL owns the fallback.
|
|
7075
|
-
}
|
|
7076
|
-
}
|
|
7171
|
+
await finalizeTurnOpStreamOps();
|
|
7077
7172
|
if (
|
|
7078
7173
|
!(await settle!({
|
|
7079
7174
|
events: [
|
|
@@ -7140,8 +7235,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7140
7235
|
sessionId: input.sessionId,
|
|
7141
7236
|
turnId: activeTurnId,
|
|
7142
7237
|
reason: recoveryKind,
|
|
7143
|
-
|
|
7144
|
-
error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
|
|
7238
|
+
...safeErrorDiagnostic(attemptError),
|
|
7145
7239
|
signalTokens: compactionNeeded?.signalTokens,
|
|
7146
7240
|
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
7147
7241
|
});
|
|
@@ -7186,13 +7280,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7186
7280
|
);
|
|
7187
7281
|
compactionRequestCleared = landmark.requestConsumed;
|
|
7188
7282
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
7189
|
-
compactionFailureMessage = String(
|
|
7190
|
-
redact(compactionFailureReasonFromError(compactError)),
|
|
7191
|
-
);
|
|
7283
|
+
compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
|
|
7192
7284
|
observability.warn("context compaction recovery compaction failed", {
|
|
7193
7285
|
sessionId: input.sessionId,
|
|
7194
7286
|
turnId: activeTurnId,
|
|
7195
|
-
|
|
7287
|
+
...safeErrorDiagnostic(compactError),
|
|
7196
7288
|
});
|
|
7197
7289
|
}
|
|
7198
7290
|
if (!compactionHandled) {
|
|
@@ -7303,6 +7395,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7303
7395
|
turnMetricOutcome = "cancelled";
|
|
7304
7396
|
return claimedResult({ status: "cancelled" });
|
|
7305
7397
|
}
|
|
7398
|
+
acknowledgeRecoveryQuiescence();
|
|
7306
7399
|
await publishDurableSessionEvents(
|
|
7307
7400
|
bus,
|
|
7308
7401
|
input.workspaceId,
|
|
@@ -7322,7 +7415,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7322
7415
|
} catch (recoveryError) {
|
|
7323
7416
|
console.error(
|
|
7324
7417
|
"sandbox lease supersession recovery failed",
|
|
7325
|
-
safeErrorDiagnostic(recoveryError
|
|
7418
|
+
safeErrorDiagnostic(recoveryError),
|
|
7326
7419
|
);
|
|
7327
7420
|
throw recoveryError;
|
|
7328
7421
|
}
|
|
@@ -7354,6 +7447,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7354
7447
|
turnMetricOutcome = "cancelled";
|
|
7355
7448
|
return claimedResult({ status: "cancelled" });
|
|
7356
7449
|
}
|
|
7450
|
+
acknowledgeRecoveryQuiescence();
|
|
7357
7451
|
await publishDurableSessionEvents(
|
|
7358
7452
|
bus,
|
|
7359
7453
|
input.workspaceId,
|
|
@@ -7369,7 +7463,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7369
7463
|
} catch (recoveryError) {
|
|
7370
7464
|
console.error(
|
|
7371
7465
|
"sandbox deadline rotation recovery failed",
|
|
7372
|
-
safeErrorDiagnostic(recoveryError
|
|
7466
|
+
safeErrorDiagnostic(recoveryError),
|
|
7373
7467
|
);
|
|
7374
7468
|
throw recoveryError;
|
|
7375
7469
|
}
|
|
@@ -7403,6 +7497,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7403
7497
|
turnMetricOutcome = "cancelled";
|
|
7404
7498
|
return claimedResult({ status: "cancelled" });
|
|
7405
7499
|
}
|
|
7500
|
+
acknowledgeRecoveryQuiescence();
|
|
7406
7501
|
await publishDurableSessionEvents(
|
|
7407
7502
|
bus,
|
|
7408
7503
|
input.workspaceId,
|
|
@@ -7418,7 +7513,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7418
7513
|
// the turn through a second cancellation path.
|
|
7419
7514
|
console.error(
|
|
7420
7515
|
"worker-shutdown recovery checkpoint failed",
|
|
7421
|
-
safeErrorDiagnostic(recoveryError
|
|
7516
|
+
safeErrorDiagnostic(recoveryError),
|
|
7422
7517
|
);
|
|
7423
7518
|
throw recoveryError;
|
|
7424
7519
|
}
|
|
@@ -7517,11 +7612,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7517
7612
|
await reconcileConversationTruth({ requireDurable: true });
|
|
7518
7613
|
}
|
|
7519
7614
|
checkpointDurable = true;
|
|
7520
|
-
} catch
|
|
7615
|
+
} catch {
|
|
7521
7616
|
observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
|
|
7522
|
-
|
|
7523
|
-
|
|
7524
|
-
|
|
7617
|
+
errorClass: "CodexCheckpointOperationError",
|
|
7618
|
+
errorCode: "codex_lease_loss_checkpoint_failed",
|
|
7619
|
+
origin: "worker",
|
|
7525
7620
|
});
|
|
7526
7621
|
}
|
|
7527
7622
|
|
|
@@ -7628,16 +7723,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7628
7723
|
await flushRuntimeBatcher();
|
|
7629
7724
|
await reconcileConversationTruth({ requireDurable: true });
|
|
7630
7725
|
checkpointDurable = true;
|
|
7631
|
-
} catch
|
|
7726
|
+
} catch {
|
|
7632
7727
|
observability.incrementCounter({
|
|
7633
7728
|
name: "opengeni_codex_failover_checkpoints_total",
|
|
7634
7729
|
help: "Durable Codex failover checkpoint attempts by outcome.",
|
|
7635
7730
|
labels: { workspace_key: codexWorkspaceKey, outcome: "failed" },
|
|
7636
7731
|
});
|
|
7637
7732
|
observability.warn("Codex failover checkpoint failed; refusing automatic replay", {
|
|
7638
|
-
|
|
7639
|
-
|
|
7640
|
-
|
|
7733
|
+
errorClass: "CodexCheckpointOperationError",
|
|
7734
|
+
errorCode: "codex_failover_checkpoint_failed",
|
|
7735
|
+
origin: "worker",
|
|
7641
7736
|
});
|
|
7642
7737
|
}
|
|
7643
7738
|
|
|
@@ -8190,6 +8285,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8190
8285
|
return claimedResult({ status: "cancelled" });
|
|
8191
8286
|
}
|
|
8192
8287
|
if (recovery.action === "recovering") {
|
|
8288
|
+
acknowledgeRecoveryQuiescence();
|
|
8193
8289
|
await publishDurableSessionEvents(
|
|
8194
8290
|
bus,
|
|
8195
8291
|
input.workspaceId,
|
|
@@ -8208,15 +8304,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8208
8304
|
// truth, recover this SAME accepted turn, then let the workflow re-claim
|
|
8209
8305
|
// it after a pacing delay. This is independent of goal state and never
|
|
8210
8306
|
// relies on a synthetic continuation prompt.
|
|
8211
|
-
const failure =
|
|
8307
|
+
const failure = agentRunFailurePayload(error, { isCodexTurn }) as ReturnType<
|
|
8212
8308
|
typeof agentRunFailurePayload
|
|
8213
8309
|
>;
|
|
8214
8310
|
if (isSessionEventPersistenceError(error)) {
|
|
8215
|
-
//
|
|
8216
|
-
//
|
|
8217
|
-
// boundary retains only SQLSTATE, stage, correlation, and safe catalog
|
|
8218
|
-
// identifiers. Provider inference has already happened and is NOT
|
|
8219
|
-
// retried by this terminal classification.
|
|
8311
|
+
// Preserve the exact source message in the internal runtime diagnostic;
|
|
8312
|
+
// SQLSTATE/catalog facts remain separate classification attributes.
|
|
8220
8313
|
observability.error("session event persistence failed", {
|
|
8221
8314
|
accountId: input.accountId,
|
|
8222
8315
|
workspaceId: input.workspaceId,
|
|
@@ -8237,6 +8330,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8237
8330
|
dbDataType: error.details.database.dataType,
|
|
8238
8331
|
dbConstraint: error.details.database.constraint,
|
|
8239
8332
|
dbRoutine: error.details.database.routine,
|
|
8333
|
+
error: error.message,
|
|
8240
8334
|
});
|
|
8241
8335
|
}
|
|
8242
8336
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
@@ -8268,6 +8362,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8268
8362
|
turnMetricOutcome = "cancelled";
|
|
8269
8363
|
return claimedResult({ status: "cancelled" });
|
|
8270
8364
|
}
|
|
8365
|
+
acknowledgeRecoveryQuiescence();
|
|
8271
8366
|
await publishDurableSessionEvents(
|
|
8272
8367
|
bus,
|
|
8273
8368
|
input.workspaceId,
|
|
@@ -8373,18 +8468,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8373
8468
|
const credentialSessionToClear = runCredentialSession;
|
|
8374
8469
|
runCredentialSession = null;
|
|
8375
8470
|
if (credentialSessionToClear) {
|
|
8376
|
-
|
|
8377
|
-
|
|
8378
|
-
|
|
8379
|
-
|
|
8380
|
-
|
|
8381
|
-
|
|
8382
|
-
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
|
|
8387
|
-
|
|
8471
|
+
const clearAttemptCredentials = async (): Promise<void> =>
|
|
8472
|
+
await clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
8473
|
+
sessionId: input.sessionId,
|
|
8474
|
+
attemptId: input.attemptId,
|
|
8475
|
+
executionGeneration,
|
|
8476
|
+
});
|
|
8477
|
+
await clearAttemptCredentialsWithSettledFence({
|
|
8478
|
+
activityStatus,
|
|
8479
|
+
runWorkspaceFencedClear: async () =>
|
|
8480
|
+
await runWorkspaceMutationForSandbox(
|
|
8481
|
+
requireResolvedSandboxForMutation(
|
|
8482
|
+
"Run credential cleanup has no exact sandbox lease target",
|
|
8483
|
+
),
|
|
8484
|
+
"runCredentialAttemptClear",
|
|
8485
|
+
clearAttemptCredentials,
|
|
8486
|
+
),
|
|
8487
|
+
clearExactAttempt: clearAttemptCredentials,
|
|
8488
|
+
onSettledAttemptFence: () => {
|
|
8489
|
+
// Terminal settlement closes the active attempt before this finally
|
|
8490
|
+
// runs, so the ordinary workspace admission correctly rejects it.
|
|
8491
|
+
// The attempt-qualified delete is nevertheless successor-safe: it
|
|
8492
|
+
// removes only this attempt/generation and clears the pointer only
|
|
8493
|
+
// when it still names that exact generation. Finish that cleanup
|
|
8494
|
+
// directly so a successful turn can continue into workspace capture,
|
|
8495
|
+
// tool teardown, and lease release.
|
|
8496
|
+
observability.info("retrying exact run credential cleanup after turn settlement", {
|
|
8497
|
+
"opengeni.session_id": input.sessionId,
|
|
8498
|
+
"opengeni.turn_id": turnId ?? "",
|
|
8499
|
+
"opengeni.attempt_id": input.attemptId,
|
|
8500
|
+
});
|
|
8501
|
+
},
|
|
8502
|
+
});
|
|
8388
8503
|
}
|
|
8389
8504
|
await drainAttemptOwnedSandboxWriters({
|
|
8390
8505
|
// Normal turn completion owns the same process boundary as
|
|
@@ -8452,18 +8567,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8452
8567
|
onReceiptFailure: (error) => {
|
|
8453
8568
|
console.error(
|
|
8454
8569
|
"agent turn quiescence receipt exhausted; signalling proof",
|
|
8455
|
-
safeErrorDiagnostic(error
|
|
8570
|
+
safeErrorDiagnostic(error),
|
|
8456
8571
|
);
|
|
8457
8572
|
},
|
|
8458
8573
|
onPublishFailure: (error) => {
|
|
8459
8574
|
console.error(
|
|
8460
8575
|
"agent turn quiescence event fanout failed",
|
|
8461
|
-
safeErrorDiagnostic(error
|
|
8576
|
+
safeErrorDiagnostic(error),
|
|
8462
8577
|
);
|
|
8463
8578
|
},
|
|
8464
8579
|
onSignalFailure: (error, attempt, retryMs) => {
|
|
8465
8580
|
console.error("agent turn quiescence proof signal failed; retrying", {
|
|
8466
|
-
error: safeErrorDiagnostic(error
|
|
8581
|
+
error: safeErrorDiagnostic(error),
|
|
8467
8582
|
attempt,
|
|
8468
8583
|
retryMs,
|
|
8469
8584
|
});
|
|
@@ -8711,7 +8826,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8711
8826
|
sandboxToRelease.release().catch((releaseError) => {
|
|
8712
8827
|
console.error(
|
|
8713
8828
|
"sandbox lease release failed (turn outcome unaffected)",
|
|
8714
|
-
safeErrorDiagnostic(releaseError
|
|
8829
|
+
safeErrorDiagnostic(releaseError),
|
|
8715
8830
|
);
|
|
8716
8831
|
}),
|
|
8717
8832
|
finalizerSignal,
|
|
@@ -8721,7 +8836,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8721
8836
|
finalizationError ??= error;
|
|
8722
8837
|
console.error(
|
|
8723
8838
|
"agent turn finalization failed (turn outcome unaffected)",
|
|
8724
|
-
safeErrorDiagnostic(error
|
|
8839
|
+
safeErrorDiagnostic(error),
|
|
8725
8840
|
);
|
|
8726
8841
|
} finally {
|
|
8727
8842
|
// If fallible finalization exited before the ordinary exact-holder
|
|
@@ -8782,9 +8897,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8782
8897
|
},
|
|
8783
8898
|
error:
|
|
8784
8899
|
finalizationError || activityError
|
|
8785
|
-
? safeErrorForTelemetry(finalizationError ?? activityError
|
|
8786
|
-
String(redact(value)),
|
|
8787
|
-
)
|
|
8900
|
+
? safeErrorForTelemetry(finalizationError ?? activityError)
|
|
8788
8901
|
: undefined,
|
|
8789
8902
|
});
|
|
8790
8903
|
assertPhysicalToolQuiescenceForCancellation({
|
|
@@ -8916,7 +9029,19 @@ export function agentRunFailurePayload(
|
|
|
8916
9029
|
attempts?: number;
|
|
8917
9030
|
retryOutcome?: string;
|
|
8918
9031
|
database?: Record<string, string>;
|
|
9032
|
+
historyPersistenceStage?: MandatoryHistoryPersistenceStage;
|
|
8919
9033
|
} {
|
|
9034
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
9035
|
+
const underlying = isSessionEventPersistenceError(error.cause)
|
|
9036
|
+
? agentRunFailurePayload(error.cause, options)
|
|
9037
|
+
: {
|
|
9038
|
+
error: error.cause instanceof Error ? error.cause.message : String(error.cause),
|
|
9039
|
+
};
|
|
9040
|
+
return {
|
|
9041
|
+
...underlying,
|
|
9042
|
+
historyPersistenceStage: error.stage,
|
|
9043
|
+
};
|
|
9044
|
+
}
|
|
8920
9045
|
const message = error instanceof Error ? error.message : String(error);
|
|
8921
9046
|
const status =
|
|
8922
9047
|
typeof error === "object" && error !== null
|
|
@@ -8942,15 +9067,8 @@ export function agentRunFailurePayload(
|
|
|
8942
9067
|
}
|
|
8943
9068
|
if (isSessionEventPersistenceError(error)) {
|
|
8944
9069
|
const { details } = error;
|
|
8945
|
-
const eventLabel = details.eventTypes.join(", ") || "session events";
|
|
8946
|
-
const failureLabel =
|
|
8947
|
-
details.code === "db_deadlock"
|
|
8948
|
-
? "Database deadlock"
|
|
8949
|
-
: details.code === "db_serialization_failure"
|
|
8950
|
-
? "Database serialization failure"
|
|
8951
|
-
: "Database failure";
|
|
8952
9070
|
return {
|
|
8953
|
-
error:
|
|
9071
|
+
error: error.message,
|
|
8954
9072
|
code: details.code,
|
|
8955
9073
|
detail:
|
|
8956
9074
|
details.retryOutcome === "exhausted"
|
|
@@ -9012,6 +9130,7 @@ export function agentRunFailurePayload(
|
|
|
9012
9130
|
"A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
|
|
9013
9131
|
code: "mcp_transport_unavailable",
|
|
9014
9132
|
retryable: true,
|
|
9133
|
+
detail: message,
|
|
9015
9134
|
};
|
|
9016
9135
|
}
|
|
9017
9136
|
if (
|
|
@@ -9229,9 +9348,10 @@ export function codexUsageLimitFailurePayload(
|
|
|
9229
9348
|
// goal re-evaluates at most this far out (it will re-pause if still capped).
|
|
9230
9349
|
const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
|
|
9231
9350
|
|
|
9232
|
-
function pendingToolCallFromSdkEvent(event: unknown): {
|
|
9351
|
+
export function pendingToolCallFromSdkEvent(event: unknown): {
|
|
9233
9352
|
callId: string;
|
|
9234
9353
|
callType: string;
|
|
9354
|
+
callName: string | null;
|
|
9235
9355
|
callItem: Record<string, unknown>;
|
|
9236
9356
|
} | null {
|
|
9237
9357
|
if (!event || typeof event !== "object") return null;
|
|
@@ -9252,10 +9372,16 @@ function pendingToolCallFromSdkEvent(event: unknown): {
|
|
|
9252
9372
|
if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
|
|
9253
9373
|
return null;
|
|
9254
9374
|
}
|
|
9255
|
-
|
|
9375
|
+
const callItem = normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]');
|
|
9376
|
+
return {
|
|
9377
|
+
callId,
|
|
9378
|
+
callType,
|
|
9379
|
+
callName: typeof raw.name === "string" ? raw.name : null,
|
|
9380
|
+
callItem,
|
|
9381
|
+
};
|
|
9256
9382
|
}
|
|
9257
9383
|
|
|
9258
|
-
function completedToolCallFromSdkEvent(event: unknown): {
|
|
9384
|
+
export function completedToolCallFromSdkEvent(event: unknown): {
|
|
9259
9385
|
callId: string;
|
|
9260
9386
|
resultItem: Record<string, unknown>;
|
|
9261
9387
|
} | null {
|
|
@@ -9270,7 +9396,11 @@ function completedToolCallFromSdkEvent(event: unknown): {
|
|
|
9270
9396
|
? (item.rawItem as Record<string, unknown>)
|
|
9271
9397
|
: {};
|
|
9272
9398
|
const callId = raw.callId ?? raw.call_id ?? item.id;
|
|
9273
|
-
|
|
9399
|
+
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
9400
|
+
return {
|
|
9401
|
+
callId,
|
|
9402
|
+
resultItem: normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]'),
|
|
9403
|
+
};
|
|
9274
9404
|
}
|
|
9275
9405
|
|
|
9276
9406
|
/**
|
|
@@ -9571,12 +9701,7 @@ export async function recordAuthoritativeModelCallFact(input: {
|
|
|
9571
9701
|
});
|
|
9572
9702
|
} catch (error) {
|
|
9573
9703
|
input.observability.warn("model call fact persist failed", {
|
|
9574
|
-
|
|
9575
|
-
workspaceId: input.workspaceId,
|
|
9576
|
-
sessionId: input.sessionId,
|
|
9577
|
-
turnId: input.turnId,
|
|
9578
|
-
sourceKey: input.sourceKey,
|
|
9579
|
-
error: error instanceof Error ? error.message : String(error),
|
|
9704
|
+
...safeErrorDiagnostic(error),
|
|
9580
9705
|
});
|
|
9581
9706
|
}
|
|
9582
9707
|
}
|