@opengeni/worker-bundle 0.20.3 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -0,0 +1,200 @@
1
+ import { normalizeProtocolJsonValue, sanitizeHistoryItemsForModel } from "@opengeni/runtime";
2
+ import { opaqueProviderArtifactFingerprint } from "@opengeni/codex";
3
+
4
+ export function historyRowsToAppend(
5
+ rawHistory: Array<Record<string, unknown>>,
6
+ // How many items of the CURRENT in-memory history are already persisted (the
7
+ // slice index into `sanitized`). This is the in-memory history length, NOT the
8
+ // total persisted-row count: after a compaction the in-memory history is the
9
+ // short [summary, ...tail, ...new] list, far shorter than the total rows in
10
+ // the table (which still hold the superseded prefix).
11
+ persistedHistoryCount: number,
12
+ // Next free WHOLE-NUMBER absolute position to write at. Decoupled from the
13
+ // slice index because compaction inserts a fractional summary position, so the
14
+ // total-row count no longer equals max(position)+1. Defaults to
15
+ // persistedHistoryCount to preserve the pre-compaction behaviour (contiguous
16
+ // positions from 0) when callers do not pass an explicit next position.
17
+ nextPosition: number = persistedHistoryCount,
18
+ toolOutputTruncationTokens?: number,
19
+ ): {
20
+ rows: Array<{ position: number; item: Record<string, unknown> }>;
21
+ nextWatermark: number;
22
+ nextPosition: number;
23
+ } {
24
+ const modelReady = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens);
25
+ if (modelReady.length <= persistedHistoryCount) {
26
+ return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
27
+ }
28
+ // Canonical model-facing inputs (including machine-input batches) are
29
+ // persisted before inference and therefore live inside the prefix represented
30
+ // by persistedHistoryCount. System messages synthesized only for this
31
+ // attempt—recovery diagnostics, credential notices, attachment materialization
32
+ // notes—must not accidentally become conversation memory during reconciliation.
33
+ // Advance the in-memory watermark past them, but only allocate durable
34
+ // positions to actual model/tool output.
35
+ const rows: Array<{ position: number; item: Record<string, unknown> }> = [];
36
+ for (const [offset, item] of modelReady.slice(persistedHistoryCount).entries()) {
37
+ if (item.type === "message" && item.role === "system") {
38
+ continue;
39
+ }
40
+ rows.push({
41
+ position: nextPosition + rows.length,
42
+ item: normalizeProtocolJsonValue(
43
+ item as Record<string, unknown>,
44
+ `$[${persistedHistoryCount + offset}]`,
45
+ ),
46
+ });
47
+ }
48
+ return {
49
+ rows,
50
+ nextWatermark: modelReady.length,
51
+ nextPosition: nextPosition + rows.length,
52
+ };
53
+ }
54
+
55
+ /** Select only opaque rows that participated in the provider's rejected wire request. */
56
+ export function selectRejectedProviderArtifactHistoryIds(
57
+ activeHistory: ReadonlyArray<{
58
+ id: string;
59
+ item: Record<string, unknown>;
60
+ providerArtifactInvalidatedAt: Date | null;
61
+ }>,
62
+ candidates: {
63
+ knownHistoryItemIds: readonly string[];
64
+ historyItemIds: readonly string[];
65
+ },
66
+ wireFingerprints: readonly string[],
67
+ ): string[] {
68
+ const remainingWireArtifacts = new Map<string, number>();
69
+ for (const fingerprint of wireFingerprints) {
70
+ remainingWireArtifacts.set(fingerprint, (remainingWireArtifacts.get(fingerprint) ?? 0) + 1);
71
+ }
72
+ const knownHistoryItemIds = new Set(candidates.knownHistoryItemIds);
73
+ const initialCandidateIds = new Set(candidates.historyItemIds);
74
+ const selected: string[] = [];
75
+ for (const row of activeHistory) {
76
+ if (row.providerArtifactInvalidatedAt !== null) continue;
77
+ // A row present before this attempt is eligible only when the exact
78
+ // model-facing projection retained it. Rows appended during this run are
79
+ // eligible after durable reconciliation.
80
+ if (knownHistoryItemIds.has(row.id) && !initialCandidateIds.has(row.id)) continue;
81
+ const fingerprint = opaqueProviderArtifactFingerprint(row.item);
82
+ if (!fingerprint) continue;
83
+ const remaining = remainingWireArtifacts.get(fingerprint) ?? 0;
84
+ if (remaining === 0) continue;
85
+ selected.push(row.id);
86
+ remainingWireArtifacts.set(fingerprint, remaining - 1);
87
+ }
88
+ return selected;
89
+ }
90
+
91
+ /** Literal final per-call boundary for replayed and current-turn model input. */
92
+ export function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
93
+ if (item.type === "message") {
94
+ return item.role === "assistant";
95
+ }
96
+ if (item.type === "reasoning" || item.type === "compaction") {
97
+ return true;
98
+ }
99
+ if (typeof item.type === "string") {
100
+ return item.type !== "message";
101
+ }
102
+ return false;
103
+ }
104
+
105
+ /**
106
+ * Resolve the EFFECTIVE/active compute backend a turn should gate
107
+ * filesystem-touching agent lifecycle hooks on (today: the repository clone).
108
+ *
109
+ * WHY (Case B — clone-onto-real-disk hazard): a session keeps its CLOUD HOME
110
+ * backend (`settings.sandboxBackend`, e.g. "modal") but its ACTIVE sandbox may
111
+ * have been swapped to a connected machine (`active_sandbox_id` → a selfhosted
112
+ * lease). `runtime.buildAgent`'s repository-clone hook keys off the backend it is
113
+ * told; if the worker passes nothing it defaults to the HOME backend and the hook
114
+ * would `git clone` a private GitHub-App repo onto the user's REAL disk — a
115
+ * bring-your-own machine owns its own filesystem and must NEVER be cloned onto. So
116
+ * we look at where the agent ACTUALLY runs, not where the session was created.
117
+ *
118
+ * Returns "selfhosted" ONLY when the selfhosted feature is on AND the session has
119
+ * a non-null active pointer whose sandbox `kind` is "selfhosted". Otherwise
120
+ * returns undefined so buildAgent falls back to the home backend — byte-for-byte
121
+ * unchanged cloud behavior.
122
+ *
123
+ * Total + best-effort by contract: it NEVER throws (a lookup failure is logged and
124
+ * falls back to the home default), so wiring it at turn start can't fail the turn.
125
+ * The DB I/O is injected (the real call site passes readActiveSandbox + getSandbox,
126
+ * the same helpers wrapTurnBoxWithRouting reuses) so the gate/decision/safety
127
+ * contract is unit-testable without a live database.
128
+ */
129
+ export function pendingToolCallFromSdkEvent(event: unknown): {
130
+ callId: string;
131
+ callType: string;
132
+ callName: string | null;
133
+ callItem: Record<string, unknown>;
134
+ } | null {
135
+ if (!event || typeof event !== "object") return null;
136
+ if ((event as { type?: unknown }).type !== "run_item_stream_event") return null;
137
+ const item = (event as { item?: { type?: unknown; rawItem?: unknown } }).item;
138
+ if (
139
+ !item ||
140
+ (item.type !== "tool_call_item" && item.type !== "tool_search_call_item") ||
141
+ !item.rawItem ||
142
+ typeof item.rawItem !== "object" ||
143
+ Array.isArray(item.rawItem)
144
+ ) {
145
+ return null;
146
+ }
147
+ const raw = item.rawItem as Record<string, unknown>;
148
+ // The hosted image call is a complete provider fact carried by one item; it
149
+ // never receives a separate function result and therefore must not enter the
150
+ // pending function-call ledger.
151
+ if (raw.type === "hosted_tool_call" && raw.name === "image_generation_call") return null;
152
+ const callId = raw.callId ?? raw.call_id ?? raw.id;
153
+ const callType = raw.type;
154
+ if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
155
+ return null;
156
+ }
157
+ const callItem = normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]');
158
+ return {
159
+ callId,
160
+ callType,
161
+ callName: typeof raw.name === "string" ? raw.name : null,
162
+ callItem,
163
+ };
164
+ }
165
+
166
+ /** Tools whose intentional visual result must survive inline-media sanitization. */
167
+ export function toolCallProducesRetainableSessionImage(name: string | null): boolean {
168
+ return name === "computer_screenshot" || name === "view_image";
169
+ }
170
+
171
+ export function completedToolCallFromSdkEvent(event: unknown): {
172
+ callId: string;
173
+ resultItem: Record<string, unknown>;
174
+ } | null {
175
+ if (!event || typeof event !== "object") return null;
176
+ if ((event as { type?: unknown }).type !== "run_item_stream_event") return null;
177
+ const item = (event as { item?: { type?: unknown; rawItem?: unknown; id?: unknown } }).item;
178
+ if (!item || (item.type !== "tool_call_output_item" && item.type !== "tool_search_output_item")) {
179
+ return null;
180
+ }
181
+ const raw =
182
+ item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem)
183
+ ? (item.rawItem as Record<string, unknown>)
184
+ : {};
185
+ const callId = raw.callId ?? raw.call_id ?? item.id;
186
+ if (typeof callId !== "string" || callId.length === 0) return null;
187
+ return {
188
+ callId,
189
+ resultItem: normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]'),
190
+ };
191
+ }
192
+
193
+ /**
194
+ * Budget/limit exhaustion detected between model calls. This is account
195
+ * state, not an agent failure: the segment ends gracefully (session idles,
196
+ * run state preserved) so a top-up or limit reset lets the same session
197
+ * continue — a failed session would reject the user's next message. An
198
+ * active goal pauses visibly (reason "limits") at the next continuation
199
+ * evaluation without consuming continuation budget.
200
+ */
@@ -0,0 +1,359 @@
1
+ import {
2
+ getMaterializedSandboxFileResources,
3
+ markSandboxFileResourcesMaterialized,
4
+ requireFile,
5
+ } from "@opengeni/db";
6
+ import {
7
+ materializeSandboxFileDownloads,
8
+ sandboxRunAs,
9
+ type CodemodeTokenWriterSession,
10
+ type RetainableSessionImageOutputHook,
11
+ type SandboxFileDownload,
12
+ type TurnToolCancellationFence,
13
+ } from "@opengeni/runtime";
14
+ import type { Settings } from "@opengeni/config";
15
+ import type { RetainedArtifactMetadata } from "@opengeni/contracts";
16
+ import type { ResumedTurnSandbox } from "../../sandbox-resume";
17
+ import type { SharedActivityServices } from "../types";
18
+ import {
19
+ assertGeneratedImageHistoryRetained,
20
+ collectGeneratedImageReceipts,
21
+ compactGeneratedImageRunState,
22
+ generatedImagesFromHistory,
23
+ retainGeneratedImage,
24
+ type GeneratedImageOutput,
25
+ type GeneratedImageReceipt,
26
+ } from "../generated-images";
27
+ import {
28
+ collectRetainedScreenshotReceipts,
29
+ compactRetainedScreenshotRunState,
30
+ materializeRetainedScreenshotHistory,
31
+ retainComputerScreenshot,
32
+ toolOutputContainsInlineImage,
33
+ typedScreenshotFromToolOutput,
34
+ unavailableRetainedSessionImage,
35
+ } from "../retained-screenshots";
36
+ import { objectStorageForSandboxDownloads } from "./file-resources";
37
+ import type { TurnEventPublisher } from "./model-usage";
38
+
39
+ export type NativeImageGenerationRetention = {
40
+ providerId: string;
41
+ providerBindingHash: string;
42
+ sessionId: string;
43
+ turnId: string;
44
+ attemptId: string;
45
+ };
46
+
47
+ export type TurnMediaArtifactDeps = {
48
+ db: SharedActivityServices["db"];
49
+ objectStorage: SharedActivityServices["objectStorage"];
50
+ observability: SharedActivityServices["observability"];
51
+ accountId: string;
52
+ workspaceId: string;
53
+ sessionId: string;
54
+ attemptId: string;
55
+ getTurnId: () => string | undefined;
56
+ getModelRunSettings: () => Settings;
57
+ getPublish: () => TurnEventPublisher | null;
58
+ toolCancellationFenceRef: { current: TurnToolCancellationFence | null };
59
+ getResolvedSandbox: () => ResumedTurnSandbox | null;
60
+ getSetupBoxSession: () => unknown;
61
+ getSandboxGroupId: () => string | null;
62
+ runWorkspaceMutation: <T>(
63
+ sandbox: ResumedTurnSandbox,
64
+ operation: string,
65
+ mutation: () => Promise<T>,
66
+ ) => Promise<T>;
67
+ };
68
+
69
+ export class TurnMediaArtifacts {
70
+ readonly retainedScreenshotReceiptsByCallId = new Map<string, RetainedArtifactMetadata>();
71
+ modelCanReceiveRetainedSessionImages = true;
72
+ readonly generatedImageReceiptsByProviderItemId = new Map<string, GeneratedImageReceipt>();
73
+ readonly generatedImageReceiptsCreatedThisTurn = new Map<string, GeneratedImageReceipt>();
74
+ private generatedImageMaterializationCache: {
75
+ instanceId: string;
76
+ fileIds: Set<string>;
77
+ } | null = null;
78
+ sdkOwnedSandboxSession: CodemodeTokenWriterSession | null = null;
79
+ sandboxFileDownloadBackend: Settings["sandboxBackend"];
80
+ nativeImageGenerationRetention: NativeImageGenerationRetention | null = null;
81
+ readonly retainedSessionImageCallIds = new Set<string>();
82
+
83
+ constructor(private readonly deps: TurnMediaArtifactDeps) {
84
+ this.sandboxFileDownloadBackend = deps.getModelRunSettings().sandboxBackend;
85
+ }
86
+
87
+ rememberGeneratedImageCreatedThisTurn = (receipt: GeneratedImageReceipt): void => {
88
+ this.generatedImageReceiptsCreatedThisTurn.set(receipt.artifact.artifactId, receipt);
89
+ };
90
+
91
+ private prepareGeneratedImageDownload = async (receipt: GeneratedImageReceipt) => {
92
+ const { db, objectStorage } = this.deps;
93
+ if (!objectStorage) {
94
+ throw new Error("Generated image sandbox materialization requires object storage");
95
+ }
96
+ const file = await requireFile(db, this.deps.workspaceId, receipt.artifact.artifactId);
97
+ const downloadStorage = objectStorageForSandboxDownloads(
98
+ this.deps.getModelRunSettings(),
99
+ objectStorage,
100
+ this.sandboxFileDownloadBackend,
101
+ );
102
+ const signed = await downloadStorage.createGetUrl({
103
+ key: file.objectKey,
104
+ });
105
+ return {
106
+ file,
107
+ download: {
108
+ fileId: file.id,
109
+ mountPath: "generated-images",
110
+ filename: file.safeFilename,
111
+ url: signed.url,
112
+ expiresAt: signed.expiresAt,
113
+ sizeBytes: file.sizeBytes,
114
+ ...(file.sha256 ? { sha256: file.sha256 } : {}),
115
+ } satisfies SandboxFileDownload,
116
+ };
117
+ };
118
+
119
+ private writeGeneratedImageDownload = async (
120
+ sessionForDownload: CodemodeTokenWriterSession,
121
+ download: SandboxFileDownload,
122
+ ): Promise<void> => {
123
+ const runAs = sandboxRunAs(this.deps.getModelRunSettings());
124
+ const publish = this.deps.getPublish();
125
+ const fence = this.deps.toolCancellationFenceRef.current;
126
+ const materialized = await materializeSandboxFileDownloads(sessionForDownload, [download], {
127
+ onRuntimeEvent: async (event) => {
128
+ await publish?.([{ type: event.type, payload: event.payload }], true);
129
+ },
130
+ ...(runAs ? { runAs } : {}),
131
+ ...(fence ? { commandRunner: fence.runSandboxCommand.bind(fence) } : {}),
132
+ });
133
+ if (materialized.failures.length > 0) {
134
+ throw new Error(materialized.failures[0]!.reason);
135
+ }
136
+ };
137
+
138
+ private warnGeneratedImageMaterializationDeferred = (error: unknown): void => {
139
+ this.deps.observability.warn("Generated image sandbox materialization deferred", {
140
+ errorClass: error instanceof Error ? error.name : "UnknownError",
141
+ errorCode: "generated_image_materialization_deferred",
142
+ origin: "worker",
143
+ });
144
+ };
145
+
146
+ materializeGeneratedImageInSandbox = async (
147
+ receipt: GeneratedImageReceipt,
148
+ sandbox: ResumedTurnSandbox,
149
+ sessionForDownload: unknown,
150
+ ): Promise<boolean> => {
151
+ try {
152
+ const { file, download } = await this.prepareGeneratedImageDownload(receipt);
153
+ if (this.generatedImageMaterializationCache?.instanceId !== sandbox.established.instanceId) {
154
+ const fileIds = this.deps.getSandboxGroupId()
155
+ ? await getMaterializedSandboxFileResources(this.deps.db, {
156
+ accountId: this.deps.accountId,
157
+ workspaceId: this.deps.workspaceId,
158
+ sandboxGroupId: this.deps.getSandboxGroupId()!,
159
+ expectedEpoch: sandbox.leaseEpoch,
160
+ instanceId: sandbox.established.instanceId,
161
+ })
162
+ : new Set<string>();
163
+ this.generatedImageMaterializationCache = {
164
+ instanceId: sandbox.established.instanceId,
165
+ fileIds,
166
+ };
167
+ }
168
+ if (this.generatedImageMaterializationCache.fileIds.has(file.id)) return true;
169
+ await this.deps.runWorkspaceMutation(
170
+ sandbox,
171
+ "generatedImageMaterialization",
172
+ async () =>
173
+ await this.writeGeneratedImageDownload(
174
+ sessionForDownload as CodemodeTokenWriterSession,
175
+ download,
176
+ ),
177
+ );
178
+ this.generatedImageMaterializationCache.fileIds.add(file.id);
179
+ const sandboxGroupId = this.deps.getSandboxGroupId();
180
+ if (sandboxGroupId) {
181
+ await markSandboxFileResourcesMaterialized(this.deps.db, {
182
+ accountId: this.deps.accountId,
183
+ workspaceId: this.deps.workspaceId,
184
+ sandboxGroupId,
185
+ expectedEpoch: sandbox.leaseEpoch,
186
+ instanceId: sandbox.established.instanceId,
187
+ fileIds: [file.id],
188
+ });
189
+ }
190
+ return true;
191
+ } catch (error) {
192
+ this.warnGeneratedImageMaterializationDeferred(error);
193
+ return false;
194
+ }
195
+ };
196
+
197
+ materializeGeneratedImageInOwnedSdkSession = async (
198
+ receipt: GeneratedImageReceipt,
199
+ sessionForDownload: unknown,
200
+ ): Promise<boolean> => {
201
+ try {
202
+ const { download } = await this.prepareGeneratedImageDownload(receipt);
203
+ await this.writeGeneratedImageDownload(
204
+ sessionForDownload as CodemodeTokenWriterSession,
205
+ download,
206
+ );
207
+ return true;
208
+ } catch (error) {
209
+ this.warnGeneratedImageMaterializationDeferred(error);
210
+ return false;
211
+ }
212
+ };
213
+
214
+ materializeGeneratedImage = async (receipt: GeneratedImageReceipt): Promise<boolean> => {
215
+ if (this.sandboxFileDownloadBackend === "none") return false;
216
+ const resolvedSandbox = this.deps.getResolvedSandbox();
217
+ const setupBoxSession = this.deps.getSetupBoxSession();
218
+ if (resolvedSandbox && setupBoxSession) {
219
+ return await this.materializeGeneratedImageInSandbox(
220
+ receipt,
221
+ resolvedSandbox,
222
+ setupBoxSession,
223
+ );
224
+ }
225
+ if (this.sdkOwnedSandboxSession) {
226
+ return await this.materializeGeneratedImageInOwnedSdkSession(
227
+ receipt,
228
+ this.sdkOwnedSandboxSession,
229
+ );
230
+ }
231
+ return false;
232
+ };
233
+
234
+ retainNativeGeneratedImage = async (
235
+ output: GeneratedImageOutput,
236
+ ): Promise<GeneratedImageReceipt> => {
237
+ if (output.providerItemId) {
238
+ const existing = this.generatedImageReceiptsByProviderItemId.get(output.providerItemId);
239
+ if (existing) return existing;
240
+ }
241
+ const context = this.nativeImageGenerationRetention;
242
+ if (!context || !output.providerItemId) {
243
+ throw new Error("Native generated image arrived without provider retention identity");
244
+ }
245
+ const retained = await retainGeneratedImage({
246
+ db: this.deps.db,
247
+ objectStorage: this.deps.objectStorage,
248
+ accountId: this.deps.accountId,
249
+ workspaceId: this.deps.workspaceId,
250
+ sessionId: context.sessionId,
251
+ turnId: context.turnId,
252
+ attemptId: context.attemptId,
253
+ sourceStrategy: "native_hosted",
254
+ providerId: context.providerId,
255
+ providerBindingHash: context.providerBindingHash,
256
+ output,
257
+ });
258
+ this.generatedImageReceiptsByProviderItemId.set(output.providerItemId, retained.receipt);
259
+ this.rememberGeneratedImageCreatedThisTurn(retained.receipt);
260
+ await this.materializeGeneratedImage(retained.receipt);
261
+ return retained.receipt;
262
+ };
263
+
264
+ retainNativeGeneratedImagesFromHistory = async (
265
+ history: Array<Record<string, unknown>>,
266
+ ): Promise<void> => {
267
+ collectGeneratedImageReceipts(history, this.generatedImageReceiptsByProviderItemId);
268
+ for (const output of generatedImagesFromHistory(
269
+ history,
270
+ this.generatedImageReceiptsByProviderItemId,
271
+ )) {
272
+ if (
273
+ !output.providerItemId ||
274
+ this.generatedImageReceiptsByProviderItemId.has(output.providerItemId)
275
+ ) {
276
+ continue;
277
+ }
278
+ await this.retainNativeGeneratedImage(output);
279
+ }
280
+ assertGeneratedImageHistoryRetained(history, this.generatedImageReceiptsByProviderItemId);
281
+ };
282
+
283
+ compactMediaRunState = (serialized: string): string =>
284
+ compactGeneratedImageRunState(
285
+ compactRetainedScreenshotRunState(serialized, this.retainedScreenshotReceiptsByCallId),
286
+ this.generatedImageReceiptsByProviderItemId,
287
+ );
288
+
289
+ retainSessionImageAtToolBoundary: RetainableSessionImageOutputHook = async ({
290
+ toolCallId,
291
+ output,
292
+ }) => {
293
+ const activeTurnId = this.deps.getTurnId();
294
+ if (!activeTurnId) {
295
+ throw new Error("Session image tool completed before turn initialization");
296
+ }
297
+ const typedScreenshot = typedScreenshotFromToolOutput({
298
+ callId: toolCallId,
299
+ output,
300
+ });
301
+ this.retainedSessionImageCallIds.add(toolCallId);
302
+ if (!typedScreenshot) {
303
+ if (toolOutputContainsInlineImage(output)) {
304
+ this.retainedScreenshotReceiptsByCallId.set(
305
+ toolCallId,
306
+ unavailableRetainedSessionImage({
307
+ sessionId: this.deps.sessionId,
308
+ turnId: activeTurnId,
309
+ attemptId: this.deps.attemptId,
310
+ toolCallId,
311
+ toolOutputId: toolCallId,
312
+ reason: "unsupported",
313
+ }),
314
+ );
315
+ }
316
+ return;
317
+ }
318
+ this.retainedScreenshotReceiptsByCallId.set(
319
+ toolCallId,
320
+ unavailableRetainedSessionImage({
321
+ sessionId: this.deps.sessionId,
322
+ turnId: activeTurnId,
323
+ attemptId: this.deps.attemptId,
324
+ toolCallId,
325
+ toolOutputId: toolCallId,
326
+ reason: "pending",
327
+ }),
328
+ );
329
+ this.retainedScreenshotReceiptsByCallId.set(
330
+ toolCallId,
331
+ await retainComputerScreenshot({
332
+ db: this.deps.db,
333
+ objectStorage: this.deps.objectStorage,
334
+ accountId: this.deps.accountId,
335
+ workspaceId: this.deps.workspaceId,
336
+ sessionId: this.deps.sessionId,
337
+ turnId: activeTurnId,
338
+ attemptId: this.deps.attemptId,
339
+ output: typedScreenshot,
340
+ }),
341
+ );
342
+ };
343
+
344
+ materializeScreenshotHistory = async (history: Array<Record<string, unknown>>) => {
345
+ collectRetainedScreenshotReceipts(history, this.retainedScreenshotReceiptsByCallId);
346
+ if (!this.modelCanReceiveRetainedSessionImages) return history;
347
+ return await materializeRetainedScreenshotHistory({
348
+ db: this.deps.db,
349
+ objectStorage: this.deps.objectStorage,
350
+ workspaceId: this.deps.workspaceId,
351
+ sessionId: this.deps.sessionId,
352
+ history,
353
+ });
354
+ };
355
+ }
356
+
357
+ export function createTurnMediaArtifacts(deps: TurnMediaArtifactDeps): TurnMediaArtifacts {
358
+ return new TurnMediaArtifacts(deps);
359
+ }