@opengeni/worker-bundle 0.16.29 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/dist/activities/agent-turn.d.ts +85 -19
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +1 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -8
  10. package/dist/activities/expiring-material-renewal.d.ts +1 -1
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +1 -1
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
  25. package/dist/activities/retained-screenshots.d.ts +36 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +1 -1
  28. package/dist/activities/run-input.d.ts +3 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +10 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -150
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/concurrency.d.ts +108 -5
  64. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  65. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  66. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  67. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  68. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  69. package/dist/editable-artifact-materializer.d.ts +262 -0
  70. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  71. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  72. package/dist/host-export.d.ts +1 -1
  73. package/dist/index.d.ts +176 -10
  74. package/dist/index.js +662 -17374
  75. package/dist/index.js.map +1 -1
  76. package/dist/memory-pressure-guard.d.ts +57 -0
  77. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  78. package/dist/observability-metrics.d.ts +33 -0
  79. package/dist/sandbox-reaper-contract.d.ts +40 -0
  80. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  81. package/dist/sandbox-resume.d.ts +34 -7
  82. package/dist/sandbox-routing.d.ts +2 -0
  83. package/dist/worker-service-lifecycle.d.ts +6 -2
  84. package/dist/workflow-bundle.js +1043 -363
  85. package/dist/workflows/activities.d.ts +7 -1
  86. package/dist/workflows/document-index.d.ts +1 -1
  87. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  88. package/dist/workflows/rig-verification.d.ts +1 -1
  89. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  90. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  91. package/dist/workflows/session.d.ts +2 -2
  92. package/dist/workflows/video-generation.d.ts +10 -0
  93. package/dist/workflows/workflow-wake.d.ts +1 -1
  94. package/dist/workflows.d.ts +9 -7
  95. package/package.json +34 -15
  96. package/src/activities/agent-turn.ts +1470 -246
  97. package/src/activities/api-integrations.ts +138 -0
  98. package/src/activities/codemode-dispatcher.ts +335 -0
  99. package/src/activities/codemode-token-renewal.ts +64 -0
  100. package/src/activities/codex-capacity.ts +6 -6
  101. package/src/activities/codex-image-generation.ts +56 -0
  102. package/src/activities/context-compaction.ts +38 -3
  103. package/src/activities/documents.ts +14 -4
  104. package/src/activities/environment.ts +45 -42
  105. package/src/activities/expiring-material-renewal.ts +1 -1
  106. package/src/activities/file-upload-reaper.ts +34 -3
  107. package/src/activities/gateway-image-generation.ts +146 -0
  108. package/src/activities/gateway-video-generation.ts +298 -0
  109. package/src/activities/generated-images.ts +860 -0
  110. package/src/activities/goals.ts +6 -2
  111. package/src/activities/google-drive-changes.ts +454 -0
  112. package/src/activities/image-generation-operation.ts +333 -0
  113. package/src/activities/image-generation-references.ts +118 -0
  114. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  115. package/src/activities/knowledge-source-sync.ts +2840 -0
  116. package/src/activities/mcp-credentials.ts +1 -0
  117. package/src/activities/packs.ts +55 -55
  118. package/src/activities/retained-screenshot-reaper.ts +2 -2
  119. package/src/activities/retained-screenshots.ts +453 -28
  120. package/src/activities/rig-verification.ts +554 -43
  121. package/src/activities/run-input.ts +22 -6
  122. package/src/activities/sandbox-images.ts +167 -0
  123. package/src/activities/sandbox-lease.ts +1065 -483
  124. package/src/activities/scheduled-tasks.ts +77 -4
  125. package/src/activities/session-state.ts +4 -4
  126. package/src/activities/streaming.ts +42 -3
  127. package/src/activities/types.ts +87 -5
  128. package/src/activities/video-generation-admission.ts +270 -0
  129. package/src/activities/video-generation-credential.ts +34 -0
  130. package/src/activities/video-generation-reconciliation.ts +578 -0
  131. package/src/activities/video-media-validation.ts +326 -0
  132. package/src/activities/video-output-retention.ts +314 -0
  133. package/src/activities/video-reference-staging.ts +263 -0
  134. package/src/activities/workflow-wake.ts +2 -2
  135. package/src/activities/workspace-capture.ts +44 -10
  136. package/src/activities-control.ts +54 -0
  137. package/src/activities-turn.ts +40 -0
  138. package/src/activities.ts +48 -124
  139. package/src/activity-services.ts +62 -0
  140. package/src/artifact-materializer-entry.ts +35 -0
  141. package/src/artifact-outbox-entry.ts +29 -0
  142. package/src/concurrency.ts +484 -6
  143. package/src/editable-artifact-hint-broker.ts +235 -0
  144. package/src/editable-artifact-materializer-launcher.ts +515 -0
  145. package/src/editable-artifact-materializer-service.ts +787 -0
  146. package/src/editable-artifact-materializer-storage.ts +413 -0
  147. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  148. package/src/editable-artifact-materializer.ts +1289 -0
  149. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  150. package/src/editable-artifact-outbox-service.ts +438 -0
  151. package/src/index.ts +277 -72
  152. package/src/memory-pressure-guard.ts +489 -0
  153. package/src/model-checkpoint-memory-collector.ts +116 -0
  154. package/src/observability-metrics.ts +199 -4
  155. package/src/sandbox-reaper-contract.ts +73 -0
  156. package/src/sandbox-reaper-timeout.ts +79 -0
  157. package/src/sandbox-resume.ts +294 -153
  158. package/src/sandbox-routing.ts +19 -24
  159. package/src/worker-service-lifecycle.ts +80 -14
  160. package/src/workflows/activities.ts +30 -0
  161. package/src/workflows/file-upload-reaper.ts +3 -1
  162. package/src/workflows/knowledge-source-sync.ts +83 -0
  163. package/src/workflows/sandbox-reaper.ts +183 -21
  164. package/src/workflows/scheduled-tasks.ts +42 -6
  165. package/src/workflows/video-generation.ts +36 -0
  166. package/src/workflows.ts +17 -1
  167. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  168. package/src/activities/toolspace-token-renewal.ts +0 -64
@@ -43,6 +43,18 @@ export type ValidatedScreenshot = {
43
43
  height: number;
44
44
  };
45
45
 
46
+ export type RetainableSessionImageMediaType = "image/png" | "image/jpeg" | "image/webp";
47
+
48
+ export type ValidatedSessionImage = {
49
+ bytes: Uint8Array;
50
+ mediaType: RetainableSessionImageMediaType;
51
+ extension: "png" | "jpg" | "webp";
52
+ sizeBytes: number;
53
+ sha256: string;
54
+ width: number;
55
+ height: number;
56
+ };
57
+
46
58
  export class ScreenshotValidationError extends Error {
47
59
  constructor(
48
60
  readonly reason: "invalid_content" | "oversized" | "unsupported",
@@ -63,6 +75,21 @@ export class RetainedScreenshotUnavailableError extends Error {
63
75
  }
64
76
  }
65
77
 
78
+ /** Read an intentional image directly at the tool execution boundary. */
79
+ export function typedScreenshotFromToolOutput(input: {
80
+ callId: string;
81
+ output: unknown;
82
+ }): TypedScreenshotToolOutput | null {
83
+ const image = imageBytesFromSdkToolOutput(input.output);
84
+ if (!image) return null;
85
+ return {
86
+ callId: input.callId,
87
+ toolOutputId: input.callId,
88
+ bytes: image.bytes,
89
+ mediaType: image.mediaType,
90
+ };
91
+ }
92
+
66
93
  /** Read the SDK's trusted typed image before event/history serialization. */
67
94
  export function typedScreenshotFromSdkEvent(event: unknown): TypedScreenshotToolOutput | null {
68
95
  if (!event || typeof event !== "object") return null;
@@ -78,34 +105,144 @@ export function typedScreenshotFromSdkEvent(event: unknown): TypedScreenshotTool
78
105
  if (streamEvent.type !== "run_item_stream_event") return null;
79
106
  const item = streamEvent.item;
80
107
  if (!item || item.type !== "tool_call_output_item") return null;
81
- const output = item.output;
82
- if (!output || typeof output !== "object" || Array.isArray(output)) return null;
83
- const outputRecord = output as Record<string, unknown>;
84
- if (
85
- outputRecord.type !== "image" ||
86
- !outputRecord.image ||
87
- typeof outputRecord.image !== "object"
88
- ) {
89
- return null;
90
- }
91
- const image = outputRecord.image as Record<string, unknown>;
92
- if (!(image.data instanceof Uint8Array) || typeof image.mediaType !== "string") return null;
108
+ const image = imageBytesFromSdkToolOutput(item.output);
109
+ if (!image) return null;
93
110
  const raw =
94
111
  item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem)
95
112
  ? (item.rawItem as Record<string, unknown>)
96
113
  : {};
97
- const callId = raw.callId ?? raw.call_id;
114
+ // Live Agents SDK results can carry the call identity only on the run-item
115
+ // wrapper; reconciled history later copies it into `rawItem`. Use the same
116
+ // fallback as the canonical completed-call extractor so both paths agree.
117
+ const callId = raw.callId ?? raw.call_id ?? item.id;
98
118
  if (typeof callId !== "string" || callId.length === 0) return null;
99
119
  const toolOutputId = raw.id ?? item.id ?? callId;
100
120
  if (typeof toolOutputId !== "string" || toolOutputId.length === 0) return null;
101
121
  return {
102
122
  callId,
103
123
  toolOutputId,
104
- bytes: image.data,
124
+ bytes: image.bytes,
105
125
  mediaType: image.mediaType,
106
126
  };
107
127
  }
108
128
 
129
+ /** Does this SDK result still contain inline bytes that persistence must replace? */
130
+ export function sdkEventContainsInlineImage(event: unknown): boolean {
131
+ if (!event || typeof event !== "object") return false;
132
+ const streamEvent = event as {
133
+ type?: unknown;
134
+ item?: { type?: unknown; output?: unknown };
135
+ };
136
+ return (
137
+ streamEvent.type === "run_item_stream_event" &&
138
+ streamEvent.item?.type === "tool_call_output_item" &&
139
+ toolOutputContainsInlineImage(streamEvent.item.output)
140
+ );
141
+ }
142
+
143
+ function imageBytesFromSdkToolOutput(
144
+ output: unknown,
145
+ ): { bytes: Uint8Array; mediaType: string } | null {
146
+ if (typeof output === "string") return decodeInlineImageDataUrl(output);
147
+ if (Array.isArray(output)) {
148
+ for (const entry of output) {
149
+ if (!entry || typeof entry !== "object" || Array.isArray(entry)) continue;
150
+ const record = entry as Record<string, unknown>;
151
+ if (record.type !== "input_image") continue;
152
+ const decoded = imageBytesFromSdkImageSource(
153
+ record.image ?? record.image_url ?? record.imageUrl,
154
+ );
155
+ if (decoded) return decoded;
156
+ }
157
+ return null;
158
+ }
159
+ if (!output || typeof output !== "object") return null;
160
+ const outputRecord = output as Record<string, unknown>;
161
+ if (
162
+ outputRecord.type !== "image" ||
163
+ !outputRecord.image ||
164
+ typeof outputRecord.image !== "object" ||
165
+ Array.isArray(outputRecord.image)
166
+ ) {
167
+ return null;
168
+ }
169
+ return imageBytesFromSdkImageSource(outputRecord.image);
170
+ }
171
+
172
+ function imageBytesFromSdkImageSource(
173
+ source: unknown,
174
+ ): { bytes: Uint8Array; mediaType: string } | null {
175
+ const direct = decodeInlineImageDataUrl(source);
176
+ if (direct) return direct;
177
+ if (!source || typeof source !== "object" || Array.isArray(source)) return null;
178
+ const image = source as Record<string, unknown>;
179
+ const fromUrl = decodeInlineImageDataUrl(image.url);
180
+ if (fromUrl) return fromUrl;
181
+ return image.data instanceof Uint8Array && typeof image.mediaType === "string"
182
+ ? { bytes: image.data, mediaType: image.mediaType }
183
+ : null;
184
+ }
185
+
186
+ function sdkImageSourceContainsInlineImage(source: unknown): boolean {
187
+ if (typeof source === "string") return source.startsWith("data:image/");
188
+ if (!source || typeof source !== "object" || Array.isArray(source)) return false;
189
+ const image = source as Record<string, unknown>;
190
+ return (
191
+ (typeof image.url === "string" && image.url.startsWith("data:image/")) ||
192
+ (image.data instanceof Uint8Array && typeof image.mediaType === "string")
193
+ );
194
+ }
195
+
196
+ export function toolOutputContainsInlineImage(output: unknown): boolean {
197
+ if (typeof output === "string") return output.startsWith("data:image/");
198
+ if (Array.isArray(output)) {
199
+ return output.some((entry) => {
200
+ if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
201
+ const record = entry as Record<string, unknown>;
202
+ return (
203
+ record.type === "input_image" &&
204
+ sdkImageSourceContainsInlineImage(record.image ?? record.image_url ?? record.imageUrl)
205
+ );
206
+ });
207
+ }
208
+ if (!output || typeof output !== "object") return false;
209
+ const record = output as Record<string, unknown>;
210
+ return record.type === "image" && sdkImageSourceContainsInlineImage(record.image);
211
+ }
212
+
213
+ function decodeInlineImageDataUrl(value: unknown): { bytes: Uint8Array; mediaType: string } | null {
214
+ if (typeof value !== "string") return null;
215
+ const match = /^data:(image\/(?:png|jpeg|webp));base64,/u.exec(value);
216
+ if (!match?.[1]) return null;
217
+ const mediaType = match[1] as RetainableSessionImageMediaType;
218
+ const encoded = value.slice(match[0].length);
219
+ const maxEncodedLength = Math.ceil(COMPUTER_SCREENSHOT_MAX_BYTES / 3) * 4;
220
+ if (
221
+ encoded.length === 0 ||
222
+ encoded.length > maxEncodedLength ||
223
+ encoded.length % 4 !== 0 ||
224
+ !/^[A-Za-z0-9+/]+={0,2}$/.test(encoded)
225
+ ) {
226
+ return null;
227
+ }
228
+ const padding = encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0;
229
+ const decodedLength = (encoded.length / 4) * 3 - padding;
230
+ if (decodedLength <= 0 || decodedLength > COMPUTER_SCREENSHOT_MAX_BYTES) {
231
+ return null;
232
+ }
233
+ const lastSextet = base64Sextet(encoded.charCodeAt(encoded.length - padding - 1));
234
+ if (
235
+ (padding === 2 && (lastSextet & 0x0f) !== 0) ||
236
+ (padding === 1 && (lastSextet & 0x03) !== 0)
237
+ ) {
238
+ return null;
239
+ }
240
+ const decoded = Buffer.from(encoded, "base64");
241
+ if (decoded.byteLength !== decodedLength) return null;
242
+ const bytes = new Uint8Array(decoded.buffer, decoded.byteOffset, decoded.byteLength);
243
+ return { bytes, mediaType };
244
+ }
245
+
109
246
  /** Validate the complete canonical PNG and derive exact integrity metadata. */
110
247
  export function validateComputerScreenshot(
111
248
  input: { bytes: Uint8Array; mediaType: string },
@@ -275,6 +412,73 @@ export function validateComputerScreenshot(
275
412
  };
276
413
  }
277
414
 
415
+ /** Validate a durable view_image/computer image from bytes, never from its claimed MIME alone. */
416
+ export function validateRetainableSessionImage(
417
+ input: { bytes: Uint8Array; declaredMediaType?: string },
418
+ limits: {
419
+ maxBytes?: number;
420
+ maxDimension?: number;
421
+ maxPixels?: number;
422
+ } = {},
423
+ ): ValidatedSessionImage {
424
+ const maxBytes = limits.maxBytes ?? COMPUTER_SCREENSHOT_MAX_BYTES;
425
+ const maxDimension = limits.maxDimension ?? COMPUTER_SCREENSHOT_MAX_DIMENSION;
426
+ const maxPixels = limits.maxPixels ?? COMPUTER_SCREENSHOT_MAX_PIXELS;
427
+ if (input.bytes.byteLength === 0) {
428
+ throw new ScreenshotValidationError("invalid_content", "session image is empty");
429
+ }
430
+ if (input.bytes.byteLength > maxBytes) {
431
+ throw new ScreenshotValidationError("oversized", `session image exceeds ${maxBytes} bytes`);
432
+ }
433
+
434
+ let identity: Pick<ValidatedSessionImage, "mediaType" | "extension" | "width" | "height">;
435
+ if (isPng(input.bytes)) {
436
+ const png = validateComputerScreenshot(
437
+ { bytes: input.bytes, mediaType: "image/png" },
438
+ { maxBytes, maxDimension, maxPixels },
439
+ );
440
+ identity = {
441
+ mediaType: "image/png",
442
+ extension: "png",
443
+ width: png.width,
444
+ height: png.height,
445
+ };
446
+ } else if (isJpeg(input.bytes)) {
447
+ identity = {
448
+ mediaType: "image/jpeg",
449
+ extension: "jpg",
450
+ ...jpegDimensions(input.bytes),
451
+ };
452
+ } else if (isWebp(input.bytes)) {
453
+ identity = {
454
+ mediaType: "image/webp",
455
+ extension: "webp",
456
+ ...webpDimensions(input.bytes),
457
+ };
458
+ } else {
459
+ throw new ScreenshotValidationError("unsupported", "session image format is unsupported");
460
+ }
461
+ assertImageDimensions(identity.width, identity.height, {
462
+ maxDimension,
463
+ maxPixels,
464
+ });
465
+ if (
466
+ input.declaredMediaType &&
467
+ canonicalMediaType(input.declaredMediaType) !== identity.mediaType
468
+ ) {
469
+ throw new ScreenshotValidationError(
470
+ "invalid_content",
471
+ "session image MIME does not match its bytes",
472
+ );
473
+ }
474
+ return {
475
+ bytes: input.bytes,
476
+ ...identity,
477
+ sizeBytes: input.bytes.byteLength,
478
+ sha256: createHash("sha256").update(input.bytes).digest("hex"),
479
+ };
480
+ }
481
+
278
482
  export function retainedScreenshotIdentity(input: {
279
483
  sessionId: string;
280
484
  turnId: string;
@@ -304,6 +508,19 @@ export function retainedScreenshotIdentity(input: {
304
508
  };
305
509
  }
306
510
 
511
+ /** Deterministic fail-closed receipt installed before retention can perform I/O. */
512
+ export function unavailableRetainedSessionImage(input: {
513
+ sessionId: string;
514
+ turnId: string;
515
+ attemptId: string;
516
+ toolCallId: string;
517
+ toolOutputId: string;
518
+ reason: RetainedOutputUnavailableReason;
519
+ }): RetainedArtifactMetadata {
520
+ const identity = retainedScreenshotIdentity(input);
521
+ return unavailable(identity.artifactId, input.reason);
522
+ }
523
+
307
524
  export async function retainComputerScreenshot(input: {
308
525
  db: Database;
309
526
  objectStorage: ObjectStorage | null;
@@ -324,9 +541,12 @@ export async function retainComputerScreenshot(input: {
324
541
  toolCallId: input.output.callId,
325
542
  toolOutputId: input.output.toolOutputId,
326
543
  });
327
- let screenshot: ValidatedScreenshot;
544
+ let screenshot: ValidatedSessionImage;
328
545
  try {
329
- screenshot = validateComputerScreenshot(input.output);
546
+ screenshot = validateRetainableSessionImage({
547
+ bytes: input.output.bytes,
548
+ declaredMediaType: input.output.mediaType,
549
+ });
330
550
  } catch (error) {
331
551
  if (error instanceof ScreenshotValidationError) {
332
552
  return unavailable(identity.artifactId, error.reason);
@@ -339,7 +559,7 @@ export async function retainComputerScreenshot(input: {
339
559
  const retentionExpiresAt = new Date(
340
560
  now.getTime() + (input.retentionMs ?? COMPUTER_SCREENSHOT_RETENTION_MS),
341
561
  );
342
- const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/computer-screenshot.png`;
562
+ const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/session-image.${screenshot.extension}`;
343
563
  let prepared: RetainedScreenshotArtifact;
344
564
  try {
345
565
  prepared = (
@@ -454,7 +674,14 @@ export function compactRetainedScreenshotHistory(
454
674
  return history.map((item) => {
455
675
  const callId = historyCallId(item);
456
676
  const receipt = callId ? receiptsByCallId.get(callId) : undefined;
457
- if (!receipt || !Array.isArray(item.output)) return item;
677
+ if (!receipt) return item;
678
+ if (typeof item.output === "string" && item.output.startsWith("data:image/")) {
679
+ return {
680
+ ...item,
681
+ output: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
682
+ };
683
+ }
684
+ if (!Array.isArray(item.output)) return item;
458
685
  let changed = false;
459
686
  const output = item.output.map((entry) => {
460
687
  if (!isInlineImageContent(entry)) return entry;
@@ -475,7 +702,13 @@ export function collectRetainedScreenshotReceipts(
475
702
  ): Map<string, RetainedArtifactMetadata> {
476
703
  for (const item of history) {
477
704
  const callId = historyCallId(item);
478
- if (!callId || !Array.isArray(item.output)) continue;
705
+ if (!callId) continue;
706
+ const direct = retainedReceiptFromDirectOutput(item.output);
707
+ if (direct) {
708
+ target.set(callId, direct);
709
+ continue;
710
+ }
711
+ if (!Array.isArray(item.output)) continue;
479
712
  for (const entry of item.output) {
480
713
  const receipt = retainedReceiptFromImageContent(entry);
481
714
  if (receipt) {
@@ -576,9 +809,7 @@ async function materializeRetainedScreenshotHistoryWithCache(
576
809
  cache: Map<string, string>,
577
810
  ): Promise<Array<Record<string, unknown>>> {
578
811
  const now = input.now ?? new Date();
579
- const materializeEntry = async (entry: unknown): Promise<unknown> => {
580
- const receipt = retainedReceiptFromImageContent(entry);
581
- if (!receipt) return entry;
812
+ const dataUrlForReceipt = async (receipt: RetainedArtifactMetadata): Promise<string> => {
582
813
  if (!receipt.available) {
583
814
  throw new RetainedScreenshotUnavailableError(receipt.artifactId, receipt.reason);
584
815
  }
@@ -606,9 +837,9 @@ async function materializeRetainedScreenshotHistoryWithCache(
606
837
  const bytes = await input.objectStorage.getFileBytes(artifact.file).catch(() => null);
607
838
  if (!bytes)
608
839
  throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
609
- const validated = validateComputerScreenshot({
840
+ const validated = validateRetainableSessionImage({
610
841
  bytes,
611
- mediaType: artifact.mediaType,
842
+ declaredMediaType: artifact.mediaType,
612
843
  });
613
844
  if (
614
845
  validated.sizeBytes !== artifact.sizeBytes ||
@@ -623,11 +854,25 @@ async function materializeRetainedScreenshotHistoryWithCache(
623
854
  dataUrl = `data:${validated.mediaType};base64,${Buffer.from(validated.bytes).toString("base64")}`;
624
855
  cache.set(receipt.artifactId, dataUrl);
625
856
  }
857
+ return dataUrl;
858
+ };
859
+ const materializeEntry = async (entry: unknown): Promise<unknown> => {
860
+ const receipt = retainedReceiptFromImageContent(entry);
861
+ if (!receipt) return entry;
862
+ const dataUrl = await dataUrlForReceipt(receipt);
626
863
  return { ...(entry as Record<string, unknown>), image: dataUrl };
627
864
  };
628
865
 
629
866
  const materialized: Array<Record<string, unknown>> = [];
630
867
  for (const item of input.history) {
868
+ const directReceipt = retainedReceiptFromDirectOutput(item.output);
869
+ if (directReceipt) {
870
+ materialized.push({
871
+ ...item,
872
+ output: await dataUrlForReceipt(directReceipt),
873
+ });
874
+ continue;
875
+ }
631
876
  if (!Array.isArray(item.output)) {
632
877
  materialized.push(item);
633
878
  continue;
@@ -782,7 +1027,7 @@ async function verifyReadyArtifact(
782
1027
  try {
783
1028
  assertStoredScreenshotHead(await storage.headFile(artifact.file), {
784
1029
  sizeBytes: artifact.sizeBytes,
785
- mediaType: "image/png",
1030
+ mediaType: artifact.mediaType as RetainableSessionImageMediaType,
786
1031
  sha256: artifact.sha256,
787
1032
  });
788
1033
  return true;
@@ -793,7 +1038,7 @@ async function verifyReadyArtifact(
793
1038
 
794
1039
  function assertStoredScreenshotHead(
795
1040
  head: ObjectHead,
796
- expected: Pick<ValidatedScreenshot, "sizeBytes" | "mediaType" | "sha256">,
1041
+ expected: Pick<ValidatedSessionImage, "sizeBytes" | "mediaType" | "sha256">,
797
1042
  ): void {
798
1043
  if (head.ContentLength !== expected.sizeBytes) {
799
1044
  throw new Error("retained screenshot object size mismatch");
@@ -812,7 +1057,18 @@ function retainedReceiptFromImageContent(entry: unknown): RetainedArtifactMetada
812
1057
  if (!image || typeof image !== "object" || Array.isArray(image)) return null;
813
1058
  const marker = image as Record<string, unknown>;
814
1059
  if (marker.type !== RETAINED_IMAGE_MARKER) return null;
815
- const parsed = RetainedArtifactMetadataSchema.safeParse(marker.artifact);
1060
+ return retainedReceipt(marker.artifact);
1061
+ }
1062
+
1063
+ function retainedReceiptFromDirectOutput(output: unknown): RetainedArtifactMetadata | null {
1064
+ if (!output || typeof output !== "object" || Array.isArray(output)) return null;
1065
+ const marker = output as Record<string, unknown>;
1066
+ if (marker.type !== RETAINED_IMAGE_MARKER) return null;
1067
+ return retainedReceipt(marker.artifact);
1068
+ }
1069
+
1070
+ function retainedReceipt(value: unknown): RetainedArtifactMetadata | null {
1071
+ const parsed = RetainedArtifactMetadataSchema.safeParse(value);
816
1072
  if (!parsed.success) return null;
817
1073
  return parsed.data.available && parsed.data.kind !== "computer_screenshot" ? null : parsed.data;
818
1074
  }
@@ -823,7 +1079,7 @@ function isInlineImageContent(entry: unknown): boolean {
823
1079
  return (
824
1080
  record.type === "input_image" &&
825
1081
  typeof record.image === "string" &&
826
- record.image.startsWith("data:image/png;base64,")
1082
+ record.image.startsWith("data:image/")
827
1083
  );
828
1084
  }
829
1085
 
@@ -832,6 +1088,175 @@ function historyCallId(item: Record<string, unknown>): string | null {
832
1088
  return typeof value === "string" && value.length > 0 ? value : null;
833
1089
  }
834
1090
 
1091
+ function canonicalMediaType(value: string): string {
1092
+ return value.split(";", 1)[0]!.trim().toLowerCase();
1093
+ }
1094
+
1095
+ function isPng(bytes: Uint8Array): boolean {
1096
+ return (
1097
+ bytes.byteLength >= PNG_SIGNATURE.byteLength &&
1098
+ PNG_SIGNATURE.every((byte, index) => bytes[index] === byte)
1099
+ );
1100
+ }
1101
+
1102
+ function isJpeg(bytes: Uint8Array): boolean {
1103
+ return bytes.byteLength >= 4 && bytes[0] === 0xff && bytes[1] === 0xd8;
1104
+ }
1105
+
1106
+ function jpegDimensions(bytes: Uint8Array): { width: number; height: number } {
1107
+ if (bytes[bytes.length - 2] !== 0xff || bytes[bytes.length - 1] !== 0xd9) {
1108
+ throw new ScreenshotValidationError(
1109
+ "invalid_content",
1110
+ "session JPEG is truncated or has trailing bytes",
1111
+ );
1112
+ }
1113
+ let offset = 2;
1114
+ while (offset + 4 <= bytes.length) {
1115
+ if (bytes[offset] !== 0xff) {
1116
+ throw new ScreenshotValidationError(
1117
+ "invalid_content",
1118
+ "session JPEG marker table is invalid",
1119
+ );
1120
+ }
1121
+ while (bytes[offset] === 0xff) offset += 1;
1122
+ const marker = bytes[offset++];
1123
+ if (marker === undefined || marker === 0xd9 || marker === 0xda) break;
1124
+ if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue;
1125
+ if (offset + 2 > bytes.length) break;
1126
+ const length = readUint16Be(bytes, offset);
1127
+ if (length < 2 || offset + length > bytes.length) {
1128
+ throw new ScreenshotValidationError("invalid_content", "session JPEG segment is invalid");
1129
+ }
1130
+ if (JPEG_SOF_MARKERS.has(marker)) {
1131
+ if (length < 7) {
1132
+ throw new ScreenshotValidationError("invalid_content", "session JPEG frame is invalid");
1133
+ }
1134
+ return {
1135
+ height: readUint16Be(bytes, offset + 3),
1136
+ width: readUint16Be(bytes, offset + 5),
1137
+ };
1138
+ }
1139
+ offset += length;
1140
+ }
1141
+ throw new ScreenshotValidationError(
1142
+ "invalid_content",
1143
+ "session JPEG has no supported frame header",
1144
+ );
1145
+ }
1146
+
1147
+ const JPEG_SOF_MARKERS = new Set([
1148
+ 0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf,
1149
+ ]);
1150
+
1151
+ function isWebp(bytes: Uint8Array): boolean {
1152
+ return bytes.byteLength >= 20 && ascii(bytes, 0, 4) === "RIFF" && ascii(bytes, 8, 4) === "WEBP";
1153
+ }
1154
+
1155
+ function webpDimensions(bytes: Uint8Array): { width: number; height: number } {
1156
+ if (readUint32Le(bytes, 4) + 8 !== bytes.byteLength) {
1157
+ throw new ScreenshotValidationError("invalid_content", "session WebP RIFF length is invalid");
1158
+ }
1159
+ let offset = 12;
1160
+ while (offset + 8 <= bytes.length) {
1161
+ const type = ascii(bytes, offset, 4);
1162
+ const length = readUint32Le(bytes, offset + 4);
1163
+ const data = offset + 8;
1164
+ const next = data + length + (length & 1);
1165
+ if (next > bytes.length) {
1166
+ throw new ScreenshotValidationError("invalid_content", "session WebP chunk is truncated");
1167
+ }
1168
+ if (type === "VP8X") {
1169
+ if (length < 10) {
1170
+ throw new ScreenshotValidationError("invalid_content", "session WebP VP8X is invalid");
1171
+ }
1172
+ return {
1173
+ width: 1 + readUint24Le(bytes, data + 4),
1174
+ height: 1 + readUint24Le(bytes, data + 7),
1175
+ };
1176
+ }
1177
+ if (type === "VP8 ") {
1178
+ if (
1179
+ length < 10 ||
1180
+ bytes[data + 3] !== 0x9d ||
1181
+ bytes[data + 4] !== 0x01 ||
1182
+ bytes[data + 5] !== 0x2a
1183
+ ) {
1184
+ throw new ScreenshotValidationError("invalid_content", "session WebP VP8 frame is invalid");
1185
+ }
1186
+ return {
1187
+ width: readUint16Le(bytes, data + 6) & 0x3fff,
1188
+ height: readUint16Le(bytes, data + 8) & 0x3fff,
1189
+ };
1190
+ }
1191
+ if (type === "VP8L") {
1192
+ if (length < 5 || bytes[data] !== 0x2f) {
1193
+ throw new ScreenshotValidationError(
1194
+ "invalid_content",
1195
+ "session WebP VP8L frame is invalid",
1196
+ );
1197
+ }
1198
+ const bits = readUint32Le(bytes, data + 1);
1199
+ return {
1200
+ width: 1 + (bits & 0x3fff),
1201
+ height: 1 + ((bits >>> 14) & 0x3fff),
1202
+ };
1203
+ }
1204
+ offset = next;
1205
+ }
1206
+ throw new ScreenshotValidationError("invalid_content", "session WebP has no image frame");
1207
+ }
1208
+
1209
+ function assertImageDimensions(
1210
+ width: number,
1211
+ height: number,
1212
+ limits: { maxDimension: number; maxPixels: number },
1213
+ ): void {
1214
+ if (
1215
+ !Number.isSafeInteger(width) ||
1216
+ !Number.isSafeInteger(height) ||
1217
+ width <= 0 ||
1218
+ height <= 0 ||
1219
+ width > limits.maxDimension ||
1220
+ height > limits.maxDimension ||
1221
+ width * height > limits.maxPixels
1222
+ ) {
1223
+ throw new ScreenshotValidationError("oversized", "session image dimensions exceed policy");
1224
+ }
1225
+ }
1226
+
1227
+ function readUint16Be(bytes: Uint8Array, offset: number): number {
1228
+ return (bytes[offset]! << 8) | bytes[offset + 1]!;
1229
+ }
1230
+
1231
+ function readUint16Le(bytes: Uint8Array, offset: number): number {
1232
+ return bytes[offset]! | (bytes[offset + 1]! << 8);
1233
+ }
1234
+
1235
+ function readUint24Le(bytes: Uint8Array, offset: number): number {
1236
+ return bytes[offset]! | (bytes[offset + 1]! << 8) | (bytes[offset + 2]! << 16);
1237
+ }
1238
+
1239
+ function readUint32Le(bytes: Uint8Array, offset: number): number {
1240
+ return (
1241
+ (bytes[offset]! |
1242
+ (bytes[offset + 1]! << 8) |
1243
+ (bytes[offset + 2]! << 16) |
1244
+ (bytes[offset + 3]! << 24)) >>>
1245
+ 0
1246
+ );
1247
+ }
1248
+
1249
+ function ascii(bytes: Uint8Array, offset: number, length: number): string {
1250
+ return String.fromCharCode(...bytes.subarray(offset, offset + length));
1251
+ }
1252
+
1253
+ function base64Sextet(code: number): number {
1254
+ if (code >= 0x41 && code <= 0x5a) return code - 0x41;
1255
+ if (code >= 0x61 && code <= 0x7a) return code - 0x61 + 26;
1256
+ if (code >= 0x30 && code <= 0x39) return code - 0x30 + 52;
1257
+ return code === 0x2b ? 62 : 63;
1258
+ }
1259
+
835
1260
  function readUint32(bytes: Uint8Array, offset: number): number {
836
1261
  return (
837
1262
  ((bytes[offset]! << 24) |