@opengeni/worker-bundle 0.16.21 → 0.16.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/activities/agent-turn.d.ts +61 -14
  2. package/dist/activities/context-compaction.d.ts +2 -0
  3. package/dist/activities/environment.d.ts +1 -0
  4. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  5. package/dist/activities/git-credential-renewal.d.ts +1 -1
  6. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  7. package/dist/activities/retained-screenshots.d.ts +93 -0
  8. package/dist/activities/run-credential-renewal.d.ts +1 -1
  9. package/dist/activities/run-input.d.ts +2 -0
  10. package/dist/activities/toolspace-token-renewal.d.ts +1 -1
  11. package/dist/activities/workspace-capture.d.ts +1 -0
  12. package/dist/activities.d.ts +3 -0
  13. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  14. package/dist/chunk-RAMNYSAW.js.map +1 -0
  15. package/dist/host-export.js +1 -1
  16. package/dist/index.js +1317 -390
  17. package/dist/index.js.map +1 -1
  18. package/dist/sandbox-resume.d.ts +6 -0
  19. package/dist/sandbox-routing.d.ts +11 -6
  20. package/dist/workflow-bundle.js +2 -1
  21. package/package.json +12 -12
  22. package/src/activities/agent-turn.ts +482 -360
  23. package/src/activities/codex-capacity.ts +10 -1
  24. package/src/activities/codex-rotation.ts +19 -5
  25. package/src/activities/context-compaction.ts +9 -3
  26. package/src/activities/environment.ts +13 -24
  27. package/src/activities/expiring-material-renewal.ts +16 -7
  28. package/src/activities/git-credential-renewal.ts +3 -3
  29. package/src/activities/recording.ts +4 -5
  30. package/src/activities/retained-screenshot-reaper.ts +202 -0
  31. package/src/activities/retained-screenshots.ts +892 -0
  32. package/src/activities/rig-verification.ts +13 -34
  33. package/src/activities/run-credential-renewal.ts +9 -2
  34. package/src/activities/run-input.ts +18 -4
  35. package/src/activities/scheduled-tasks.ts +19 -3
  36. package/src/activities/toolspace-token-renewal.ts +9 -2
  37. package/src/activities/workspace-capture.ts +33 -3
  38. package/src/activities.ts +10 -2
  39. package/src/host-export-pump.ts +28 -11
  40. package/src/http.ts +3 -6
  41. package/src/observability-metrics.ts +3 -3
  42. package/src/sandbox-resume.ts +45 -12
  43. package/src/sandbox-routing.ts +30 -15
  44. package/src/worker-service-lifecycle.ts +8 -4
  45. package/src/workflows/file-upload-reaper.ts +4 -1
  46. package/dist/activities/redaction.d.ts +0 -1
  47. package/dist/chunk-BJUW5LCM.js.map +0 -1
  48. package/src/activities/redaction.ts +0 -9
@@ -0,0 +1,892 @@
1
+ import {
2
+ COMPUTER_SCREENSHOT_MAX_BYTES,
3
+ COMPUTER_SCREENSHOT_MAX_DIMENSION,
4
+ COMPUTER_SCREENSHOT_MAX_PIXELS,
5
+ COMPUTER_SCREENSHOT_RETENTION_MS,
6
+ COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES,
7
+ RetainedArtifactMetadataSchema,
8
+ retainedScreenshotReferenceFromFile,
9
+ type RetainedArtifactMetadata,
10
+ type RetainedArtifactReference,
11
+ type RetainedOutputUnavailableReason,
12
+ } from "@opengeni/contracts";
13
+ import {
14
+ RetainedScreenshotQuotaExceededError,
15
+ getRetainedScreenshotArtifact,
16
+ prepareRetainedScreenshotArtifact,
17
+ recordRetainedScreenshotArtifactError,
18
+ settleRetainedScreenshotArtifactReady,
19
+ type Database,
20
+ type RetainedScreenshotArtifact,
21
+ } from "@opengeni/db";
22
+ import type { ObjectHead, ObjectStorage } from "@opengeni/storage";
23
+ import { createHash } from "node:crypto";
24
+
25
+ const PNG_SIGNATURE = Uint8Array.of(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a);
26
+ const PNG_ALLOWED_CRITICAL_CHUNKS = new Set(["IHDR", "PLTE", "IDAT", "IEND"]);
27
+ const PNG_MAX_CHUNKS = 16_384;
28
+ const RETAINED_IMAGE_MARKER = "retained_artifact";
29
+
30
+ export type TypedScreenshotToolOutput = {
31
+ callId: string;
32
+ toolOutputId: string;
33
+ bytes: Uint8Array;
34
+ mediaType: string;
35
+ };
36
+
37
+ export type ValidatedScreenshot = {
38
+ bytes: Uint8Array;
39
+ mediaType: "image/png";
40
+ sizeBytes: number;
41
+ sha256: string;
42
+ width: number;
43
+ height: number;
44
+ };
45
+
46
+ export class ScreenshotValidationError extends Error {
47
+ constructor(
48
+ readonly reason: "invalid_content" | "oversized" | "unsupported",
49
+ message: string,
50
+ ) {
51
+ super(message);
52
+ this.name = "ScreenshotValidationError";
53
+ }
54
+ }
55
+
56
+ export class RetainedScreenshotUnavailableError extends Error {
57
+ constructor(
58
+ readonly artifactId: string,
59
+ readonly reason: RetainedOutputUnavailableReason,
60
+ ) {
61
+ super(`Retained screenshot ${artifactId} is unavailable: ${reason}`);
62
+ this.name = "RetainedScreenshotUnavailableError";
63
+ }
64
+ }
65
+
66
+ /** Read the SDK's trusted typed image before event/history serialization. */
67
+ export function typedScreenshotFromSdkEvent(event: unknown): TypedScreenshotToolOutput | null {
68
+ if (!event || typeof event !== "object") return null;
69
+ const streamEvent = event as {
70
+ type?: unknown;
71
+ item?: {
72
+ id?: unknown;
73
+ type?: unknown;
74
+ rawItem?: unknown;
75
+ output?: unknown;
76
+ };
77
+ };
78
+ if (streamEvent.type !== "run_item_stream_event") return null;
79
+ const item = streamEvent.item;
80
+ if (!item || item.type !== "tool_call_output_item") return null;
81
+ const output = item.output;
82
+ if (!output || typeof output !== "object" || Array.isArray(output)) return null;
83
+ const outputRecord = output as Record<string, unknown>;
84
+ if (
85
+ outputRecord.type !== "image" ||
86
+ !outputRecord.image ||
87
+ typeof outputRecord.image !== "object"
88
+ ) {
89
+ return null;
90
+ }
91
+ const image = outputRecord.image as Record<string, unknown>;
92
+ if (!(image.data instanceof Uint8Array) || typeof image.mediaType !== "string") return null;
93
+ const raw =
94
+ item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem)
95
+ ? (item.rawItem as Record<string, unknown>)
96
+ : {};
97
+ const callId = raw.callId ?? raw.call_id;
98
+ if (typeof callId !== "string" || callId.length === 0) return null;
99
+ const toolOutputId = raw.id ?? item.id ?? callId;
100
+ if (typeof toolOutputId !== "string" || toolOutputId.length === 0) return null;
101
+ return {
102
+ callId,
103
+ toolOutputId,
104
+ bytes: image.data,
105
+ mediaType: image.mediaType,
106
+ };
107
+ }
108
+
109
+ /** Validate the complete canonical PNG and derive exact integrity metadata. */
110
+ export function validateComputerScreenshot(
111
+ input: { bytes: Uint8Array; mediaType: string },
112
+ limits: {
113
+ maxBytes?: number;
114
+ maxDimension?: number;
115
+ maxPixels?: number;
116
+ } = {},
117
+ ): ValidatedScreenshot {
118
+ const maxBytes = limits.maxBytes ?? COMPUTER_SCREENSHOT_MAX_BYTES;
119
+ const maxDimension = limits.maxDimension ?? COMPUTER_SCREENSHOT_MAX_DIMENSION;
120
+ const maxPixels = limits.maxPixels ?? COMPUTER_SCREENSHOT_MAX_PIXELS;
121
+ if (input.mediaType !== "image/png") {
122
+ throw new ScreenshotValidationError(
123
+ "unsupported",
124
+ "computer screenshot MIME must be image/png",
125
+ );
126
+ }
127
+ if (input.bytes.byteLength === 0) {
128
+ throw new ScreenshotValidationError("invalid_content", "computer screenshot is empty");
129
+ }
130
+ if (input.bytes.byteLength > maxBytes) {
131
+ throw new ScreenshotValidationError(
132
+ "oversized",
133
+ `computer screenshot exceeds ${maxBytes} bytes`,
134
+ );
135
+ }
136
+ if (input.bytes.byteLength < PNG_SIGNATURE.byteLength + 25) {
137
+ throw new ScreenshotValidationError("invalid_content", "computer screenshot PNG is truncated");
138
+ }
139
+ for (let index = 0; index < PNG_SIGNATURE.byteLength; index += 1) {
140
+ if (input.bytes[index] !== PNG_SIGNATURE[index]) {
141
+ throw new ScreenshotValidationError(
142
+ "invalid_content",
143
+ "computer screenshot PNG signature is invalid",
144
+ );
145
+ }
146
+ }
147
+
148
+ let offset = PNG_SIGNATURE.byteLength;
149
+ let chunks = 0;
150
+ let width = 0;
151
+ let height = 0;
152
+ let sawIhdr = false;
153
+ let sawIdat = false;
154
+ let endedIdat = false;
155
+ let sawIend = false;
156
+ while (offset < input.bytes.byteLength) {
157
+ chunks += 1;
158
+ if (chunks > PNG_MAX_CHUNKS || offset + 12 > input.bytes.byteLength) {
159
+ throw new ScreenshotValidationError(
160
+ "invalid_content",
161
+ "computer screenshot PNG chunk table is invalid",
162
+ );
163
+ }
164
+ const length = readUint32(input.bytes, offset);
165
+ const typeOffset = offset + 4;
166
+ const dataOffset = typeOffset + 4;
167
+ const dataEnd = dataOffset + length;
168
+ const crcOffset = dataEnd;
169
+ const nextOffset = crcOffset + 4;
170
+ if (!Number.isSafeInteger(nextOffset) || nextOffset > input.bytes.byteLength) {
171
+ throw new ScreenshotValidationError(
172
+ "invalid_content",
173
+ "computer screenshot PNG chunk is truncated",
174
+ );
175
+ }
176
+ const type = asciiChunkType(input.bytes, typeOffset);
177
+ const expectedCrc = readUint32(input.bytes, crcOffset);
178
+ const actualCrc = crc32(input.bytes.subarray(typeOffset, dataEnd));
179
+ if (expectedCrc !== actualCrc) {
180
+ throw new ScreenshotValidationError(
181
+ "invalid_content",
182
+ `computer screenshot PNG ${type} CRC is invalid`,
183
+ );
184
+ }
185
+ if (chunks === 1 && type !== "IHDR") {
186
+ throw new ScreenshotValidationError(
187
+ "invalid_content",
188
+ "computer screenshot PNG must begin with IHDR",
189
+ );
190
+ }
191
+ if (isCriticalChunk(type) && !PNG_ALLOWED_CRITICAL_CHUNKS.has(type)) {
192
+ throw new ScreenshotValidationError(
193
+ "invalid_content",
194
+ `computer screenshot PNG has unsupported critical chunk ${type}`,
195
+ );
196
+ }
197
+
198
+ if (type === "IHDR") {
199
+ if (sawIhdr || length !== 13) {
200
+ throw new ScreenshotValidationError(
201
+ "invalid_content",
202
+ "computer screenshot PNG IHDR is invalid",
203
+ );
204
+ }
205
+ sawIhdr = true;
206
+ width = readUint32(input.bytes, dataOffset);
207
+ height = readUint32(input.bytes, dataOffset + 4);
208
+ const bitDepth = input.bytes[dataOffset + 8];
209
+ const colorType = input.bytes[dataOffset + 9];
210
+ const compression = input.bytes[dataOffset + 10];
211
+ const filter = input.bytes[dataOffset + 11];
212
+ const interlace = input.bytes[dataOffset + 12];
213
+ if (
214
+ !validPngColorMode(bitDepth, colorType) ||
215
+ compression !== 0 ||
216
+ filter !== 0 ||
217
+ (interlace !== 0 && interlace !== 1)
218
+ ) {
219
+ throw new ScreenshotValidationError(
220
+ "invalid_content",
221
+ "computer screenshot PNG IHDR fields are invalid",
222
+ );
223
+ }
224
+ if (
225
+ width <= 0 ||
226
+ height <= 0 ||
227
+ width > maxDimension ||
228
+ height > maxDimension ||
229
+ width * height > maxPixels
230
+ ) {
231
+ throw new ScreenshotValidationError(
232
+ "oversized",
233
+ "computer screenshot dimensions exceed policy",
234
+ );
235
+ }
236
+ } else if (type === "IDAT") {
237
+ if (!sawIhdr || sawIend || endedIdat || length === 0) {
238
+ throw new ScreenshotValidationError(
239
+ "invalid_content",
240
+ "computer screenshot PNG IDAT ordering is invalid",
241
+ );
242
+ }
243
+ sawIdat = true;
244
+ } else if (sawIdat && type !== "IEND") {
245
+ endedIdat = true;
246
+ }
247
+
248
+ if (type === "IEND") {
249
+ if (!sawIhdr || !sawIdat || sawIend || length !== 0) {
250
+ throw new ScreenshotValidationError(
251
+ "invalid_content",
252
+ "computer screenshot PNG IEND is invalid",
253
+ );
254
+ }
255
+ sawIend = true;
256
+ if (nextOffset !== input.bytes.byteLength) {
257
+ throw new ScreenshotValidationError(
258
+ "invalid_content",
259
+ "computer screenshot PNG has trailing/polyglot bytes",
260
+ );
261
+ }
262
+ }
263
+ offset = nextOffset;
264
+ }
265
+ if (!sawIhdr || !sawIdat || !sawIend || offset !== input.bytes.byteLength) {
266
+ throw new ScreenshotValidationError("invalid_content", "computer screenshot PNG is incomplete");
267
+ }
268
+ return {
269
+ bytes: input.bytes,
270
+ mediaType: "image/png",
271
+ sizeBytes: input.bytes.byteLength,
272
+ sha256: createHash("sha256").update(input.bytes).digest("hex"),
273
+ width,
274
+ height,
275
+ };
276
+ }
277
+
278
+ export function retainedScreenshotIdentity(input: {
279
+ sessionId: string;
280
+ turnId: string;
281
+ attemptId: string;
282
+ toolCallId: string;
283
+ toolOutputId: string;
284
+ }): { artifactId: string; settlementKey: string } {
285
+ const settlementKey = createHash("sha256")
286
+ .update("opengeni:computer-screenshot:v1\0")
287
+ .update(input.sessionId)
288
+ .update("\0")
289
+ .update(input.turnId)
290
+ .update("\0")
291
+ .update(input.attemptId)
292
+ .update("\0")
293
+ .update(input.toolCallId)
294
+ .update("\0")
295
+ .update(input.toolOutputId)
296
+ .digest("hex");
297
+ const uuidBytes = Uint8Array.from(Buffer.from(settlementKey.slice(0, 32), "hex"));
298
+ uuidBytes[6] = (uuidBytes[6]! & 0x0f) | 0x80;
299
+ uuidBytes[8] = (uuidBytes[8]! & 0x3f) | 0x80;
300
+ const hex = Buffer.from(uuidBytes).toString("hex");
301
+ return {
302
+ artifactId: `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`,
303
+ settlementKey,
304
+ };
305
+ }
306
+
307
+ export async function retainComputerScreenshot(input: {
308
+ db: Database;
309
+ objectStorage: ObjectStorage | null;
310
+ accountId: string;
311
+ workspaceId: string;
312
+ sessionId: string;
313
+ turnId: string;
314
+ attemptId: string;
315
+ output: TypedScreenshotToolOutput;
316
+ now?: Date;
317
+ retentionMs?: number;
318
+ workspaceQuotaBytes?: number;
319
+ }): Promise<RetainedArtifactMetadata> {
320
+ const identity = retainedScreenshotIdentity({
321
+ sessionId: input.sessionId,
322
+ turnId: input.turnId,
323
+ attemptId: input.attemptId,
324
+ toolCallId: input.output.callId,
325
+ toolOutputId: input.output.toolOutputId,
326
+ });
327
+ let screenshot: ValidatedScreenshot;
328
+ try {
329
+ screenshot = validateComputerScreenshot(input.output);
330
+ } catch (error) {
331
+ if (error instanceof ScreenshotValidationError) {
332
+ return unavailable(identity.artifactId, error.reason);
333
+ }
334
+ throw error;
335
+ }
336
+ if (!input.objectStorage) return unavailable(identity.artifactId, "missing_storage");
337
+
338
+ const now = input.now ?? new Date();
339
+ const retentionExpiresAt = new Date(
340
+ now.getTime() + (input.retentionMs ?? COMPUTER_SCREENSHOT_RETENTION_MS),
341
+ );
342
+ const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/computer-screenshot.png`;
343
+ let prepared: RetainedScreenshotArtifact;
344
+ try {
345
+ prepared = (
346
+ await prepareRetainedScreenshotArtifact(input.db, {
347
+ artifactId: identity.artifactId,
348
+ accountId: input.accountId,
349
+ workspaceId: input.workspaceId,
350
+ sessionId: input.sessionId,
351
+ turnId: input.turnId,
352
+ attemptId: input.attemptId,
353
+ settlementKey: identity.settlementKey,
354
+ toolCallId: input.output.callId,
355
+ toolOutputId: input.output.toolOutputId,
356
+ mediaType: screenshot.mediaType,
357
+ sizeBytes: screenshot.sizeBytes,
358
+ sha256: screenshot.sha256,
359
+ width: screenshot.width,
360
+ height: screenshot.height,
361
+ retentionExpiresAt,
362
+ bucket: input.objectStorage.bucket,
363
+ objectKey,
364
+ workspaceQuotaBytes: input.workspaceQuotaBytes ?? COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES,
365
+ })
366
+ ).artifact;
367
+ } catch (error) {
368
+ if (error instanceof RetainedScreenshotQuotaExceededError) {
369
+ return unavailable(identity.artifactId, "quota_exceeded");
370
+ }
371
+ throw error;
372
+ }
373
+
374
+ if (prepared.status === "ready") {
375
+ return (await verifyReadyArtifact(input.objectStorage, prepared))
376
+ ? reference(prepared)
377
+ : unavailable(identity.artifactId, "missing_storage");
378
+ }
379
+ if (prepared.status !== "pending" && prepared.status !== "reconciling") {
380
+ return unavailable(identity.artifactId, unavailableReasonForStatus(prepared.status));
381
+ }
382
+
383
+ try {
384
+ await input.objectStorage.putObject({
385
+ key: objectKey,
386
+ contentType: screenshot.mediaType,
387
+ body: screenshot.bytes,
388
+ sha256: screenshot.sha256,
389
+ });
390
+ const head = await input.objectStorage.headFile(prepared.file);
391
+ assertStoredScreenshotHead(head, screenshot);
392
+ } catch (error) {
393
+ await recordRetainedScreenshotArtifactError(input.db, {
394
+ accountId: input.accountId,
395
+ workspaceId: input.workspaceId,
396
+ artifactId: identity.artifactId,
397
+ settlementKey: identity.settlementKey,
398
+ error: error instanceof Error ? error.message : String(error),
399
+ }).catch(() => undefined);
400
+ return unavailable(identity.artifactId, "storage_write_failed");
401
+ }
402
+
403
+ try {
404
+ const settled = await settleRetainedScreenshotArtifactReady(input.db, {
405
+ accountId: input.accountId,
406
+ workspaceId: input.workspaceId,
407
+ artifactId: identity.artifactId,
408
+ settlementKey: identity.settlementKey,
409
+ });
410
+ return reference(settled);
411
+ } catch (error) {
412
+ // PUT completed. If a concurrent cascade removed the deterministic intent,
413
+ // compensate the now-unreferenced key. Otherwise leave the pending row for
414
+ // exact retry/background reconciliation instead of deleting live truth.
415
+ const stillReferenced = await getRetainedScreenshotArtifact(
416
+ input.db,
417
+ input.workspaceId,
418
+ input.sessionId,
419
+ identity.artifactId,
420
+ ).catch(() => undefined);
421
+ if (stillReferenced?.status === "ready") {
422
+ return (await verifyReadyArtifact(input.objectStorage, stillReferenced))
423
+ ? reference(stillReferenced)
424
+ : unavailable(identity.artifactId, "missing_storage");
425
+ }
426
+ if (
427
+ stillReferenced === null ||
428
+ stillReferenced?.status === "cleanup_queued" ||
429
+ stillReferenced?.status === "cleanup_pending" ||
430
+ stillReferenced?.status === "failed" ||
431
+ stillReferenced?.status === "expired" ||
432
+ stillReferenced?.status === "deleted"
433
+ ) {
434
+ await input.objectStorage.deleteObject(objectKey).catch(() => undefined);
435
+ } else if (stillReferenced) {
436
+ await recordRetainedScreenshotArtifactError(input.db, {
437
+ accountId: input.accountId,
438
+ workspaceId: input.workspaceId,
439
+ artifactId: identity.artifactId,
440
+ settlementKey: identity.settlementKey,
441
+ error: error instanceof Error ? error.message : String(error),
442
+ }).catch(() => undefined);
443
+ }
444
+ return unavailable(identity.artifactId, "pending");
445
+ }
446
+ }
447
+
448
+ /** Replace only new screenshot bytes with a compact receipt before first persistence. */
449
+ export function compactRetainedScreenshotHistory(
450
+ history: Array<Record<string, unknown>>,
451
+ receiptsByCallId: ReadonlyMap<string, RetainedArtifactMetadata>,
452
+ ): Array<Record<string, unknown>> {
453
+ if (receiptsByCallId.size === 0) return history;
454
+ return history.map((item) => {
455
+ const callId = historyCallId(item);
456
+ const receipt = callId ? receiptsByCallId.get(callId) : undefined;
457
+ if (!receipt || !Array.isArray(item.output)) return item;
458
+ let changed = false;
459
+ const output = item.output.map((entry) => {
460
+ if (!isInlineImageContent(entry)) return entry;
461
+ changed = true;
462
+ return {
463
+ ...(entry as Record<string, unknown>),
464
+ image: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
465
+ };
466
+ });
467
+ return changed ? { ...item, output } : item;
468
+ });
469
+ }
470
+
471
+ /** Collect durable screenshot receipts before their provider-only expansion. */
472
+ export function collectRetainedScreenshotReceipts(
473
+ history: ReadonlyArray<Record<string, unknown>>,
474
+ target: Map<string, RetainedArtifactMetadata> = new Map(),
475
+ ): Map<string, RetainedArtifactMetadata> {
476
+ for (const item of history) {
477
+ const callId = historyCallId(item);
478
+ if (!callId || !Array.isArray(item.output)) continue;
479
+ for (const entry of item.output) {
480
+ const receipt = retainedReceiptFromImageContent(entry);
481
+ if (receipt) {
482
+ target.set(callId, receipt);
483
+ break;
484
+ }
485
+ }
486
+ }
487
+ return target;
488
+ }
489
+
490
+ /**
491
+ * Compact every protocol-item copy inside an SDK RunState before persistence.
492
+ * The traversal intentionally mirrors the repository's existing serialized
493
+ * RunState sanitizers. A parse failure or no-op returns the original string.
494
+ */
495
+ export function compactRetainedScreenshotRunState(
496
+ serialized: string,
497
+ receiptsByCallId: ReadonlyMap<string, RetainedArtifactMetadata>,
498
+ ): string {
499
+ if (receiptsByCallId.size === 0) return serialized;
500
+ const parsed = parseSerializedRunState(serialized);
501
+ if (!parsed) return serialized;
502
+ let changed = false;
503
+ visitSerializedRunStateItemArrays(parsed, (items) => {
504
+ const compacted = compactRetainedScreenshotHistory(items, receiptsByCallId);
505
+ if (compacted !== items && compacted.some((item, index) => item !== items[index])) {
506
+ items.splice(0, items.length, ...compacted);
507
+ changed = true;
508
+ }
509
+ });
510
+ return changed ? JSON.stringify(parsed) : serialized;
511
+ }
512
+
513
+ export function collectRetainedScreenshotRunStateReceipts(
514
+ serialized: string,
515
+ target: Map<string, RetainedArtifactMetadata> = new Map(),
516
+ ): Map<string, RetainedArtifactMetadata> {
517
+ const parsed = parseSerializedRunState(serialized);
518
+ if (!parsed) return target;
519
+ visitSerializedRunStateItemArrays(parsed, (items) => {
520
+ collectRetainedScreenshotReceipts(items, target);
521
+ });
522
+ return target;
523
+ }
524
+
525
+ /**
526
+ * Materialize canonical receipts to the exact SDK input_image data URL shape.
527
+ * Missing/deleted/expired/corrupt bytes fail closed; no item is omitted,
528
+ * reordered, rewritten in storage, or converted to text.
529
+ */
530
+ export async function materializeRetainedScreenshotHistory(input: {
531
+ db: Database;
532
+ objectStorage: ObjectStorage | null;
533
+ workspaceId: string;
534
+ sessionId: string;
535
+ history: Array<Record<string, unknown>>;
536
+ now?: Date;
537
+ }): Promise<Array<Record<string, unknown>>> {
538
+ return await materializeRetainedScreenshotHistoryWithCache(input, new Map());
539
+ }
540
+
541
+ /** Expand durable markers in an SDK RunState immediately before resume. */
542
+ export async function materializeRetainedScreenshotRunState(input: {
543
+ db: Database;
544
+ objectStorage: ObjectStorage | null;
545
+ workspaceId: string;
546
+ sessionId: string;
547
+ serialized: string;
548
+ now?: Date;
549
+ }): Promise<string> {
550
+ const parsed = parseSerializedRunState(input.serialized);
551
+ if (!parsed) return input.serialized;
552
+ const cache = new Map<string, string>();
553
+ let changed = false;
554
+ await visitSerializedRunStateItemArraysAsync(parsed, async (items) => {
555
+ const materialized = await materializeRetainedScreenshotHistoryWithCache(
556
+ { ...input, history: items },
557
+ cache,
558
+ );
559
+ if (materialized.some((item, index) => item !== items[index])) {
560
+ items.splice(0, items.length, ...materialized);
561
+ changed = true;
562
+ }
563
+ });
564
+ return changed ? JSON.stringify(parsed) : input.serialized;
565
+ }
566
+
567
+ async function materializeRetainedScreenshotHistoryWithCache(
568
+ input: {
569
+ db: Database;
570
+ objectStorage: ObjectStorage | null;
571
+ workspaceId: string;
572
+ sessionId: string;
573
+ history: Array<Record<string, unknown>>;
574
+ now?: Date;
575
+ },
576
+ cache: Map<string, string>,
577
+ ): Promise<Array<Record<string, unknown>>> {
578
+ const now = input.now ?? new Date();
579
+ const materializeEntry = async (entry: unknown): Promise<unknown> => {
580
+ const receipt = retainedReceiptFromImageContent(entry);
581
+ if (!receipt) return entry;
582
+ if (!receipt.available) {
583
+ throw new RetainedScreenshotUnavailableError(receipt.artifactId, receipt.reason);
584
+ }
585
+ let dataUrl = cache.get(receipt.artifactId);
586
+ if (!dataUrl) {
587
+ if (!input.objectStorage) {
588
+ throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
589
+ }
590
+ const artifact = await getRetainedScreenshotArtifact(
591
+ input.db,
592
+ input.workspaceId,
593
+ input.sessionId,
594
+ receipt.artifactId,
595
+ );
596
+ if (!artifact) throw new RetainedScreenshotUnavailableError(receipt.artifactId, "deleted");
597
+ if (artifact.status !== "ready") {
598
+ throw new RetainedScreenshotUnavailableError(
599
+ receipt.artifactId,
600
+ unavailableReasonForStatus(artifact.status),
601
+ );
602
+ }
603
+ if (artifact.retentionExpiresAt.getTime() <= now.getTime()) {
604
+ throw new RetainedScreenshotUnavailableError(receipt.artifactId, "expired");
605
+ }
606
+ const bytes = await input.objectStorage.getFileBytes(artifact.file).catch(() => null);
607
+ if (!bytes)
608
+ throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
609
+ const validated = validateComputerScreenshot({
610
+ bytes,
611
+ mediaType: artifact.mediaType,
612
+ });
613
+ if (
614
+ validated.sizeBytes !== artifact.sizeBytes ||
615
+ validated.sha256 !== artifact.sha256 ||
616
+ validated.width !== artifact.width ||
617
+ validated.height !== artifact.height ||
618
+ receipt.sha256 !== artifact.sha256 ||
619
+ receipt.originalBytes !== artifact.sizeBytes
620
+ ) {
621
+ throw new RetainedScreenshotUnavailableError(receipt.artifactId, "invalid_content");
622
+ }
623
+ dataUrl = `data:${validated.mediaType};base64,${Buffer.from(validated.bytes).toString("base64")}`;
624
+ cache.set(receipt.artifactId, dataUrl);
625
+ }
626
+ return { ...(entry as Record<string, unknown>), image: dataUrl };
627
+ };
628
+
629
+ const materialized: Array<Record<string, unknown>> = [];
630
+ for (const item of input.history) {
631
+ if (!Array.isArray(item.output)) {
632
+ materialized.push(item);
633
+ continue;
634
+ }
635
+ let changed = false;
636
+ const output: unknown[] = [];
637
+ for (const entry of item.output) {
638
+ const next = await materializeEntry(entry);
639
+ changed ||= next !== entry;
640
+ output.push(next);
641
+ }
642
+ materialized.push(changed ? { ...item, output } : item);
643
+ }
644
+ return materialized;
645
+ }
646
+
647
+ function parseSerializedRunState(serialized: string): Record<string, unknown> | null {
648
+ try {
649
+ const parsed: unknown = JSON.parse(serialized);
650
+ return parsed && typeof parsed === "object" && !Array.isArray(parsed)
651
+ ? (parsed as Record<string, unknown>)
652
+ : null;
653
+ } catch {
654
+ return null;
655
+ }
656
+ }
657
+
658
+ function visitSerializedRunStateItemArrays(
659
+ root: Record<string, unknown>,
660
+ visitor: (items: Array<Record<string, unknown>>) => void,
661
+ ): void {
662
+ const originalInput = root.originalInput;
663
+ if (Array.isArray(originalInput)) {
664
+ visitor(originalInput as Array<Record<string, unknown>>);
665
+ }
666
+ if (Array.isArray(root.generatedItems)) {
667
+ for (const wrapper of root.generatedItems) {
668
+ if (!wrapper || typeof wrapper !== "object" || Array.isArray(wrapper)) continue;
669
+ const record = wrapper as Record<string, unknown>;
670
+ const rawItem = record.rawItem;
671
+ if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
672
+ const items = [rawItem as Record<string, unknown>];
673
+ visitor(items);
674
+ if (items[0] !== rawItem) record.rawItem = items[0];
675
+ }
676
+ }
677
+ if (Array.isArray(root.modelResponses)) {
678
+ for (const response of root.modelResponses) {
679
+ if (!response || typeof response !== "object" || Array.isArray(response)) continue;
680
+ const output = (response as Record<string, unknown>).output;
681
+ if (Array.isArray(output)) visitor(output as Array<Record<string, unknown>>);
682
+ }
683
+ }
684
+ const lastModelResponse = root.lastModelResponse;
685
+ if (
686
+ lastModelResponse &&
687
+ typeof lastModelResponse === "object" &&
688
+ !Array.isArray(lastModelResponse)
689
+ ) {
690
+ const output = (lastModelResponse as Record<string, unknown>).output;
691
+ if (Array.isArray(output)) visitor(output as Array<Record<string, unknown>>);
692
+ }
693
+ }
694
+
695
+ async function visitSerializedRunStateItemArraysAsync(
696
+ root: Record<string, unknown>,
697
+ visitor: (items: Array<Record<string, unknown>>) => Promise<void>,
698
+ ): Promise<void> {
699
+ const originalInput = root.originalInput;
700
+ if (Array.isArray(originalInput)) {
701
+ await visitor(originalInput as Array<Record<string, unknown>>);
702
+ }
703
+ if (Array.isArray(root.generatedItems)) {
704
+ for (const wrapper of root.generatedItems) {
705
+ if (!wrapper || typeof wrapper !== "object" || Array.isArray(wrapper)) continue;
706
+ const record = wrapper as Record<string, unknown>;
707
+ const rawItem = record.rawItem;
708
+ if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
709
+ const items = [rawItem as Record<string, unknown>];
710
+ await visitor(items);
711
+ if (items[0] !== rawItem) record.rawItem = items[0];
712
+ }
713
+ }
714
+ if (Array.isArray(root.modelResponses)) {
715
+ for (const response of root.modelResponses) {
716
+ if (!response || typeof response !== "object" || Array.isArray(response)) continue;
717
+ const output = (response as Record<string, unknown>).output;
718
+ if (Array.isArray(output)) await visitor(output as Array<Record<string, unknown>>);
719
+ }
720
+ }
721
+ const lastModelResponse = root.lastModelResponse;
722
+ if (
723
+ lastModelResponse &&
724
+ typeof lastModelResponse === "object" &&
725
+ !Array.isArray(lastModelResponse)
726
+ ) {
727
+ const output = (lastModelResponse as Record<string, unknown>).output;
728
+ if (Array.isArray(output)) await visitor(output as Array<Record<string, unknown>>);
729
+ }
730
+ }
731
+
732
+ function reference(artifact: RetainedScreenshotArtifact): RetainedArtifactReference {
733
+ if (!artifact.sessionId) {
734
+ throw new Error(`Ready retained screenshot is detached: ${artifact.artifactId}`);
735
+ }
736
+ const value = retainedScreenshotReferenceFromFile({
737
+ ...artifact.file,
738
+ sessionId: artifact.sessionId,
739
+ width: artifact.width,
740
+ height: artifact.height,
741
+ expiresAt: artifact.retentionExpiresAt.toISOString(),
742
+ });
743
+ if (!value)
744
+ throw new Error(`Ready retained screenshot receipt is invalid: ${artifact.artifactId}`);
745
+ return value;
746
+ }
747
+
748
+ function unavailable(
749
+ artifactId: string,
750
+ reason: RetainedOutputUnavailableReason,
751
+ ): RetainedArtifactMetadata {
752
+ return RetainedArtifactMetadataSchema.parse({
753
+ available: false,
754
+ artifactId,
755
+ reason,
756
+ });
757
+ }
758
+
759
+ function unavailableReasonForStatus(
760
+ status: RetainedScreenshotArtifact["status"],
761
+ ): RetainedOutputUnavailableReason {
762
+ switch (status) {
763
+ case "pending":
764
+ case "reconciling":
765
+ return "pending";
766
+ case "cleanup_queued":
767
+ case "cleanup_pending":
768
+ return "failed";
769
+ case "failed":
770
+ case "expired":
771
+ case "deleted":
772
+ return status;
773
+ case "ready":
774
+ return "missing_storage";
775
+ }
776
+ }
777
+
778
+ async function verifyReadyArtifact(
779
+ storage: ObjectStorage,
780
+ artifact: RetainedScreenshotArtifact,
781
+ ): Promise<boolean> {
782
+ try {
783
+ assertStoredScreenshotHead(await storage.headFile(artifact.file), {
784
+ sizeBytes: artifact.sizeBytes,
785
+ mediaType: "image/png",
786
+ sha256: artifact.sha256,
787
+ });
788
+ return true;
789
+ } catch {
790
+ return false;
791
+ }
792
+ }
793
+
794
+ function assertStoredScreenshotHead(
795
+ head: ObjectHead,
796
+ expected: Pick<ValidatedScreenshot, "sizeBytes" | "mediaType" | "sha256">,
797
+ ): void {
798
+ if (head.ContentLength !== expected.sizeBytes) {
799
+ throw new Error("retained screenshot object size mismatch");
800
+ }
801
+ if (head.ContentType !== expected.mediaType) {
802
+ throw new Error("retained screenshot object MIME mismatch");
803
+ }
804
+ if (head.Metadata?.sha256 !== expected.sha256) {
805
+ throw new Error("retained screenshot object SHA-256 metadata mismatch");
806
+ }
807
+ }
808
+
809
+ function retainedReceiptFromImageContent(entry: unknown): RetainedArtifactMetadata | null {
810
+ if (!entry || typeof entry !== "object" || Array.isArray(entry)) return null;
811
+ const image = (entry as Record<string, unknown>).image;
812
+ if (!image || typeof image !== "object" || Array.isArray(image)) return null;
813
+ const marker = image as Record<string, unknown>;
814
+ if (marker.type !== RETAINED_IMAGE_MARKER) return null;
815
+ const parsed = RetainedArtifactMetadataSchema.safeParse(marker.artifact);
816
+ if (!parsed.success) return null;
817
+ return parsed.data.available && parsed.data.kind !== "computer_screenshot" ? null : parsed.data;
818
+ }
819
+
820
+ function isInlineImageContent(entry: unknown): boolean {
821
+ if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
822
+ const record = entry as Record<string, unknown>;
823
+ return (
824
+ record.type === "input_image" &&
825
+ typeof record.image === "string" &&
826
+ record.image.startsWith("data:image/png;base64,")
827
+ );
828
+ }
829
+
830
+ function historyCallId(item: Record<string, unknown>): string | null {
831
+ const value = item.callId ?? item.call_id;
832
+ return typeof value === "string" && value.length > 0 ? value : null;
833
+ }
834
+
835
+ function readUint32(bytes: Uint8Array, offset: number): number {
836
+ return (
837
+ ((bytes[offset]! << 24) |
838
+ (bytes[offset + 1]! << 16) |
839
+ (bytes[offset + 2]! << 8) |
840
+ bytes[offset + 3]!) >>>
841
+ 0
842
+ );
843
+ }
844
+
845
+ function asciiChunkType(bytes: Uint8Array, offset: number): string {
846
+ const chars = [bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]];
847
+ if (
848
+ chars.some(
849
+ (value) => value === undefined || value < 65 || (value > 90 && value < 97) || value > 122,
850
+ )
851
+ ) {
852
+ throw new ScreenshotValidationError(
853
+ "invalid_content",
854
+ "computer screenshot PNG chunk type is invalid",
855
+ );
856
+ }
857
+ return String.fromCharCode(...(chars as number[]));
858
+ }
859
+
860
+ function isCriticalChunk(type: string): boolean {
861
+ return type.charCodeAt(0) >= 65 && type.charCodeAt(0) <= 90;
862
+ }
863
+
864
+ function validPngColorMode(bitDepth: number | undefined, colorType: number | undefined): boolean {
865
+ if (bitDepth === undefined || colorType === undefined) return false;
866
+ switch (colorType) {
867
+ case 0:
868
+ return [1, 2, 4, 8, 16].includes(bitDepth);
869
+ case 2:
870
+ case 4:
871
+ case 6:
872
+ return bitDepth === 8 || bitDepth === 16;
873
+ case 3:
874
+ return [1, 2, 4, 8].includes(bitDepth);
875
+ default:
876
+ return false;
877
+ }
878
+ }
879
+
880
+ const CRC32_TABLE = Uint32Array.from({ length: 256 }, (_, value) => {
881
+ let current = value;
882
+ for (let bit = 0; bit < 8; bit += 1) {
883
+ current = (current & 1) !== 0 ? 0xedb88320 ^ (current >>> 1) : current >>> 1;
884
+ }
885
+ return current >>> 0;
886
+ });
887
+
888
+ function crc32(bytes: Uint8Array): number {
889
+ let crc = 0xffffffff;
890
+ for (const byte of bytes) crc = CRC32_TABLE[(crc ^ byte) & 0xff]! ^ (crc >>> 8);
891
+ return (crc ^ 0xffffffff) >>> 0;
892
+ }