@opengeni/worker-bundle 0.16.29 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +85 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +662 -17374
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +2 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1043 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1470 -246
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +6 -6
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +45 -42
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +77 -4
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +48 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +199 -4
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +19 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +3 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
|
@@ -43,6 +43,18 @@ export type ValidatedScreenshot = {
|
|
|
43
43
|
height: number;
|
|
44
44
|
};
|
|
45
45
|
|
|
46
|
+
export type RetainableSessionImageMediaType = "image/png" | "image/jpeg" | "image/webp";
|
|
47
|
+
|
|
48
|
+
export type ValidatedSessionImage = {
|
|
49
|
+
bytes: Uint8Array;
|
|
50
|
+
mediaType: RetainableSessionImageMediaType;
|
|
51
|
+
extension: "png" | "jpg" | "webp";
|
|
52
|
+
sizeBytes: number;
|
|
53
|
+
sha256: string;
|
|
54
|
+
width: number;
|
|
55
|
+
height: number;
|
|
56
|
+
};
|
|
57
|
+
|
|
46
58
|
export class ScreenshotValidationError extends Error {
|
|
47
59
|
constructor(
|
|
48
60
|
readonly reason: "invalid_content" | "oversized" | "unsupported",
|
|
@@ -63,6 +75,21 @@ export class RetainedScreenshotUnavailableError extends Error {
|
|
|
63
75
|
}
|
|
64
76
|
}
|
|
65
77
|
|
|
78
|
+
/** Read an intentional image directly at the tool execution boundary. */
|
|
79
|
+
export function typedScreenshotFromToolOutput(input: {
|
|
80
|
+
callId: string;
|
|
81
|
+
output: unknown;
|
|
82
|
+
}): TypedScreenshotToolOutput | null {
|
|
83
|
+
const image = imageBytesFromSdkToolOutput(input.output);
|
|
84
|
+
if (!image) return null;
|
|
85
|
+
return {
|
|
86
|
+
callId: input.callId,
|
|
87
|
+
toolOutputId: input.callId,
|
|
88
|
+
bytes: image.bytes,
|
|
89
|
+
mediaType: image.mediaType,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
66
93
|
/** Read the SDK's trusted typed image before event/history serialization. */
|
|
67
94
|
export function typedScreenshotFromSdkEvent(event: unknown): TypedScreenshotToolOutput | null {
|
|
68
95
|
if (!event || typeof event !== "object") return null;
|
|
@@ -78,34 +105,144 @@ export function typedScreenshotFromSdkEvent(event: unknown): TypedScreenshotTool
|
|
|
78
105
|
if (streamEvent.type !== "run_item_stream_event") return null;
|
|
79
106
|
const item = streamEvent.item;
|
|
80
107
|
if (!item || item.type !== "tool_call_output_item") return null;
|
|
81
|
-
const
|
|
82
|
-
if (!
|
|
83
|
-
const outputRecord = output as Record<string, unknown>;
|
|
84
|
-
if (
|
|
85
|
-
outputRecord.type !== "image" ||
|
|
86
|
-
!outputRecord.image ||
|
|
87
|
-
typeof outputRecord.image !== "object"
|
|
88
|
-
) {
|
|
89
|
-
return null;
|
|
90
|
-
}
|
|
91
|
-
const image = outputRecord.image as Record<string, unknown>;
|
|
92
|
-
if (!(image.data instanceof Uint8Array) || typeof image.mediaType !== "string") return null;
|
|
108
|
+
const image = imageBytesFromSdkToolOutput(item.output);
|
|
109
|
+
if (!image) return null;
|
|
93
110
|
const raw =
|
|
94
111
|
item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem)
|
|
95
112
|
? (item.rawItem as Record<string, unknown>)
|
|
96
113
|
: {};
|
|
97
|
-
|
|
114
|
+
// Live Agents SDK results can carry the call identity only on the run-item
|
|
115
|
+
// wrapper; reconciled history later copies it into `rawItem`. Use the same
|
|
116
|
+
// fallback as the canonical completed-call extractor so both paths agree.
|
|
117
|
+
const callId = raw.callId ?? raw.call_id ?? item.id;
|
|
98
118
|
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
99
119
|
const toolOutputId = raw.id ?? item.id ?? callId;
|
|
100
120
|
if (typeof toolOutputId !== "string" || toolOutputId.length === 0) return null;
|
|
101
121
|
return {
|
|
102
122
|
callId,
|
|
103
123
|
toolOutputId,
|
|
104
|
-
bytes: image.
|
|
124
|
+
bytes: image.bytes,
|
|
105
125
|
mediaType: image.mediaType,
|
|
106
126
|
};
|
|
107
127
|
}
|
|
108
128
|
|
|
129
|
+
/** Does this SDK result still contain inline bytes that persistence must replace? */
|
|
130
|
+
export function sdkEventContainsInlineImage(event: unknown): boolean {
|
|
131
|
+
if (!event || typeof event !== "object") return false;
|
|
132
|
+
const streamEvent = event as {
|
|
133
|
+
type?: unknown;
|
|
134
|
+
item?: { type?: unknown; output?: unknown };
|
|
135
|
+
};
|
|
136
|
+
return (
|
|
137
|
+
streamEvent.type === "run_item_stream_event" &&
|
|
138
|
+
streamEvent.item?.type === "tool_call_output_item" &&
|
|
139
|
+
toolOutputContainsInlineImage(streamEvent.item.output)
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function imageBytesFromSdkToolOutput(
|
|
144
|
+
output: unknown,
|
|
145
|
+
): { bytes: Uint8Array; mediaType: string } | null {
|
|
146
|
+
if (typeof output === "string") return decodeInlineImageDataUrl(output);
|
|
147
|
+
if (Array.isArray(output)) {
|
|
148
|
+
for (const entry of output) {
|
|
149
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) continue;
|
|
150
|
+
const record = entry as Record<string, unknown>;
|
|
151
|
+
if (record.type !== "input_image") continue;
|
|
152
|
+
const decoded = imageBytesFromSdkImageSource(
|
|
153
|
+
record.image ?? record.image_url ?? record.imageUrl,
|
|
154
|
+
);
|
|
155
|
+
if (decoded) return decoded;
|
|
156
|
+
}
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
if (!output || typeof output !== "object") return null;
|
|
160
|
+
const outputRecord = output as Record<string, unknown>;
|
|
161
|
+
if (
|
|
162
|
+
outputRecord.type !== "image" ||
|
|
163
|
+
!outputRecord.image ||
|
|
164
|
+
typeof outputRecord.image !== "object" ||
|
|
165
|
+
Array.isArray(outputRecord.image)
|
|
166
|
+
) {
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
return imageBytesFromSdkImageSource(outputRecord.image);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function imageBytesFromSdkImageSource(
|
|
173
|
+
source: unknown,
|
|
174
|
+
): { bytes: Uint8Array; mediaType: string } | null {
|
|
175
|
+
const direct = decodeInlineImageDataUrl(source);
|
|
176
|
+
if (direct) return direct;
|
|
177
|
+
if (!source || typeof source !== "object" || Array.isArray(source)) return null;
|
|
178
|
+
const image = source as Record<string, unknown>;
|
|
179
|
+
const fromUrl = decodeInlineImageDataUrl(image.url);
|
|
180
|
+
if (fromUrl) return fromUrl;
|
|
181
|
+
return image.data instanceof Uint8Array && typeof image.mediaType === "string"
|
|
182
|
+
? { bytes: image.data, mediaType: image.mediaType }
|
|
183
|
+
: null;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function sdkImageSourceContainsInlineImage(source: unknown): boolean {
|
|
187
|
+
if (typeof source === "string") return source.startsWith("data:image/");
|
|
188
|
+
if (!source || typeof source !== "object" || Array.isArray(source)) return false;
|
|
189
|
+
const image = source as Record<string, unknown>;
|
|
190
|
+
return (
|
|
191
|
+
(typeof image.url === "string" && image.url.startsWith("data:image/")) ||
|
|
192
|
+
(image.data instanceof Uint8Array && typeof image.mediaType === "string")
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export function toolOutputContainsInlineImage(output: unknown): boolean {
|
|
197
|
+
if (typeof output === "string") return output.startsWith("data:image/");
|
|
198
|
+
if (Array.isArray(output)) {
|
|
199
|
+
return output.some((entry) => {
|
|
200
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
|
|
201
|
+
const record = entry as Record<string, unknown>;
|
|
202
|
+
return (
|
|
203
|
+
record.type === "input_image" &&
|
|
204
|
+
sdkImageSourceContainsInlineImage(record.image ?? record.image_url ?? record.imageUrl)
|
|
205
|
+
);
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
if (!output || typeof output !== "object") return false;
|
|
209
|
+
const record = output as Record<string, unknown>;
|
|
210
|
+
return record.type === "image" && sdkImageSourceContainsInlineImage(record.image);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function decodeInlineImageDataUrl(value: unknown): { bytes: Uint8Array; mediaType: string } | null {
|
|
214
|
+
if (typeof value !== "string") return null;
|
|
215
|
+
const match = /^data:(image\/(?:png|jpeg|webp));base64,/u.exec(value);
|
|
216
|
+
if (!match?.[1]) return null;
|
|
217
|
+
const mediaType = match[1] as RetainableSessionImageMediaType;
|
|
218
|
+
const encoded = value.slice(match[0].length);
|
|
219
|
+
const maxEncodedLength = Math.ceil(COMPUTER_SCREENSHOT_MAX_BYTES / 3) * 4;
|
|
220
|
+
if (
|
|
221
|
+
encoded.length === 0 ||
|
|
222
|
+
encoded.length > maxEncodedLength ||
|
|
223
|
+
encoded.length % 4 !== 0 ||
|
|
224
|
+
!/^[A-Za-z0-9+/]+={0,2}$/.test(encoded)
|
|
225
|
+
) {
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
const padding = encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0;
|
|
229
|
+
const decodedLength = (encoded.length / 4) * 3 - padding;
|
|
230
|
+
if (decodedLength <= 0 || decodedLength > COMPUTER_SCREENSHOT_MAX_BYTES) {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
const lastSextet = base64Sextet(encoded.charCodeAt(encoded.length - padding - 1));
|
|
234
|
+
if (
|
|
235
|
+
(padding === 2 && (lastSextet & 0x0f) !== 0) ||
|
|
236
|
+
(padding === 1 && (lastSextet & 0x03) !== 0)
|
|
237
|
+
) {
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
const decoded = Buffer.from(encoded, "base64");
|
|
241
|
+
if (decoded.byteLength !== decodedLength) return null;
|
|
242
|
+
const bytes = new Uint8Array(decoded.buffer, decoded.byteOffset, decoded.byteLength);
|
|
243
|
+
return { bytes, mediaType };
|
|
244
|
+
}
|
|
245
|
+
|
|
109
246
|
/** Validate the complete canonical PNG and derive exact integrity metadata. */
|
|
110
247
|
export function validateComputerScreenshot(
|
|
111
248
|
input: { bytes: Uint8Array; mediaType: string },
|
|
@@ -275,6 +412,73 @@ export function validateComputerScreenshot(
|
|
|
275
412
|
};
|
|
276
413
|
}
|
|
277
414
|
|
|
415
|
+
/** Validate a durable view_image/computer image from bytes, never from its claimed MIME alone. */
|
|
416
|
+
export function validateRetainableSessionImage(
|
|
417
|
+
input: { bytes: Uint8Array; declaredMediaType?: string },
|
|
418
|
+
limits: {
|
|
419
|
+
maxBytes?: number;
|
|
420
|
+
maxDimension?: number;
|
|
421
|
+
maxPixels?: number;
|
|
422
|
+
} = {},
|
|
423
|
+
): ValidatedSessionImage {
|
|
424
|
+
const maxBytes = limits.maxBytes ?? COMPUTER_SCREENSHOT_MAX_BYTES;
|
|
425
|
+
const maxDimension = limits.maxDimension ?? COMPUTER_SCREENSHOT_MAX_DIMENSION;
|
|
426
|
+
const maxPixels = limits.maxPixels ?? COMPUTER_SCREENSHOT_MAX_PIXELS;
|
|
427
|
+
if (input.bytes.byteLength === 0) {
|
|
428
|
+
throw new ScreenshotValidationError("invalid_content", "session image is empty");
|
|
429
|
+
}
|
|
430
|
+
if (input.bytes.byteLength > maxBytes) {
|
|
431
|
+
throw new ScreenshotValidationError("oversized", `session image exceeds ${maxBytes} bytes`);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
let identity: Pick<ValidatedSessionImage, "mediaType" | "extension" | "width" | "height">;
|
|
435
|
+
if (isPng(input.bytes)) {
|
|
436
|
+
const png = validateComputerScreenshot(
|
|
437
|
+
{ bytes: input.bytes, mediaType: "image/png" },
|
|
438
|
+
{ maxBytes, maxDimension, maxPixels },
|
|
439
|
+
);
|
|
440
|
+
identity = {
|
|
441
|
+
mediaType: "image/png",
|
|
442
|
+
extension: "png",
|
|
443
|
+
width: png.width,
|
|
444
|
+
height: png.height,
|
|
445
|
+
};
|
|
446
|
+
} else if (isJpeg(input.bytes)) {
|
|
447
|
+
identity = {
|
|
448
|
+
mediaType: "image/jpeg",
|
|
449
|
+
extension: "jpg",
|
|
450
|
+
...jpegDimensions(input.bytes),
|
|
451
|
+
};
|
|
452
|
+
} else if (isWebp(input.bytes)) {
|
|
453
|
+
identity = {
|
|
454
|
+
mediaType: "image/webp",
|
|
455
|
+
extension: "webp",
|
|
456
|
+
...webpDimensions(input.bytes),
|
|
457
|
+
};
|
|
458
|
+
} else {
|
|
459
|
+
throw new ScreenshotValidationError("unsupported", "session image format is unsupported");
|
|
460
|
+
}
|
|
461
|
+
assertImageDimensions(identity.width, identity.height, {
|
|
462
|
+
maxDimension,
|
|
463
|
+
maxPixels,
|
|
464
|
+
});
|
|
465
|
+
if (
|
|
466
|
+
input.declaredMediaType &&
|
|
467
|
+
canonicalMediaType(input.declaredMediaType) !== identity.mediaType
|
|
468
|
+
) {
|
|
469
|
+
throw new ScreenshotValidationError(
|
|
470
|
+
"invalid_content",
|
|
471
|
+
"session image MIME does not match its bytes",
|
|
472
|
+
);
|
|
473
|
+
}
|
|
474
|
+
return {
|
|
475
|
+
bytes: input.bytes,
|
|
476
|
+
...identity,
|
|
477
|
+
sizeBytes: input.bytes.byteLength,
|
|
478
|
+
sha256: createHash("sha256").update(input.bytes).digest("hex"),
|
|
479
|
+
};
|
|
480
|
+
}
|
|
481
|
+
|
|
278
482
|
export function retainedScreenshotIdentity(input: {
|
|
279
483
|
sessionId: string;
|
|
280
484
|
turnId: string;
|
|
@@ -304,6 +508,19 @@ export function retainedScreenshotIdentity(input: {
|
|
|
304
508
|
};
|
|
305
509
|
}
|
|
306
510
|
|
|
511
|
+
/** Deterministic fail-closed receipt installed before retention can perform I/O. */
|
|
512
|
+
export function unavailableRetainedSessionImage(input: {
|
|
513
|
+
sessionId: string;
|
|
514
|
+
turnId: string;
|
|
515
|
+
attemptId: string;
|
|
516
|
+
toolCallId: string;
|
|
517
|
+
toolOutputId: string;
|
|
518
|
+
reason: RetainedOutputUnavailableReason;
|
|
519
|
+
}): RetainedArtifactMetadata {
|
|
520
|
+
const identity = retainedScreenshotIdentity(input);
|
|
521
|
+
return unavailable(identity.artifactId, input.reason);
|
|
522
|
+
}
|
|
523
|
+
|
|
307
524
|
export async function retainComputerScreenshot(input: {
|
|
308
525
|
db: Database;
|
|
309
526
|
objectStorage: ObjectStorage | null;
|
|
@@ -324,9 +541,12 @@ export async function retainComputerScreenshot(input: {
|
|
|
324
541
|
toolCallId: input.output.callId,
|
|
325
542
|
toolOutputId: input.output.toolOutputId,
|
|
326
543
|
});
|
|
327
|
-
let screenshot:
|
|
544
|
+
let screenshot: ValidatedSessionImage;
|
|
328
545
|
try {
|
|
329
|
-
screenshot =
|
|
546
|
+
screenshot = validateRetainableSessionImage({
|
|
547
|
+
bytes: input.output.bytes,
|
|
548
|
+
declaredMediaType: input.output.mediaType,
|
|
549
|
+
});
|
|
330
550
|
} catch (error) {
|
|
331
551
|
if (error instanceof ScreenshotValidationError) {
|
|
332
552
|
return unavailable(identity.artifactId, error.reason);
|
|
@@ -339,7 +559,7 @@ export async function retainComputerScreenshot(input: {
|
|
|
339
559
|
const retentionExpiresAt = new Date(
|
|
340
560
|
now.getTime() + (input.retentionMs ?? COMPUTER_SCREENSHOT_RETENTION_MS),
|
|
341
561
|
);
|
|
342
|
-
const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/
|
|
562
|
+
const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/session-image.${screenshot.extension}`;
|
|
343
563
|
let prepared: RetainedScreenshotArtifact;
|
|
344
564
|
try {
|
|
345
565
|
prepared = (
|
|
@@ -454,7 +674,14 @@ export function compactRetainedScreenshotHistory(
|
|
|
454
674
|
return history.map((item) => {
|
|
455
675
|
const callId = historyCallId(item);
|
|
456
676
|
const receipt = callId ? receiptsByCallId.get(callId) : undefined;
|
|
457
|
-
if (!receipt
|
|
677
|
+
if (!receipt) return item;
|
|
678
|
+
if (typeof item.output === "string" && item.output.startsWith("data:image/")) {
|
|
679
|
+
return {
|
|
680
|
+
...item,
|
|
681
|
+
output: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
if (!Array.isArray(item.output)) return item;
|
|
458
685
|
let changed = false;
|
|
459
686
|
const output = item.output.map((entry) => {
|
|
460
687
|
if (!isInlineImageContent(entry)) return entry;
|
|
@@ -475,7 +702,13 @@ export function collectRetainedScreenshotReceipts(
|
|
|
475
702
|
): Map<string, RetainedArtifactMetadata> {
|
|
476
703
|
for (const item of history) {
|
|
477
704
|
const callId = historyCallId(item);
|
|
478
|
-
if (!callId
|
|
705
|
+
if (!callId) continue;
|
|
706
|
+
const direct = retainedReceiptFromDirectOutput(item.output);
|
|
707
|
+
if (direct) {
|
|
708
|
+
target.set(callId, direct);
|
|
709
|
+
continue;
|
|
710
|
+
}
|
|
711
|
+
if (!Array.isArray(item.output)) continue;
|
|
479
712
|
for (const entry of item.output) {
|
|
480
713
|
const receipt = retainedReceiptFromImageContent(entry);
|
|
481
714
|
if (receipt) {
|
|
@@ -576,9 +809,7 @@ async function materializeRetainedScreenshotHistoryWithCache(
|
|
|
576
809
|
cache: Map<string, string>,
|
|
577
810
|
): Promise<Array<Record<string, unknown>>> {
|
|
578
811
|
const now = input.now ?? new Date();
|
|
579
|
-
const
|
|
580
|
-
const receipt = retainedReceiptFromImageContent(entry);
|
|
581
|
-
if (!receipt) return entry;
|
|
812
|
+
const dataUrlForReceipt = async (receipt: RetainedArtifactMetadata): Promise<string> => {
|
|
582
813
|
if (!receipt.available) {
|
|
583
814
|
throw new RetainedScreenshotUnavailableError(receipt.artifactId, receipt.reason);
|
|
584
815
|
}
|
|
@@ -606,9 +837,9 @@ async function materializeRetainedScreenshotHistoryWithCache(
|
|
|
606
837
|
const bytes = await input.objectStorage.getFileBytes(artifact.file).catch(() => null);
|
|
607
838
|
if (!bytes)
|
|
608
839
|
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
|
|
609
|
-
const validated =
|
|
840
|
+
const validated = validateRetainableSessionImage({
|
|
610
841
|
bytes,
|
|
611
|
-
|
|
842
|
+
declaredMediaType: artifact.mediaType,
|
|
612
843
|
});
|
|
613
844
|
if (
|
|
614
845
|
validated.sizeBytes !== artifact.sizeBytes ||
|
|
@@ -623,11 +854,25 @@ async function materializeRetainedScreenshotHistoryWithCache(
|
|
|
623
854
|
dataUrl = `data:${validated.mediaType};base64,${Buffer.from(validated.bytes).toString("base64")}`;
|
|
624
855
|
cache.set(receipt.artifactId, dataUrl);
|
|
625
856
|
}
|
|
857
|
+
return dataUrl;
|
|
858
|
+
};
|
|
859
|
+
const materializeEntry = async (entry: unknown): Promise<unknown> => {
|
|
860
|
+
const receipt = retainedReceiptFromImageContent(entry);
|
|
861
|
+
if (!receipt) return entry;
|
|
862
|
+
const dataUrl = await dataUrlForReceipt(receipt);
|
|
626
863
|
return { ...(entry as Record<string, unknown>), image: dataUrl };
|
|
627
864
|
};
|
|
628
865
|
|
|
629
866
|
const materialized: Array<Record<string, unknown>> = [];
|
|
630
867
|
for (const item of input.history) {
|
|
868
|
+
const directReceipt = retainedReceiptFromDirectOutput(item.output);
|
|
869
|
+
if (directReceipt) {
|
|
870
|
+
materialized.push({
|
|
871
|
+
...item,
|
|
872
|
+
output: await dataUrlForReceipt(directReceipt),
|
|
873
|
+
});
|
|
874
|
+
continue;
|
|
875
|
+
}
|
|
631
876
|
if (!Array.isArray(item.output)) {
|
|
632
877
|
materialized.push(item);
|
|
633
878
|
continue;
|
|
@@ -782,7 +1027,7 @@ async function verifyReadyArtifact(
|
|
|
782
1027
|
try {
|
|
783
1028
|
assertStoredScreenshotHead(await storage.headFile(artifact.file), {
|
|
784
1029
|
sizeBytes: artifact.sizeBytes,
|
|
785
|
-
mediaType:
|
|
1030
|
+
mediaType: artifact.mediaType as RetainableSessionImageMediaType,
|
|
786
1031
|
sha256: artifact.sha256,
|
|
787
1032
|
});
|
|
788
1033
|
return true;
|
|
@@ -793,7 +1038,7 @@ async function verifyReadyArtifact(
|
|
|
793
1038
|
|
|
794
1039
|
function assertStoredScreenshotHead(
|
|
795
1040
|
head: ObjectHead,
|
|
796
|
-
expected: Pick<
|
|
1041
|
+
expected: Pick<ValidatedSessionImage, "sizeBytes" | "mediaType" | "sha256">,
|
|
797
1042
|
): void {
|
|
798
1043
|
if (head.ContentLength !== expected.sizeBytes) {
|
|
799
1044
|
throw new Error("retained screenshot object size mismatch");
|
|
@@ -812,7 +1057,18 @@ function retainedReceiptFromImageContent(entry: unknown): RetainedArtifactMetada
|
|
|
812
1057
|
if (!image || typeof image !== "object" || Array.isArray(image)) return null;
|
|
813
1058
|
const marker = image as Record<string, unknown>;
|
|
814
1059
|
if (marker.type !== RETAINED_IMAGE_MARKER) return null;
|
|
815
|
-
|
|
1060
|
+
return retainedReceipt(marker.artifact);
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
function retainedReceiptFromDirectOutput(output: unknown): RetainedArtifactMetadata | null {
|
|
1064
|
+
if (!output || typeof output !== "object" || Array.isArray(output)) return null;
|
|
1065
|
+
const marker = output as Record<string, unknown>;
|
|
1066
|
+
if (marker.type !== RETAINED_IMAGE_MARKER) return null;
|
|
1067
|
+
return retainedReceipt(marker.artifact);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
function retainedReceipt(value: unknown): RetainedArtifactMetadata | null {
|
|
1071
|
+
const parsed = RetainedArtifactMetadataSchema.safeParse(value);
|
|
816
1072
|
if (!parsed.success) return null;
|
|
817
1073
|
return parsed.data.available && parsed.data.kind !== "computer_screenshot" ? null : parsed.data;
|
|
818
1074
|
}
|
|
@@ -823,7 +1079,7 @@ function isInlineImageContent(entry: unknown): boolean {
|
|
|
823
1079
|
return (
|
|
824
1080
|
record.type === "input_image" &&
|
|
825
1081
|
typeof record.image === "string" &&
|
|
826
|
-
record.image.startsWith("data:image/
|
|
1082
|
+
record.image.startsWith("data:image/")
|
|
827
1083
|
);
|
|
828
1084
|
}
|
|
829
1085
|
|
|
@@ -832,6 +1088,175 @@ function historyCallId(item: Record<string, unknown>): string | null {
|
|
|
832
1088
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
833
1089
|
}
|
|
834
1090
|
|
|
1091
|
+
function canonicalMediaType(value: string): string {
|
|
1092
|
+
return value.split(";", 1)[0]!.trim().toLowerCase();
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
function isPng(bytes: Uint8Array): boolean {
|
|
1096
|
+
return (
|
|
1097
|
+
bytes.byteLength >= PNG_SIGNATURE.byteLength &&
|
|
1098
|
+
PNG_SIGNATURE.every((byte, index) => bytes[index] === byte)
|
|
1099
|
+
);
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
function isJpeg(bytes: Uint8Array): boolean {
|
|
1103
|
+
return bytes.byteLength >= 4 && bytes[0] === 0xff && bytes[1] === 0xd8;
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
function jpegDimensions(bytes: Uint8Array): { width: number; height: number } {
|
|
1107
|
+
if (bytes[bytes.length - 2] !== 0xff || bytes[bytes.length - 1] !== 0xd9) {
|
|
1108
|
+
throw new ScreenshotValidationError(
|
|
1109
|
+
"invalid_content",
|
|
1110
|
+
"session JPEG is truncated or has trailing bytes",
|
|
1111
|
+
);
|
|
1112
|
+
}
|
|
1113
|
+
let offset = 2;
|
|
1114
|
+
while (offset + 4 <= bytes.length) {
|
|
1115
|
+
if (bytes[offset] !== 0xff) {
|
|
1116
|
+
throw new ScreenshotValidationError(
|
|
1117
|
+
"invalid_content",
|
|
1118
|
+
"session JPEG marker table is invalid",
|
|
1119
|
+
);
|
|
1120
|
+
}
|
|
1121
|
+
while (bytes[offset] === 0xff) offset += 1;
|
|
1122
|
+
const marker = bytes[offset++];
|
|
1123
|
+
if (marker === undefined || marker === 0xd9 || marker === 0xda) break;
|
|
1124
|
+
if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue;
|
|
1125
|
+
if (offset + 2 > bytes.length) break;
|
|
1126
|
+
const length = readUint16Be(bytes, offset);
|
|
1127
|
+
if (length < 2 || offset + length > bytes.length) {
|
|
1128
|
+
throw new ScreenshotValidationError("invalid_content", "session JPEG segment is invalid");
|
|
1129
|
+
}
|
|
1130
|
+
if (JPEG_SOF_MARKERS.has(marker)) {
|
|
1131
|
+
if (length < 7) {
|
|
1132
|
+
throw new ScreenshotValidationError("invalid_content", "session JPEG frame is invalid");
|
|
1133
|
+
}
|
|
1134
|
+
return {
|
|
1135
|
+
height: readUint16Be(bytes, offset + 3),
|
|
1136
|
+
width: readUint16Be(bytes, offset + 5),
|
|
1137
|
+
};
|
|
1138
|
+
}
|
|
1139
|
+
offset += length;
|
|
1140
|
+
}
|
|
1141
|
+
throw new ScreenshotValidationError(
|
|
1142
|
+
"invalid_content",
|
|
1143
|
+
"session JPEG has no supported frame header",
|
|
1144
|
+
);
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
const JPEG_SOF_MARKERS = new Set([
|
|
1148
|
+
0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf,
|
|
1149
|
+
]);
|
|
1150
|
+
|
|
1151
|
+
function isWebp(bytes: Uint8Array): boolean {
|
|
1152
|
+
return bytes.byteLength >= 20 && ascii(bytes, 0, 4) === "RIFF" && ascii(bytes, 8, 4) === "WEBP";
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
function webpDimensions(bytes: Uint8Array): { width: number; height: number } {
|
|
1156
|
+
if (readUint32Le(bytes, 4) + 8 !== bytes.byteLength) {
|
|
1157
|
+
throw new ScreenshotValidationError("invalid_content", "session WebP RIFF length is invalid");
|
|
1158
|
+
}
|
|
1159
|
+
let offset = 12;
|
|
1160
|
+
while (offset + 8 <= bytes.length) {
|
|
1161
|
+
const type = ascii(bytes, offset, 4);
|
|
1162
|
+
const length = readUint32Le(bytes, offset + 4);
|
|
1163
|
+
const data = offset + 8;
|
|
1164
|
+
const next = data + length + (length & 1);
|
|
1165
|
+
if (next > bytes.length) {
|
|
1166
|
+
throw new ScreenshotValidationError("invalid_content", "session WebP chunk is truncated");
|
|
1167
|
+
}
|
|
1168
|
+
if (type === "VP8X") {
|
|
1169
|
+
if (length < 10) {
|
|
1170
|
+
throw new ScreenshotValidationError("invalid_content", "session WebP VP8X is invalid");
|
|
1171
|
+
}
|
|
1172
|
+
return {
|
|
1173
|
+
width: 1 + readUint24Le(bytes, data + 4),
|
|
1174
|
+
height: 1 + readUint24Le(bytes, data + 7),
|
|
1175
|
+
};
|
|
1176
|
+
}
|
|
1177
|
+
if (type === "VP8 ") {
|
|
1178
|
+
if (
|
|
1179
|
+
length < 10 ||
|
|
1180
|
+
bytes[data + 3] !== 0x9d ||
|
|
1181
|
+
bytes[data + 4] !== 0x01 ||
|
|
1182
|
+
bytes[data + 5] !== 0x2a
|
|
1183
|
+
) {
|
|
1184
|
+
throw new ScreenshotValidationError("invalid_content", "session WebP VP8 frame is invalid");
|
|
1185
|
+
}
|
|
1186
|
+
return {
|
|
1187
|
+
width: readUint16Le(bytes, data + 6) & 0x3fff,
|
|
1188
|
+
height: readUint16Le(bytes, data + 8) & 0x3fff,
|
|
1189
|
+
};
|
|
1190
|
+
}
|
|
1191
|
+
if (type === "VP8L") {
|
|
1192
|
+
if (length < 5 || bytes[data] !== 0x2f) {
|
|
1193
|
+
throw new ScreenshotValidationError(
|
|
1194
|
+
"invalid_content",
|
|
1195
|
+
"session WebP VP8L frame is invalid",
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1198
|
+
const bits = readUint32Le(bytes, data + 1);
|
|
1199
|
+
return {
|
|
1200
|
+
width: 1 + (bits & 0x3fff),
|
|
1201
|
+
height: 1 + ((bits >>> 14) & 0x3fff),
|
|
1202
|
+
};
|
|
1203
|
+
}
|
|
1204
|
+
offset = next;
|
|
1205
|
+
}
|
|
1206
|
+
throw new ScreenshotValidationError("invalid_content", "session WebP has no image frame");
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
function assertImageDimensions(
|
|
1210
|
+
width: number,
|
|
1211
|
+
height: number,
|
|
1212
|
+
limits: { maxDimension: number; maxPixels: number },
|
|
1213
|
+
): void {
|
|
1214
|
+
if (
|
|
1215
|
+
!Number.isSafeInteger(width) ||
|
|
1216
|
+
!Number.isSafeInteger(height) ||
|
|
1217
|
+
width <= 0 ||
|
|
1218
|
+
height <= 0 ||
|
|
1219
|
+
width > limits.maxDimension ||
|
|
1220
|
+
height > limits.maxDimension ||
|
|
1221
|
+
width * height > limits.maxPixels
|
|
1222
|
+
) {
|
|
1223
|
+
throw new ScreenshotValidationError("oversized", "session image dimensions exceed policy");
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
function readUint16Be(bytes: Uint8Array, offset: number): number {
|
|
1228
|
+
return (bytes[offset]! << 8) | bytes[offset + 1]!;
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
function readUint16Le(bytes: Uint8Array, offset: number): number {
|
|
1232
|
+
return bytes[offset]! | (bytes[offset + 1]! << 8);
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
function readUint24Le(bytes: Uint8Array, offset: number): number {
|
|
1236
|
+
return bytes[offset]! | (bytes[offset + 1]! << 8) | (bytes[offset + 2]! << 16);
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
function readUint32Le(bytes: Uint8Array, offset: number): number {
|
|
1240
|
+
return (
|
|
1241
|
+
(bytes[offset]! |
|
|
1242
|
+
(bytes[offset + 1]! << 8) |
|
|
1243
|
+
(bytes[offset + 2]! << 16) |
|
|
1244
|
+
(bytes[offset + 3]! << 24)) >>>
|
|
1245
|
+
0
|
|
1246
|
+
);
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
function ascii(bytes: Uint8Array, offset: number, length: number): string {
|
|
1250
|
+
return String.fromCharCode(...bytes.subarray(offset, offset + length));
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
function base64Sextet(code: number): number {
|
|
1254
|
+
if (code >= 0x41 && code <= 0x5a) return code - 0x41;
|
|
1255
|
+
if (code >= 0x61 && code <= 0x7a) return code - 0x61 + 26;
|
|
1256
|
+
if (code >= 0x30 && code <= 0x39) return code - 0x30 + 52;
|
|
1257
|
+
return code === 0x2b ? 62 : 63;
|
|
1258
|
+
}
|
|
1259
|
+
|
|
835
1260
|
function readUint32(bytes: Uint8Array, offset: number): number {
|
|
836
1261
|
return (
|
|
837
1262
|
((bytes[offset]! << 24) |
|