@opengeni/worker-bundle 0.16.21 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +145 -32
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +3 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -7
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +2 -2
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +129 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +2 -2
- package/dist/activities/run-input.d.ts +5 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +11 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -147
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +675 -16460
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +40 -7
- package/dist/sandbox-routing.d.ts +13 -6
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1044 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1905 -559
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +16 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +47 -6
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +52 -60
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +1317 -0
- package/src/activities/rig-verification.ts +567 -77
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +40 -10
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +96 -7
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +77 -13
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +50 -118
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +202 -7
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +331 -157
- package/src/sandbox-routing.ts +49 -39
- package/src/worker-service-lifecycle.ts +82 -12
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +6 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
- package/src/activities/toolspace-token-renewal.ts +0 -57
|
@@ -0,0 +1,1317 @@
|
|
|
1
|
+
import {
|
|
2
|
+
COMPUTER_SCREENSHOT_MAX_BYTES,
|
|
3
|
+
COMPUTER_SCREENSHOT_MAX_DIMENSION,
|
|
4
|
+
COMPUTER_SCREENSHOT_MAX_PIXELS,
|
|
5
|
+
COMPUTER_SCREENSHOT_RETENTION_MS,
|
|
6
|
+
COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES,
|
|
7
|
+
RetainedArtifactMetadataSchema,
|
|
8
|
+
retainedScreenshotReferenceFromFile,
|
|
9
|
+
type RetainedArtifactMetadata,
|
|
10
|
+
type RetainedArtifactReference,
|
|
11
|
+
type RetainedOutputUnavailableReason,
|
|
12
|
+
} from "@opengeni/contracts";
|
|
13
|
+
import {
|
|
14
|
+
RetainedScreenshotQuotaExceededError,
|
|
15
|
+
getRetainedScreenshotArtifact,
|
|
16
|
+
prepareRetainedScreenshotArtifact,
|
|
17
|
+
recordRetainedScreenshotArtifactError,
|
|
18
|
+
settleRetainedScreenshotArtifactReady,
|
|
19
|
+
type Database,
|
|
20
|
+
type RetainedScreenshotArtifact,
|
|
21
|
+
} from "@opengeni/db";
|
|
22
|
+
import type { ObjectHead, ObjectStorage } from "@opengeni/storage";
|
|
23
|
+
import { createHash } from "node:crypto";
|
|
24
|
+
|
|
25
|
+
const PNG_SIGNATURE = Uint8Array.of(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a);
|
|
26
|
+
const PNG_ALLOWED_CRITICAL_CHUNKS = new Set(["IHDR", "PLTE", "IDAT", "IEND"]);
|
|
27
|
+
const PNG_MAX_CHUNKS = 16_384;
|
|
28
|
+
const RETAINED_IMAGE_MARKER = "retained_artifact";
|
|
29
|
+
|
|
30
|
+
export type TypedScreenshotToolOutput = {
|
|
31
|
+
callId: string;
|
|
32
|
+
toolOutputId: string;
|
|
33
|
+
bytes: Uint8Array;
|
|
34
|
+
mediaType: string;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
export type ValidatedScreenshot = {
|
|
38
|
+
bytes: Uint8Array;
|
|
39
|
+
mediaType: "image/png";
|
|
40
|
+
sizeBytes: number;
|
|
41
|
+
sha256: string;
|
|
42
|
+
width: number;
|
|
43
|
+
height: number;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
export type RetainableSessionImageMediaType = "image/png" | "image/jpeg" | "image/webp";
|
|
47
|
+
|
|
48
|
+
export type ValidatedSessionImage = {
|
|
49
|
+
bytes: Uint8Array;
|
|
50
|
+
mediaType: RetainableSessionImageMediaType;
|
|
51
|
+
extension: "png" | "jpg" | "webp";
|
|
52
|
+
sizeBytes: number;
|
|
53
|
+
sha256: string;
|
|
54
|
+
width: number;
|
|
55
|
+
height: number;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export class ScreenshotValidationError extends Error {
|
|
59
|
+
constructor(
|
|
60
|
+
readonly reason: "invalid_content" | "oversized" | "unsupported",
|
|
61
|
+
message: string,
|
|
62
|
+
) {
|
|
63
|
+
super(message);
|
|
64
|
+
this.name = "ScreenshotValidationError";
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export class RetainedScreenshotUnavailableError extends Error {
|
|
69
|
+
constructor(
|
|
70
|
+
readonly artifactId: string,
|
|
71
|
+
readonly reason: RetainedOutputUnavailableReason,
|
|
72
|
+
) {
|
|
73
|
+
super(`Retained screenshot ${artifactId} is unavailable: ${reason}`);
|
|
74
|
+
this.name = "RetainedScreenshotUnavailableError";
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Read an intentional image directly at the tool execution boundary. */
|
|
79
|
+
export function typedScreenshotFromToolOutput(input: {
|
|
80
|
+
callId: string;
|
|
81
|
+
output: unknown;
|
|
82
|
+
}): TypedScreenshotToolOutput | null {
|
|
83
|
+
const image = imageBytesFromSdkToolOutput(input.output);
|
|
84
|
+
if (!image) return null;
|
|
85
|
+
return {
|
|
86
|
+
callId: input.callId,
|
|
87
|
+
toolOutputId: input.callId,
|
|
88
|
+
bytes: image.bytes,
|
|
89
|
+
mediaType: image.mediaType,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Read the SDK's trusted typed image before event/history serialization. */
|
|
94
|
+
export function typedScreenshotFromSdkEvent(event: unknown): TypedScreenshotToolOutput | null {
|
|
95
|
+
if (!event || typeof event !== "object") return null;
|
|
96
|
+
const streamEvent = event as {
|
|
97
|
+
type?: unknown;
|
|
98
|
+
item?: {
|
|
99
|
+
id?: unknown;
|
|
100
|
+
type?: unknown;
|
|
101
|
+
rawItem?: unknown;
|
|
102
|
+
output?: unknown;
|
|
103
|
+
};
|
|
104
|
+
};
|
|
105
|
+
if (streamEvent.type !== "run_item_stream_event") return null;
|
|
106
|
+
const item = streamEvent.item;
|
|
107
|
+
if (!item || item.type !== "tool_call_output_item") return null;
|
|
108
|
+
const image = imageBytesFromSdkToolOutput(item.output);
|
|
109
|
+
if (!image) return null;
|
|
110
|
+
const raw =
|
|
111
|
+
item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem)
|
|
112
|
+
? (item.rawItem as Record<string, unknown>)
|
|
113
|
+
: {};
|
|
114
|
+
// Live Agents SDK results can carry the call identity only on the run-item
|
|
115
|
+
// wrapper; reconciled history later copies it into `rawItem`. Use the same
|
|
116
|
+
// fallback as the canonical completed-call extractor so both paths agree.
|
|
117
|
+
const callId = raw.callId ?? raw.call_id ?? item.id;
|
|
118
|
+
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
119
|
+
const toolOutputId = raw.id ?? item.id ?? callId;
|
|
120
|
+
if (typeof toolOutputId !== "string" || toolOutputId.length === 0) return null;
|
|
121
|
+
return {
|
|
122
|
+
callId,
|
|
123
|
+
toolOutputId,
|
|
124
|
+
bytes: image.bytes,
|
|
125
|
+
mediaType: image.mediaType,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Does this SDK result still contain inline bytes that persistence must replace? */
|
|
130
|
+
export function sdkEventContainsInlineImage(event: unknown): boolean {
|
|
131
|
+
if (!event || typeof event !== "object") return false;
|
|
132
|
+
const streamEvent = event as {
|
|
133
|
+
type?: unknown;
|
|
134
|
+
item?: { type?: unknown; output?: unknown };
|
|
135
|
+
};
|
|
136
|
+
return (
|
|
137
|
+
streamEvent.type === "run_item_stream_event" &&
|
|
138
|
+
streamEvent.item?.type === "tool_call_output_item" &&
|
|
139
|
+
toolOutputContainsInlineImage(streamEvent.item.output)
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function imageBytesFromSdkToolOutput(
|
|
144
|
+
output: unknown,
|
|
145
|
+
): { bytes: Uint8Array; mediaType: string } | null {
|
|
146
|
+
if (typeof output === "string") return decodeInlineImageDataUrl(output);
|
|
147
|
+
if (Array.isArray(output)) {
|
|
148
|
+
for (const entry of output) {
|
|
149
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) continue;
|
|
150
|
+
const record = entry as Record<string, unknown>;
|
|
151
|
+
if (record.type !== "input_image") continue;
|
|
152
|
+
const decoded = imageBytesFromSdkImageSource(
|
|
153
|
+
record.image ?? record.image_url ?? record.imageUrl,
|
|
154
|
+
);
|
|
155
|
+
if (decoded) return decoded;
|
|
156
|
+
}
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
if (!output || typeof output !== "object") return null;
|
|
160
|
+
const outputRecord = output as Record<string, unknown>;
|
|
161
|
+
if (
|
|
162
|
+
outputRecord.type !== "image" ||
|
|
163
|
+
!outputRecord.image ||
|
|
164
|
+
typeof outputRecord.image !== "object" ||
|
|
165
|
+
Array.isArray(outputRecord.image)
|
|
166
|
+
) {
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
return imageBytesFromSdkImageSource(outputRecord.image);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function imageBytesFromSdkImageSource(
|
|
173
|
+
source: unknown,
|
|
174
|
+
): { bytes: Uint8Array; mediaType: string } | null {
|
|
175
|
+
const direct = decodeInlineImageDataUrl(source);
|
|
176
|
+
if (direct) return direct;
|
|
177
|
+
if (!source || typeof source !== "object" || Array.isArray(source)) return null;
|
|
178
|
+
const image = source as Record<string, unknown>;
|
|
179
|
+
const fromUrl = decodeInlineImageDataUrl(image.url);
|
|
180
|
+
if (fromUrl) return fromUrl;
|
|
181
|
+
return image.data instanceof Uint8Array && typeof image.mediaType === "string"
|
|
182
|
+
? { bytes: image.data, mediaType: image.mediaType }
|
|
183
|
+
: null;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function sdkImageSourceContainsInlineImage(source: unknown): boolean {
|
|
187
|
+
if (typeof source === "string") return source.startsWith("data:image/");
|
|
188
|
+
if (!source || typeof source !== "object" || Array.isArray(source)) return false;
|
|
189
|
+
const image = source as Record<string, unknown>;
|
|
190
|
+
return (
|
|
191
|
+
(typeof image.url === "string" && image.url.startsWith("data:image/")) ||
|
|
192
|
+
(image.data instanceof Uint8Array && typeof image.mediaType === "string")
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export function toolOutputContainsInlineImage(output: unknown): boolean {
|
|
197
|
+
if (typeof output === "string") return output.startsWith("data:image/");
|
|
198
|
+
if (Array.isArray(output)) {
|
|
199
|
+
return output.some((entry) => {
|
|
200
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
|
|
201
|
+
const record = entry as Record<string, unknown>;
|
|
202
|
+
return (
|
|
203
|
+
record.type === "input_image" &&
|
|
204
|
+
sdkImageSourceContainsInlineImage(record.image ?? record.image_url ?? record.imageUrl)
|
|
205
|
+
);
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
if (!output || typeof output !== "object") return false;
|
|
209
|
+
const record = output as Record<string, unknown>;
|
|
210
|
+
return record.type === "image" && sdkImageSourceContainsInlineImage(record.image);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function decodeInlineImageDataUrl(value: unknown): { bytes: Uint8Array; mediaType: string } | null {
|
|
214
|
+
if (typeof value !== "string") return null;
|
|
215
|
+
const match = /^data:(image\/(?:png|jpeg|webp));base64,/u.exec(value);
|
|
216
|
+
if (!match?.[1]) return null;
|
|
217
|
+
const mediaType = match[1] as RetainableSessionImageMediaType;
|
|
218
|
+
const encoded = value.slice(match[0].length);
|
|
219
|
+
const maxEncodedLength = Math.ceil(COMPUTER_SCREENSHOT_MAX_BYTES / 3) * 4;
|
|
220
|
+
if (
|
|
221
|
+
encoded.length === 0 ||
|
|
222
|
+
encoded.length > maxEncodedLength ||
|
|
223
|
+
encoded.length % 4 !== 0 ||
|
|
224
|
+
!/^[A-Za-z0-9+/]+={0,2}$/.test(encoded)
|
|
225
|
+
) {
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
const padding = encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0;
|
|
229
|
+
const decodedLength = (encoded.length / 4) * 3 - padding;
|
|
230
|
+
if (decodedLength <= 0 || decodedLength > COMPUTER_SCREENSHOT_MAX_BYTES) {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
const lastSextet = base64Sextet(encoded.charCodeAt(encoded.length - padding - 1));
|
|
234
|
+
if (
|
|
235
|
+
(padding === 2 && (lastSextet & 0x0f) !== 0) ||
|
|
236
|
+
(padding === 1 && (lastSextet & 0x03) !== 0)
|
|
237
|
+
) {
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
const decoded = Buffer.from(encoded, "base64");
|
|
241
|
+
if (decoded.byteLength !== decodedLength) return null;
|
|
242
|
+
const bytes = new Uint8Array(decoded.buffer, decoded.byteOffset, decoded.byteLength);
|
|
243
|
+
return { bytes, mediaType };
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Validate the complete canonical PNG and derive exact integrity metadata. */
|
|
247
|
+
export function validateComputerScreenshot(
|
|
248
|
+
input: { bytes: Uint8Array; mediaType: string },
|
|
249
|
+
limits: {
|
|
250
|
+
maxBytes?: number;
|
|
251
|
+
maxDimension?: number;
|
|
252
|
+
maxPixels?: number;
|
|
253
|
+
} = {},
|
|
254
|
+
): ValidatedScreenshot {
|
|
255
|
+
const maxBytes = limits.maxBytes ?? COMPUTER_SCREENSHOT_MAX_BYTES;
|
|
256
|
+
const maxDimension = limits.maxDimension ?? COMPUTER_SCREENSHOT_MAX_DIMENSION;
|
|
257
|
+
const maxPixels = limits.maxPixels ?? COMPUTER_SCREENSHOT_MAX_PIXELS;
|
|
258
|
+
if (input.mediaType !== "image/png") {
|
|
259
|
+
throw new ScreenshotValidationError(
|
|
260
|
+
"unsupported",
|
|
261
|
+
"computer screenshot MIME must be image/png",
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
if (input.bytes.byteLength === 0) {
|
|
265
|
+
throw new ScreenshotValidationError("invalid_content", "computer screenshot is empty");
|
|
266
|
+
}
|
|
267
|
+
if (input.bytes.byteLength > maxBytes) {
|
|
268
|
+
throw new ScreenshotValidationError(
|
|
269
|
+
"oversized",
|
|
270
|
+
`computer screenshot exceeds ${maxBytes} bytes`,
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
if (input.bytes.byteLength < PNG_SIGNATURE.byteLength + 25) {
|
|
274
|
+
throw new ScreenshotValidationError("invalid_content", "computer screenshot PNG is truncated");
|
|
275
|
+
}
|
|
276
|
+
for (let index = 0; index < PNG_SIGNATURE.byteLength; index += 1) {
|
|
277
|
+
if (input.bytes[index] !== PNG_SIGNATURE[index]) {
|
|
278
|
+
throw new ScreenshotValidationError(
|
|
279
|
+
"invalid_content",
|
|
280
|
+
"computer screenshot PNG signature is invalid",
|
|
281
|
+
);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
let offset = PNG_SIGNATURE.byteLength;
|
|
286
|
+
let chunks = 0;
|
|
287
|
+
let width = 0;
|
|
288
|
+
let height = 0;
|
|
289
|
+
let sawIhdr = false;
|
|
290
|
+
let sawIdat = false;
|
|
291
|
+
let endedIdat = false;
|
|
292
|
+
let sawIend = false;
|
|
293
|
+
while (offset < input.bytes.byteLength) {
|
|
294
|
+
chunks += 1;
|
|
295
|
+
if (chunks > PNG_MAX_CHUNKS || offset + 12 > input.bytes.byteLength) {
|
|
296
|
+
throw new ScreenshotValidationError(
|
|
297
|
+
"invalid_content",
|
|
298
|
+
"computer screenshot PNG chunk table is invalid",
|
|
299
|
+
);
|
|
300
|
+
}
|
|
301
|
+
const length = readUint32(input.bytes, offset);
|
|
302
|
+
const typeOffset = offset + 4;
|
|
303
|
+
const dataOffset = typeOffset + 4;
|
|
304
|
+
const dataEnd = dataOffset + length;
|
|
305
|
+
const crcOffset = dataEnd;
|
|
306
|
+
const nextOffset = crcOffset + 4;
|
|
307
|
+
if (!Number.isSafeInteger(nextOffset) || nextOffset > input.bytes.byteLength) {
|
|
308
|
+
throw new ScreenshotValidationError(
|
|
309
|
+
"invalid_content",
|
|
310
|
+
"computer screenshot PNG chunk is truncated",
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
const type = asciiChunkType(input.bytes, typeOffset);
|
|
314
|
+
const expectedCrc = readUint32(input.bytes, crcOffset);
|
|
315
|
+
const actualCrc = crc32(input.bytes.subarray(typeOffset, dataEnd));
|
|
316
|
+
if (expectedCrc !== actualCrc) {
|
|
317
|
+
throw new ScreenshotValidationError(
|
|
318
|
+
"invalid_content",
|
|
319
|
+
`computer screenshot PNG ${type} CRC is invalid`,
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
if (chunks === 1 && type !== "IHDR") {
|
|
323
|
+
throw new ScreenshotValidationError(
|
|
324
|
+
"invalid_content",
|
|
325
|
+
"computer screenshot PNG must begin with IHDR",
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
if (isCriticalChunk(type) && !PNG_ALLOWED_CRITICAL_CHUNKS.has(type)) {
|
|
329
|
+
throw new ScreenshotValidationError(
|
|
330
|
+
"invalid_content",
|
|
331
|
+
`computer screenshot PNG has unsupported critical chunk ${type}`,
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
if (type === "IHDR") {
|
|
336
|
+
if (sawIhdr || length !== 13) {
|
|
337
|
+
throw new ScreenshotValidationError(
|
|
338
|
+
"invalid_content",
|
|
339
|
+
"computer screenshot PNG IHDR is invalid",
|
|
340
|
+
);
|
|
341
|
+
}
|
|
342
|
+
sawIhdr = true;
|
|
343
|
+
width = readUint32(input.bytes, dataOffset);
|
|
344
|
+
height = readUint32(input.bytes, dataOffset + 4);
|
|
345
|
+
const bitDepth = input.bytes[dataOffset + 8];
|
|
346
|
+
const colorType = input.bytes[dataOffset + 9];
|
|
347
|
+
const compression = input.bytes[dataOffset + 10];
|
|
348
|
+
const filter = input.bytes[dataOffset + 11];
|
|
349
|
+
const interlace = input.bytes[dataOffset + 12];
|
|
350
|
+
if (
|
|
351
|
+
!validPngColorMode(bitDepth, colorType) ||
|
|
352
|
+
compression !== 0 ||
|
|
353
|
+
filter !== 0 ||
|
|
354
|
+
(interlace !== 0 && interlace !== 1)
|
|
355
|
+
) {
|
|
356
|
+
throw new ScreenshotValidationError(
|
|
357
|
+
"invalid_content",
|
|
358
|
+
"computer screenshot PNG IHDR fields are invalid",
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
if (
|
|
362
|
+
width <= 0 ||
|
|
363
|
+
height <= 0 ||
|
|
364
|
+
width > maxDimension ||
|
|
365
|
+
height > maxDimension ||
|
|
366
|
+
width * height > maxPixels
|
|
367
|
+
) {
|
|
368
|
+
throw new ScreenshotValidationError(
|
|
369
|
+
"oversized",
|
|
370
|
+
"computer screenshot dimensions exceed policy",
|
|
371
|
+
);
|
|
372
|
+
}
|
|
373
|
+
} else if (type === "IDAT") {
|
|
374
|
+
if (!sawIhdr || sawIend || endedIdat || length === 0) {
|
|
375
|
+
throw new ScreenshotValidationError(
|
|
376
|
+
"invalid_content",
|
|
377
|
+
"computer screenshot PNG IDAT ordering is invalid",
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
sawIdat = true;
|
|
381
|
+
} else if (sawIdat && type !== "IEND") {
|
|
382
|
+
endedIdat = true;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
if (type === "IEND") {
|
|
386
|
+
if (!sawIhdr || !sawIdat || sawIend || length !== 0) {
|
|
387
|
+
throw new ScreenshotValidationError(
|
|
388
|
+
"invalid_content",
|
|
389
|
+
"computer screenshot PNG IEND is invalid",
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
sawIend = true;
|
|
393
|
+
if (nextOffset !== input.bytes.byteLength) {
|
|
394
|
+
throw new ScreenshotValidationError(
|
|
395
|
+
"invalid_content",
|
|
396
|
+
"computer screenshot PNG has trailing/polyglot bytes",
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
offset = nextOffset;
|
|
401
|
+
}
|
|
402
|
+
if (!sawIhdr || !sawIdat || !sawIend || offset !== input.bytes.byteLength) {
|
|
403
|
+
throw new ScreenshotValidationError("invalid_content", "computer screenshot PNG is incomplete");
|
|
404
|
+
}
|
|
405
|
+
return {
|
|
406
|
+
bytes: input.bytes,
|
|
407
|
+
mediaType: "image/png",
|
|
408
|
+
sizeBytes: input.bytes.byteLength,
|
|
409
|
+
sha256: createHash("sha256").update(input.bytes).digest("hex"),
|
|
410
|
+
width,
|
|
411
|
+
height,
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/** Validate a durable view_image/computer image from bytes, never from its claimed MIME alone. */
|
|
416
|
+
export function validateRetainableSessionImage(
|
|
417
|
+
input: { bytes: Uint8Array; declaredMediaType?: string },
|
|
418
|
+
limits: {
|
|
419
|
+
maxBytes?: number;
|
|
420
|
+
maxDimension?: number;
|
|
421
|
+
maxPixels?: number;
|
|
422
|
+
} = {},
|
|
423
|
+
): ValidatedSessionImage {
|
|
424
|
+
const maxBytes = limits.maxBytes ?? COMPUTER_SCREENSHOT_MAX_BYTES;
|
|
425
|
+
const maxDimension = limits.maxDimension ?? COMPUTER_SCREENSHOT_MAX_DIMENSION;
|
|
426
|
+
const maxPixels = limits.maxPixels ?? COMPUTER_SCREENSHOT_MAX_PIXELS;
|
|
427
|
+
if (input.bytes.byteLength === 0) {
|
|
428
|
+
throw new ScreenshotValidationError("invalid_content", "session image is empty");
|
|
429
|
+
}
|
|
430
|
+
if (input.bytes.byteLength > maxBytes) {
|
|
431
|
+
throw new ScreenshotValidationError("oversized", `session image exceeds ${maxBytes} bytes`);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
let identity: Pick<ValidatedSessionImage, "mediaType" | "extension" | "width" | "height">;
|
|
435
|
+
if (isPng(input.bytes)) {
|
|
436
|
+
const png = validateComputerScreenshot(
|
|
437
|
+
{ bytes: input.bytes, mediaType: "image/png" },
|
|
438
|
+
{ maxBytes, maxDimension, maxPixels },
|
|
439
|
+
);
|
|
440
|
+
identity = {
|
|
441
|
+
mediaType: "image/png",
|
|
442
|
+
extension: "png",
|
|
443
|
+
width: png.width,
|
|
444
|
+
height: png.height,
|
|
445
|
+
};
|
|
446
|
+
} else if (isJpeg(input.bytes)) {
|
|
447
|
+
identity = {
|
|
448
|
+
mediaType: "image/jpeg",
|
|
449
|
+
extension: "jpg",
|
|
450
|
+
...jpegDimensions(input.bytes),
|
|
451
|
+
};
|
|
452
|
+
} else if (isWebp(input.bytes)) {
|
|
453
|
+
identity = {
|
|
454
|
+
mediaType: "image/webp",
|
|
455
|
+
extension: "webp",
|
|
456
|
+
...webpDimensions(input.bytes),
|
|
457
|
+
};
|
|
458
|
+
} else {
|
|
459
|
+
throw new ScreenshotValidationError("unsupported", "session image format is unsupported");
|
|
460
|
+
}
|
|
461
|
+
assertImageDimensions(identity.width, identity.height, {
|
|
462
|
+
maxDimension,
|
|
463
|
+
maxPixels,
|
|
464
|
+
});
|
|
465
|
+
if (
|
|
466
|
+
input.declaredMediaType &&
|
|
467
|
+
canonicalMediaType(input.declaredMediaType) !== identity.mediaType
|
|
468
|
+
) {
|
|
469
|
+
throw new ScreenshotValidationError(
|
|
470
|
+
"invalid_content",
|
|
471
|
+
"session image MIME does not match its bytes",
|
|
472
|
+
);
|
|
473
|
+
}
|
|
474
|
+
return {
|
|
475
|
+
bytes: input.bytes,
|
|
476
|
+
...identity,
|
|
477
|
+
sizeBytes: input.bytes.byteLength,
|
|
478
|
+
sha256: createHash("sha256").update(input.bytes).digest("hex"),
|
|
479
|
+
};
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
export function retainedScreenshotIdentity(input: {
|
|
483
|
+
sessionId: string;
|
|
484
|
+
turnId: string;
|
|
485
|
+
attemptId: string;
|
|
486
|
+
toolCallId: string;
|
|
487
|
+
toolOutputId: string;
|
|
488
|
+
}): { artifactId: string; settlementKey: string } {
|
|
489
|
+
const settlementKey = createHash("sha256")
|
|
490
|
+
.update("opengeni:computer-screenshot:v1\0")
|
|
491
|
+
.update(input.sessionId)
|
|
492
|
+
.update("\0")
|
|
493
|
+
.update(input.turnId)
|
|
494
|
+
.update("\0")
|
|
495
|
+
.update(input.attemptId)
|
|
496
|
+
.update("\0")
|
|
497
|
+
.update(input.toolCallId)
|
|
498
|
+
.update("\0")
|
|
499
|
+
.update(input.toolOutputId)
|
|
500
|
+
.digest("hex");
|
|
501
|
+
const uuidBytes = Uint8Array.from(Buffer.from(settlementKey.slice(0, 32), "hex"));
|
|
502
|
+
uuidBytes[6] = (uuidBytes[6]! & 0x0f) | 0x80;
|
|
503
|
+
uuidBytes[8] = (uuidBytes[8]! & 0x3f) | 0x80;
|
|
504
|
+
const hex = Buffer.from(uuidBytes).toString("hex");
|
|
505
|
+
return {
|
|
506
|
+
artifactId: `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`,
|
|
507
|
+
settlementKey,
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/** Deterministic fail-closed receipt installed before retention can perform I/O. */
|
|
512
|
+
export function unavailableRetainedSessionImage(input: {
|
|
513
|
+
sessionId: string;
|
|
514
|
+
turnId: string;
|
|
515
|
+
attemptId: string;
|
|
516
|
+
toolCallId: string;
|
|
517
|
+
toolOutputId: string;
|
|
518
|
+
reason: RetainedOutputUnavailableReason;
|
|
519
|
+
}): RetainedArtifactMetadata {
|
|
520
|
+
const identity = retainedScreenshotIdentity(input);
|
|
521
|
+
return unavailable(identity.artifactId, input.reason);
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
export async function retainComputerScreenshot(input: {
|
|
525
|
+
db: Database;
|
|
526
|
+
objectStorage: ObjectStorage | null;
|
|
527
|
+
accountId: string;
|
|
528
|
+
workspaceId: string;
|
|
529
|
+
sessionId: string;
|
|
530
|
+
turnId: string;
|
|
531
|
+
attemptId: string;
|
|
532
|
+
output: TypedScreenshotToolOutput;
|
|
533
|
+
now?: Date;
|
|
534
|
+
retentionMs?: number;
|
|
535
|
+
workspaceQuotaBytes?: number;
|
|
536
|
+
}): Promise<RetainedArtifactMetadata> {
|
|
537
|
+
const identity = retainedScreenshotIdentity({
|
|
538
|
+
sessionId: input.sessionId,
|
|
539
|
+
turnId: input.turnId,
|
|
540
|
+
attemptId: input.attemptId,
|
|
541
|
+
toolCallId: input.output.callId,
|
|
542
|
+
toolOutputId: input.output.toolOutputId,
|
|
543
|
+
});
|
|
544
|
+
let screenshot: ValidatedSessionImage;
|
|
545
|
+
try {
|
|
546
|
+
screenshot = validateRetainableSessionImage({
|
|
547
|
+
bytes: input.output.bytes,
|
|
548
|
+
declaredMediaType: input.output.mediaType,
|
|
549
|
+
});
|
|
550
|
+
} catch (error) {
|
|
551
|
+
if (error instanceof ScreenshotValidationError) {
|
|
552
|
+
return unavailable(identity.artifactId, error.reason);
|
|
553
|
+
}
|
|
554
|
+
throw error;
|
|
555
|
+
}
|
|
556
|
+
if (!input.objectStorage) return unavailable(identity.artifactId, "missing_storage");
|
|
557
|
+
|
|
558
|
+
const now = input.now ?? new Date();
|
|
559
|
+
const retentionExpiresAt = new Date(
|
|
560
|
+
now.getTime() + (input.retentionMs ?? COMPUTER_SCREENSHOT_RETENTION_MS),
|
|
561
|
+
);
|
|
562
|
+
const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/session-image.${screenshot.extension}`;
|
|
563
|
+
let prepared: RetainedScreenshotArtifact;
|
|
564
|
+
try {
|
|
565
|
+
prepared = (
|
|
566
|
+
await prepareRetainedScreenshotArtifact(input.db, {
|
|
567
|
+
artifactId: identity.artifactId,
|
|
568
|
+
accountId: input.accountId,
|
|
569
|
+
workspaceId: input.workspaceId,
|
|
570
|
+
sessionId: input.sessionId,
|
|
571
|
+
turnId: input.turnId,
|
|
572
|
+
attemptId: input.attemptId,
|
|
573
|
+
settlementKey: identity.settlementKey,
|
|
574
|
+
toolCallId: input.output.callId,
|
|
575
|
+
toolOutputId: input.output.toolOutputId,
|
|
576
|
+
mediaType: screenshot.mediaType,
|
|
577
|
+
sizeBytes: screenshot.sizeBytes,
|
|
578
|
+
sha256: screenshot.sha256,
|
|
579
|
+
width: screenshot.width,
|
|
580
|
+
height: screenshot.height,
|
|
581
|
+
retentionExpiresAt,
|
|
582
|
+
bucket: input.objectStorage.bucket,
|
|
583
|
+
objectKey,
|
|
584
|
+
workspaceQuotaBytes: input.workspaceQuotaBytes ?? COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES,
|
|
585
|
+
})
|
|
586
|
+
).artifact;
|
|
587
|
+
} catch (error) {
|
|
588
|
+
if (error instanceof RetainedScreenshotQuotaExceededError) {
|
|
589
|
+
return unavailable(identity.artifactId, "quota_exceeded");
|
|
590
|
+
}
|
|
591
|
+
throw error;
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
if (prepared.status === "ready") {
|
|
595
|
+
return (await verifyReadyArtifact(input.objectStorage, prepared))
|
|
596
|
+
? reference(prepared)
|
|
597
|
+
: unavailable(identity.artifactId, "missing_storage");
|
|
598
|
+
}
|
|
599
|
+
if (prepared.status !== "pending" && prepared.status !== "reconciling") {
|
|
600
|
+
return unavailable(identity.artifactId, unavailableReasonForStatus(prepared.status));
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
try {
|
|
604
|
+
await input.objectStorage.putObject({
|
|
605
|
+
key: objectKey,
|
|
606
|
+
contentType: screenshot.mediaType,
|
|
607
|
+
body: screenshot.bytes,
|
|
608
|
+
sha256: screenshot.sha256,
|
|
609
|
+
});
|
|
610
|
+
const head = await input.objectStorage.headFile(prepared.file);
|
|
611
|
+
assertStoredScreenshotHead(head, screenshot);
|
|
612
|
+
} catch (error) {
|
|
613
|
+
await recordRetainedScreenshotArtifactError(input.db, {
|
|
614
|
+
accountId: input.accountId,
|
|
615
|
+
workspaceId: input.workspaceId,
|
|
616
|
+
artifactId: identity.artifactId,
|
|
617
|
+
settlementKey: identity.settlementKey,
|
|
618
|
+
error: error instanceof Error ? error.message : String(error),
|
|
619
|
+
}).catch(() => undefined);
|
|
620
|
+
return unavailable(identity.artifactId, "storage_write_failed");
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
try {
|
|
624
|
+
const settled = await settleRetainedScreenshotArtifactReady(input.db, {
|
|
625
|
+
accountId: input.accountId,
|
|
626
|
+
workspaceId: input.workspaceId,
|
|
627
|
+
artifactId: identity.artifactId,
|
|
628
|
+
settlementKey: identity.settlementKey,
|
|
629
|
+
});
|
|
630
|
+
return reference(settled);
|
|
631
|
+
} catch (error) {
|
|
632
|
+
// PUT completed. If a concurrent cascade removed the deterministic intent,
|
|
633
|
+
// compensate the now-unreferenced key. Otherwise leave the pending row for
|
|
634
|
+
// exact retry/background reconciliation instead of deleting live truth.
|
|
635
|
+
const stillReferenced = await getRetainedScreenshotArtifact(
|
|
636
|
+
input.db,
|
|
637
|
+
input.workspaceId,
|
|
638
|
+
input.sessionId,
|
|
639
|
+
identity.artifactId,
|
|
640
|
+
).catch(() => undefined);
|
|
641
|
+
if (stillReferenced?.status === "ready") {
|
|
642
|
+
return (await verifyReadyArtifact(input.objectStorage, stillReferenced))
|
|
643
|
+
? reference(stillReferenced)
|
|
644
|
+
: unavailable(identity.artifactId, "missing_storage");
|
|
645
|
+
}
|
|
646
|
+
if (
|
|
647
|
+
stillReferenced === null ||
|
|
648
|
+
stillReferenced?.status === "cleanup_queued" ||
|
|
649
|
+
stillReferenced?.status === "cleanup_pending" ||
|
|
650
|
+
stillReferenced?.status === "failed" ||
|
|
651
|
+
stillReferenced?.status === "expired" ||
|
|
652
|
+
stillReferenced?.status === "deleted"
|
|
653
|
+
) {
|
|
654
|
+
await input.objectStorage.deleteObject(objectKey).catch(() => undefined);
|
|
655
|
+
} else if (stillReferenced) {
|
|
656
|
+
await recordRetainedScreenshotArtifactError(input.db, {
|
|
657
|
+
accountId: input.accountId,
|
|
658
|
+
workspaceId: input.workspaceId,
|
|
659
|
+
artifactId: identity.artifactId,
|
|
660
|
+
settlementKey: identity.settlementKey,
|
|
661
|
+
error: error instanceof Error ? error.message : String(error),
|
|
662
|
+
}).catch(() => undefined);
|
|
663
|
+
}
|
|
664
|
+
return unavailable(identity.artifactId, "pending");
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/** Replace only new screenshot bytes with a compact receipt before first persistence. */
|
|
669
|
+
export function compactRetainedScreenshotHistory(
|
|
670
|
+
history: Array<Record<string, unknown>>,
|
|
671
|
+
receiptsByCallId: ReadonlyMap<string, RetainedArtifactMetadata>,
|
|
672
|
+
): Array<Record<string, unknown>> {
|
|
673
|
+
if (receiptsByCallId.size === 0) return history;
|
|
674
|
+
return history.map((item) => {
|
|
675
|
+
const callId = historyCallId(item);
|
|
676
|
+
const receipt = callId ? receiptsByCallId.get(callId) : undefined;
|
|
677
|
+
if (!receipt) return item;
|
|
678
|
+
if (typeof item.output === "string" && item.output.startsWith("data:image/")) {
|
|
679
|
+
return {
|
|
680
|
+
...item,
|
|
681
|
+
output: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
if (!Array.isArray(item.output)) return item;
|
|
685
|
+
let changed = false;
|
|
686
|
+
const output = item.output.map((entry) => {
|
|
687
|
+
if (!isInlineImageContent(entry)) return entry;
|
|
688
|
+
changed = true;
|
|
689
|
+
return {
|
|
690
|
+
...(entry as Record<string, unknown>),
|
|
691
|
+
image: { type: RETAINED_IMAGE_MARKER, artifact: receipt },
|
|
692
|
+
};
|
|
693
|
+
});
|
|
694
|
+
return changed ? { ...item, output } : item;
|
|
695
|
+
});
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
/** Collect durable screenshot receipts before their provider-only expansion. */
|
|
699
|
+
export function collectRetainedScreenshotReceipts(
|
|
700
|
+
history: ReadonlyArray<Record<string, unknown>>,
|
|
701
|
+
target: Map<string, RetainedArtifactMetadata> = new Map(),
|
|
702
|
+
): Map<string, RetainedArtifactMetadata> {
|
|
703
|
+
for (const item of history) {
|
|
704
|
+
const callId = historyCallId(item);
|
|
705
|
+
if (!callId) continue;
|
|
706
|
+
const direct = retainedReceiptFromDirectOutput(item.output);
|
|
707
|
+
if (direct) {
|
|
708
|
+
target.set(callId, direct);
|
|
709
|
+
continue;
|
|
710
|
+
}
|
|
711
|
+
if (!Array.isArray(item.output)) continue;
|
|
712
|
+
for (const entry of item.output) {
|
|
713
|
+
const receipt = retainedReceiptFromImageContent(entry);
|
|
714
|
+
if (receipt) {
|
|
715
|
+
target.set(callId, receipt);
|
|
716
|
+
break;
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
return target;
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/**
|
|
724
|
+
* Compact every protocol-item copy inside an SDK RunState before persistence.
|
|
725
|
+
* The traversal intentionally mirrors the repository's existing serialized
|
|
726
|
+
* RunState sanitizers. A parse failure or no-op returns the original string.
|
|
727
|
+
*/
|
|
728
|
+
export function compactRetainedScreenshotRunState(
|
|
729
|
+
serialized: string,
|
|
730
|
+
receiptsByCallId: ReadonlyMap<string, RetainedArtifactMetadata>,
|
|
731
|
+
): string {
|
|
732
|
+
if (receiptsByCallId.size === 0) return serialized;
|
|
733
|
+
const parsed = parseSerializedRunState(serialized);
|
|
734
|
+
if (!parsed) return serialized;
|
|
735
|
+
let changed = false;
|
|
736
|
+
visitSerializedRunStateItemArrays(parsed, (items) => {
|
|
737
|
+
const compacted = compactRetainedScreenshotHistory(items, receiptsByCallId);
|
|
738
|
+
if (compacted !== items && compacted.some((item, index) => item !== items[index])) {
|
|
739
|
+
items.splice(0, items.length, ...compacted);
|
|
740
|
+
changed = true;
|
|
741
|
+
}
|
|
742
|
+
});
|
|
743
|
+
return changed ? JSON.stringify(parsed) : serialized;
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
export function collectRetainedScreenshotRunStateReceipts(
|
|
747
|
+
serialized: string,
|
|
748
|
+
target: Map<string, RetainedArtifactMetadata> = new Map(),
|
|
749
|
+
): Map<string, RetainedArtifactMetadata> {
|
|
750
|
+
const parsed = parseSerializedRunState(serialized);
|
|
751
|
+
if (!parsed) return target;
|
|
752
|
+
visitSerializedRunStateItemArrays(parsed, (items) => {
|
|
753
|
+
collectRetainedScreenshotReceipts(items, target);
|
|
754
|
+
});
|
|
755
|
+
return target;
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
/**
|
|
759
|
+
* Materialize canonical receipts to the exact SDK input_image data URL shape.
|
|
760
|
+
* Missing/deleted/expired/corrupt bytes fail closed; no item is omitted,
|
|
761
|
+
* reordered, rewritten in storage, or converted to text.
|
|
762
|
+
*/
|
|
763
|
+
export async function materializeRetainedScreenshotHistory(input: {
|
|
764
|
+
db: Database;
|
|
765
|
+
objectStorage: ObjectStorage | null;
|
|
766
|
+
workspaceId: string;
|
|
767
|
+
sessionId: string;
|
|
768
|
+
history: Array<Record<string, unknown>>;
|
|
769
|
+
now?: Date;
|
|
770
|
+
}): Promise<Array<Record<string, unknown>>> {
|
|
771
|
+
return await materializeRetainedScreenshotHistoryWithCache(input, new Map());
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
/** Expand durable markers in an SDK RunState immediately before resume. */
|
|
775
|
+
export async function materializeRetainedScreenshotRunState(input: {
|
|
776
|
+
db: Database;
|
|
777
|
+
objectStorage: ObjectStorage | null;
|
|
778
|
+
workspaceId: string;
|
|
779
|
+
sessionId: string;
|
|
780
|
+
serialized: string;
|
|
781
|
+
now?: Date;
|
|
782
|
+
}): Promise<string> {
|
|
783
|
+
const parsed = parseSerializedRunState(input.serialized);
|
|
784
|
+
if (!parsed) return input.serialized;
|
|
785
|
+
const cache = new Map<string, string>();
|
|
786
|
+
let changed = false;
|
|
787
|
+
await visitSerializedRunStateItemArraysAsync(parsed, async (items) => {
|
|
788
|
+
const materialized = await materializeRetainedScreenshotHistoryWithCache(
|
|
789
|
+
{ ...input, history: items },
|
|
790
|
+
cache,
|
|
791
|
+
);
|
|
792
|
+
if (materialized.some((item, index) => item !== items[index])) {
|
|
793
|
+
items.splice(0, items.length, ...materialized);
|
|
794
|
+
changed = true;
|
|
795
|
+
}
|
|
796
|
+
});
|
|
797
|
+
return changed ? JSON.stringify(parsed) : input.serialized;
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
async function materializeRetainedScreenshotHistoryWithCache(
|
|
801
|
+
input: {
|
|
802
|
+
db: Database;
|
|
803
|
+
objectStorage: ObjectStorage | null;
|
|
804
|
+
workspaceId: string;
|
|
805
|
+
sessionId: string;
|
|
806
|
+
history: Array<Record<string, unknown>>;
|
|
807
|
+
now?: Date;
|
|
808
|
+
},
|
|
809
|
+
cache: Map<string, string>,
|
|
810
|
+
): Promise<Array<Record<string, unknown>>> {
|
|
811
|
+
const now = input.now ?? new Date();
|
|
812
|
+
const dataUrlForReceipt = async (receipt: RetainedArtifactMetadata): Promise<string> => {
|
|
813
|
+
if (!receipt.available) {
|
|
814
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, receipt.reason);
|
|
815
|
+
}
|
|
816
|
+
let dataUrl = cache.get(receipt.artifactId);
|
|
817
|
+
if (!dataUrl) {
|
|
818
|
+
if (!input.objectStorage) {
|
|
819
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
|
|
820
|
+
}
|
|
821
|
+
const artifact = await getRetainedScreenshotArtifact(
|
|
822
|
+
input.db,
|
|
823
|
+
input.workspaceId,
|
|
824
|
+
input.sessionId,
|
|
825
|
+
receipt.artifactId,
|
|
826
|
+
);
|
|
827
|
+
if (!artifact) throw new RetainedScreenshotUnavailableError(receipt.artifactId, "deleted");
|
|
828
|
+
if (artifact.status !== "ready") {
|
|
829
|
+
throw new RetainedScreenshotUnavailableError(
|
|
830
|
+
receipt.artifactId,
|
|
831
|
+
unavailableReasonForStatus(artifact.status),
|
|
832
|
+
);
|
|
833
|
+
}
|
|
834
|
+
if (artifact.retentionExpiresAt.getTime() <= now.getTime()) {
|
|
835
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "expired");
|
|
836
|
+
}
|
|
837
|
+
const bytes = await input.objectStorage.getFileBytes(artifact.file).catch(() => null);
|
|
838
|
+
if (!bytes)
|
|
839
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
|
|
840
|
+
const validated = validateRetainableSessionImage({
|
|
841
|
+
bytes,
|
|
842
|
+
declaredMediaType: artifact.mediaType,
|
|
843
|
+
});
|
|
844
|
+
if (
|
|
845
|
+
validated.sizeBytes !== artifact.sizeBytes ||
|
|
846
|
+
validated.sha256 !== artifact.sha256 ||
|
|
847
|
+
validated.width !== artifact.width ||
|
|
848
|
+
validated.height !== artifact.height ||
|
|
849
|
+
receipt.sha256 !== artifact.sha256 ||
|
|
850
|
+
receipt.originalBytes !== artifact.sizeBytes
|
|
851
|
+
) {
|
|
852
|
+
throw new RetainedScreenshotUnavailableError(receipt.artifactId, "invalid_content");
|
|
853
|
+
}
|
|
854
|
+
dataUrl = `data:${validated.mediaType};base64,${Buffer.from(validated.bytes).toString("base64")}`;
|
|
855
|
+
cache.set(receipt.artifactId, dataUrl);
|
|
856
|
+
}
|
|
857
|
+
return dataUrl;
|
|
858
|
+
};
|
|
859
|
+
const materializeEntry = async (entry: unknown): Promise<unknown> => {
|
|
860
|
+
const receipt = retainedReceiptFromImageContent(entry);
|
|
861
|
+
if (!receipt) return entry;
|
|
862
|
+
const dataUrl = await dataUrlForReceipt(receipt);
|
|
863
|
+
return { ...(entry as Record<string, unknown>), image: dataUrl };
|
|
864
|
+
};
|
|
865
|
+
|
|
866
|
+
const materialized: Array<Record<string, unknown>> = [];
|
|
867
|
+
for (const item of input.history) {
|
|
868
|
+
const directReceipt = retainedReceiptFromDirectOutput(item.output);
|
|
869
|
+
if (directReceipt) {
|
|
870
|
+
materialized.push({
|
|
871
|
+
...item,
|
|
872
|
+
output: await dataUrlForReceipt(directReceipt),
|
|
873
|
+
});
|
|
874
|
+
continue;
|
|
875
|
+
}
|
|
876
|
+
if (!Array.isArray(item.output)) {
|
|
877
|
+
materialized.push(item);
|
|
878
|
+
continue;
|
|
879
|
+
}
|
|
880
|
+
let changed = false;
|
|
881
|
+
const output: unknown[] = [];
|
|
882
|
+
for (const entry of item.output) {
|
|
883
|
+
const next = await materializeEntry(entry);
|
|
884
|
+
changed ||= next !== entry;
|
|
885
|
+
output.push(next);
|
|
886
|
+
}
|
|
887
|
+
materialized.push(changed ? { ...item, output } : item);
|
|
888
|
+
}
|
|
889
|
+
return materialized;
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
function parseSerializedRunState(serialized: string): Record<string, unknown> | null {
|
|
893
|
+
try {
|
|
894
|
+
const parsed: unknown = JSON.parse(serialized);
|
|
895
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
|
896
|
+
? (parsed as Record<string, unknown>)
|
|
897
|
+
: null;
|
|
898
|
+
} catch {
|
|
899
|
+
return null;
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
function visitSerializedRunStateItemArrays(
|
|
904
|
+
root: Record<string, unknown>,
|
|
905
|
+
visitor: (items: Array<Record<string, unknown>>) => void,
|
|
906
|
+
): void {
|
|
907
|
+
const originalInput = root.originalInput;
|
|
908
|
+
if (Array.isArray(originalInput)) {
|
|
909
|
+
visitor(originalInput as Array<Record<string, unknown>>);
|
|
910
|
+
}
|
|
911
|
+
if (Array.isArray(root.generatedItems)) {
|
|
912
|
+
for (const wrapper of root.generatedItems) {
|
|
913
|
+
if (!wrapper || typeof wrapper !== "object" || Array.isArray(wrapper)) continue;
|
|
914
|
+
const record = wrapper as Record<string, unknown>;
|
|
915
|
+
const rawItem = record.rawItem;
|
|
916
|
+
if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
|
|
917
|
+
const items = [rawItem as Record<string, unknown>];
|
|
918
|
+
visitor(items);
|
|
919
|
+
if (items[0] !== rawItem) record.rawItem = items[0];
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
if (Array.isArray(root.modelResponses)) {
|
|
923
|
+
for (const response of root.modelResponses) {
|
|
924
|
+
if (!response || typeof response !== "object" || Array.isArray(response)) continue;
|
|
925
|
+
const output = (response as Record<string, unknown>).output;
|
|
926
|
+
if (Array.isArray(output)) visitor(output as Array<Record<string, unknown>>);
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
const lastModelResponse = root.lastModelResponse;
|
|
930
|
+
if (
|
|
931
|
+
lastModelResponse &&
|
|
932
|
+
typeof lastModelResponse === "object" &&
|
|
933
|
+
!Array.isArray(lastModelResponse)
|
|
934
|
+
) {
|
|
935
|
+
const output = (lastModelResponse as Record<string, unknown>).output;
|
|
936
|
+
if (Array.isArray(output)) visitor(output as Array<Record<string, unknown>>);
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
async function visitSerializedRunStateItemArraysAsync(
|
|
941
|
+
root: Record<string, unknown>,
|
|
942
|
+
visitor: (items: Array<Record<string, unknown>>) => Promise<void>,
|
|
943
|
+
): Promise<void> {
|
|
944
|
+
const originalInput = root.originalInput;
|
|
945
|
+
if (Array.isArray(originalInput)) {
|
|
946
|
+
await visitor(originalInput as Array<Record<string, unknown>>);
|
|
947
|
+
}
|
|
948
|
+
if (Array.isArray(root.generatedItems)) {
|
|
949
|
+
for (const wrapper of root.generatedItems) {
|
|
950
|
+
if (!wrapper || typeof wrapper !== "object" || Array.isArray(wrapper)) continue;
|
|
951
|
+
const record = wrapper as Record<string, unknown>;
|
|
952
|
+
const rawItem = record.rawItem;
|
|
953
|
+
if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
|
|
954
|
+
const items = [rawItem as Record<string, unknown>];
|
|
955
|
+
await visitor(items);
|
|
956
|
+
if (items[0] !== rawItem) record.rawItem = items[0];
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
if (Array.isArray(root.modelResponses)) {
|
|
960
|
+
for (const response of root.modelResponses) {
|
|
961
|
+
if (!response || typeof response !== "object" || Array.isArray(response)) continue;
|
|
962
|
+
const output = (response as Record<string, unknown>).output;
|
|
963
|
+
if (Array.isArray(output)) await visitor(output as Array<Record<string, unknown>>);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
const lastModelResponse = root.lastModelResponse;
|
|
967
|
+
if (
|
|
968
|
+
lastModelResponse &&
|
|
969
|
+
typeof lastModelResponse === "object" &&
|
|
970
|
+
!Array.isArray(lastModelResponse)
|
|
971
|
+
) {
|
|
972
|
+
const output = (lastModelResponse as Record<string, unknown>).output;
|
|
973
|
+
if (Array.isArray(output)) await visitor(output as Array<Record<string, unknown>>);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
function reference(artifact: RetainedScreenshotArtifact): RetainedArtifactReference {
|
|
978
|
+
if (!artifact.sessionId) {
|
|
979
|
+
throw new Error(`Ready retained screenshot is detached: ${artifact.artifactId}`);
|
|
980
|
+
}
|
|
981
|
+
const value = retainedScreenshotReferenceFromFile({
|
|
982
|
+
...artifact.file,
|
|
983
|
+
sessionId: artifact.sessionId,
|
|
984
|
+
width: artifact.width,
|
|
985
|
+
height: artifact.height,
|
|
986
|
+
expiresAt: artifact.retentionExpiresAt.toISOString(),
|
|
987
|
+
});
|
|
988
|
+
if (!value)
|
|
989
|
+
throw new Error(`Ready retained screenshot receipt is invalid: ${artifact.artifactId}`);
|
|
990
|
+
return value;
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
function unavailable(
|
|
994
|
+
artifactId: string,
|
|
995
|
+
reason: RetainedOutputUnavailableReason,
|
|
996
|
+
): RetainedArtifactMetadata {
|
|
997
|
+
return RetainedArtifactMetadataSchema.parse({
|
|
998
|
+
available: false,
|
|
999
|
+
artifactId,
|
|
1000
|
+
reason,
|
|
1001
|
+
});
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
function unavailableReasonForStatus(
|
|
1005
|
+
status: RetainedScreenshotArtifact["status"],
|
|
1006
|
+
): RetainedOutputUnavailableReason {
|
|
1007
|
+
switch (status) {
|
|
1008
|
+
case "pending":
|
|
1009
|
+
case "reconciling":
|
|
1010
|
+
return "pending";
|
|
1011
|
+
case "cleanup_queued":
|
|
1012
|
+
case "cleanup_pending":
|
|
1013
|
+
return "failed";
|
|
1014
|
+
case "failed":
|
|
1015
|
+
case "expired":
|
|
1016
|
+
case "deleted":
|
|
1017
|
+
return status;
|
|
1018
|
+
case "ready":
|
|
1019
|
+
return "missing_storage";
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
async function verifyReadyArtifact(
|
|
1024
|
+
storage: ObjectStorage,
|
|
1025
|
+
artifact: RetainedScreenshotArtifact,
|
|
1026
|
+
): Promise<boolean> {
|
|
1027
|
+
try {
|
|
1028
|
+
assertStoredScreenshotHead(await storage.headFile(artifact.file), {
|
|
1029
|
+
sizeBytes: artifact.sizeBytes,
|
|
1030
|
+
mediaType: artifact.mediaType as RetainableSessionImageMediaType,
|
|
1031
|
+
sha256: artifact.sha256,
|
|
1032
|
+
});
|
|
1033
|
+
return true;
|
|
1034
|
+
} catch {
|
|
1035
|
+
return false;
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
function assertStoredScreenshotHead(
|
|
1040
|
+
head: ObjectHead,
|
|
1041
|
+
expected: Pick<ValidatedSessionImage, "sizeBytes" | "mediaType" | "sha256">,
|
|
1042
|
+
): void {
|
|
1043
|
+
if (head.ContentLength !== expected.sizeBytes) {
|
|
1044
|
+
throw new Error("retained screenshot object size mismatch");
|
|
1045
|
+
}
|
|
1046
|
+
if (head.ContentType !== expected.mediaType) {
|
|
1047
|
+
throw new Error("retained screenshot object MIME mismatch");
|
|
1048
|
+
}
|
|
1049
|
+
if (head.Metadata?.sha256 !== expected.sha256) {
|
|
1050
|
+
throw new Error("retained screenshot object SHA-256 metadata mismatch");
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
function retainedReceiptFromImageContent(entry: unknown): RetainedArtifactMetadata | null {
|
|
1055
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return null;
|
|
1056
|
+
const image = (entry as Record<string, unknown>).image;
|
|
1057
|
+
if (!image || typeof image !== "object" || Array.isArray(image)) return null;
|
|
1058
|
+
const marker = image as Record<string, unknown>;
|
|
1059
|
+
if (marker.type !== RETAINED_IMAGE_MARKER) return null;
|
|
1060
|
+
return retainedReceipt(marker.artifact);
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
function retainedReceiptFromDirectOutput(output: unknown): RetainedArtifactMetadata | null {
|
|
1064
|
+
if (!output || typeof output !== "object" || Array.isArray(output)) return null;
|
|
1065
|
+
const marker = output as Record<string, unknown>;
|
|
1066
|
+
if (marker.type !== RETAINED_IMAGE_MARKER) return null;
|
|
1067
|
+
return retainedReceipt(marker.artifact);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
function retainedReceipt(value: unknown): RetainedArtifactMetadata | null {
|
|
1071
|
+
const parsed = RetainedArtifactMetadataSchema.safeParse(value);
|
|
1072
|
+
if (!parsed.success) return null;
|
|
1073
|
+
return parsed.data.available && parsed.data.kind !== "computer_screenshot" ? null : parsed.data;
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
function isInlineImageContent(entry: unknown): boolean {
|
|
1077
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
|
|
1078
|
+
const record = entry as Record<string, unknown>;
|
|
1079
|
+
return (
|
|
1080
|
+
record.type === "input_image" &&
|
|
1081
|
+
typeof record.image === "string" &&
|
|
1082
|
+
record.image.startsWith("data:image/")
|
|
1083
|
+
);
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
function historyCallId(item: Record<string, unknown>): string | null {
|
|
1087
|
+
const value = item.callId ?? item.call_id;
|
|
1088
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
function canonicalMediaType(value: string): string {
|
|
1092
|
+
return value.split(";", 1)[0]!.trim().toLowerCase();
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
function isPng(bytes: Uint8Array): boolean {
|
|
1096
|
+
return (
|
|
1097
|
+
bytes.byteLength >= PNG_SIGNATURE.byteLength &&
|
|
1098
|
+
PNG_SIGNATURE.every((byte, index) => bytes[index] === byte)
|
|
1099
|
+
);
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
function isJpeg(bytes: Uint8Array): boolean {
|
|
1103
|
+
return bytes.byteLength >= 4 && bytes[0] === 0xff && bytes[1] === 0xd8;
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
function jpegDimensions(bytes: Uint8Array): { width: number; height: number } {
|
|
1107
|
+
if (bytes[bytes.length - 2] !== 0xff || bytes[bytes.length - 1] !== 0xd9) {
|
|
1108
|
+
throw new ScreenshotValidationError(
|
|
1109
|
+
"invalid_content",
|
|
1110
|
+
"session JPEG is truncated or has trailing bytes",
|
|
1111
|
+
);
|
|
1112
|
+
}
|
|
1113
|
+
let offset = 2;
|
|
1114
|
+
while (offset + 4 <= bytes.length) {
|
|
1115
|
+
if (bytes[offset] !== 0xff) {
|
|
1116
|
+
throw new ScreenshotValidationError(
|
|
1117
|
+
"invalid_content",
|
|
1118
|
+
"session JPEG marker table is invalid",
|
|
1119
|
+
);
|
|
1120
|
+
}
|
|
1121
|
+
while (bytes[offset] === 0xff) offset += 1;
|
|
1122
|
+
const marker = bytes[offset++];
|
|
1123
|
+
if (marker === undefined || marker === 0xd9 || marker === 0xda) break;
|
|
1124
|
+
if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue;
|
|
1125
|
+
if (offset + 2 > bytes.length) break;
|
|
1126
|
+
const length = readUint16Be(bytes, offset);
|
|
1127
|
+
if (length < 2 || offset + length > bytes.length) {
|
|
1128
|
+
throw new ScreenshotValidationError("invalid_content", "session JPEG segment is invalid");
|
|
1129
|
+
}
|
|
1130
|
+
if (JPEG_SOF_MARKERS.has(marker)) {
|
|
1131
|
+
if (length < 7) {
|
|
1132
|
+
throw new ScreenshotValidationError("invalid_content", "session JPEG frame is invalid");
|
|
1133
|
+
}
|
|
1134
|
+
return {
|
|
1135
|
+
height: readUint16Be(bytes, offset + 3),
|
|
1136
|
+
width: readUint16Be(bytes, offset + 5),
|
|
1137
|
+
};
|
|
1138
|
+
}
|
|
1139
|
+
offset += length;
|
|
1140
|
+
}
|
|
1141
|
+
throw new ScreenshotValidationError(
|
|
1142
|
+
"invalid_content",
|
|
1143
|
+
"session JPEG has no supported frame header",
|
|
1144
|
+
);
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
const JPEG_SOF_MARKERS = new Set([
|
|
1148
|
+
0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf,
|
|
1149
|
+
]);
|
|
1150
|
+
|
|
1151
|
+
function isWebp(bytes: Uint8Array): boolean {
|
|
1152
|
+
return bytes.byteLength >= 20 && ascii(bytes, 0, 4) === "RIFF" && ascii(bytes, 8, 4) === "WEBP";
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
function webpDimensions(bytes: Uint8Array): { width: number; height: number } {
|
|
1156
|
+
if (readUint32Le(bytes, 4) + 8 !== bytes.byteLength) {
|
|
1157
|
+
throw new ScreenshotValidationError("invalid_content", "session WebP RIFF length is invalid");
|
|
1158
|
+
}
|
|
1159
|
+
let offset = 12;
|
|
1160
|
+
while (offset + 8 <= bytes.length) {
|
|
1161
|
+
const type = ascii(bytes, offset, 4);
|
|
1162
|
+
const length = readUint32Le(bytes, offset + 4);
|
|
1163
|
+
const data = offset + 8;
|
|
1164
|
+
const next = data + length + (length & 1);
|
|
1165
|
+
if (next > bytes.length) {
|
|
1166
|
+
throw new ScreenshotValidationError("invalid_content", "session WebP chunk is truncated");
|
|
1167
|
+
}
|
|
1168
|
+
if (type === "VP8X") {
|
|
1169
|
+
if (length < 10) {
|
|
1170
|
+
throw new ScreenshotValidationError("invalid_content", "session WebP VP8X is invalid");
|
|
1171
|
+
}
|
|
1172
|
+
return {
|
|
1173
|
+
width: 1 + readUint24Le(bytes, data + 4),
|
|
1174
|
+
height: 1 + readUint24Le(bytes, data + 7),
|
|
1175
|
+
};
|
|
1176
|
+
}
|
|
1177
|
+
if (type === "VP8 ") {
|
|
1178
|
+
if (
|
|
1179
|
+
length < 10 ||
|
|
1180
|
+
bytes[data + 3] !== 0x9d ||
|
|
1181
|
+
bytes[data + 4] !== 0x01 ||
|
|
1182
|
+
bytes[data + 5] !== 0x2a
|
|
1183
|
+
) {
|
|
1184
|
+
throw new ScreenshotValidationError("invalid_content", "session WebP VP8 frame is invalid");
|
|
1185
|
+
}
|
|
1186
|
+
return {
|
|
1187
|
+
width: readUint16Le(bytes, data + 6) & 0x3fff,
|
|
1188
|
+
height: readUint16Le(bytes, data + 8) & 0x3fff,
|
|
1189
|
+
};
|
|
1190
|
+
}
|
|
1191
|
+
if (type === "VP8L") {
|
|
1192
|
+
if (length < 5 || bytes[data] !== 0x2f) {
|
|
1193
|
+
throw new ScreenshotValidationError(
|
|
1194
|
+
"invalid_content",
|
|
1195
|
+
"session WebP VP8L frame is invalid",
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1198
|
+
const bits = readUint32Le(bytes, data + 1);
|
|
1199
|
+
return {
|
|
1200
|
+
width: 1 + (bits & 0x3fff),
|
|
1201
|
+
height: 1 + ((bits >>> 14) & 0x3fff),
|
|
1202
|
+
};
|
|
1203
|
+
}
|
|
1204
|
+
offset = next;
|
|
1205
|
+
}
|
|
1206
|
+
throw new ScreenshotValidationError("invalid_content", "session WebP has no image frame");
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
function assertImageDimensions(
|
|
1210
|
+
width: number,
|
|
1211
|
+
height: number,
|
|
1212
|
+
limits: { maxDimension: number; maxPixels: number },
|
|
1213
|
+
): void {
|
|
1214
|
+
if (
|
|
1215
|
+
!Number.isSafeInteger(width) ||
|
|
1216
|
+
!Number.isSafeInteger(height) ||
|
|
1217
|
+
width <= 0 ||
|
|
1218
|
+
height <= 0 ||
|
|
1219
|
+
width > limits.maxDimension ||
|
|
1220
|
+
height > limits.maxDimension ||
|
|
1221
|
+
width * height > limits.maxPixels
|
|
1222
|
+
) {
|
|
1223
|
+
throw new ScreenshotValidationError("oversized", "session image dimensions exceed policy");
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
function readUint16Be(bytes: Uint8Array, offset: number): number {
|
|
1228
|
+
return (bytes[offset]! << 8) | bytes[offset + 1]!;
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
function readUint16Le(bytes: Uint8Array, offset: number): number {
|
|
1232
|
+
return bytes[offset]! | (bytes[offset + 1]! << 8);
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
function readUint24Le(bytes: Uint8Array, offset: number): number {
|
|
1236
|
+
return bytes[offset]! | (bytes[offset + 1]! << 8) | (bytes[offset + 2]! << 16);
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
function readUint32Le(bytes: Uint8Array, offset: number): number {
|
|
1240
|
+
return (
|
|
1241
|
+
(bytes[offset]! |
|
|
1242
|
+
(bytes[offset + 1]! << 8) |
|
|
1243
|
+
(bytes[offset + 2]! << 16) |
|
|
1244
|
+
(bytes[offset + 3]! << 24)) >>>
|
|
1245
|
+
0
|
|
1246
|
+
);
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
function ascii(bytes: Uint8Array, offset: number, length: number): string {
|
|
1250
|
+
return String.fromCharCode(...bytes.subarray(offset, offset + length));
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
function base64Sextet(code: number): number {
|
|
1254
|
+
if (code >= 0x41 && code <= 0x5a) return code - 0x41;
|
|
1255
|
+
if (code >= 0x61 && code <= 0x7a) return code - 0x61 + 26;
|
|
1256
|
+
if (code >= 0x30 && code <= 0x39) return code - 0x30 + 52;
|
|
1257
|
+
return code === 0x2b ? 62 : 63;
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
function readUint32(bytes: Uint8Array, offset: number): number {
|
|
1261
|
+
return (
|
|
1262
|
+
((bytes[offset]! << 24) |
|
|
1263
|
+
(bytes[offset + 1]! << 16) |
|
|
1264
|
+
(bytes[offset + 2]! << 8) |
|
|
1265
|
+
bytes[offset + 3]!) >>>
|
|
1266
|
+
0
|
|
1267
|
+
);
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
function asciiChunkType(bytes: Uint8Array, offset: number): string {
|
|
1271
|
+
const chars = [bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]];
|
|
1272
|
+
if (
|
|
1273
|
+
chars.some(
|
|
1274
|
+
(value) => value === undefined || value < 65 || (value > 90 && value < 97) || value > 122,
|
|
1275
|
+
)
|
|
1276
|
+
) {
|
|
1277
|
+
throw new ScreenshotValidationError(
|
|
1278
|
+
"invalid_content",
|
|
1279
|
+
"computer screenshot PNG chunk type is invalid",
|
|
1280
|
+
);
|
|
1281
|
+
}
|
|
1282
|
+
return String.fromCharCode(...(chars as number[]));
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
function isCriticalChunk(type: string): boolean {
|
|
1286
|
+
return type.charCodeAt(0) >= 65 && type.charCodeAt(0) <= 90;
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
function validPngColorMode(bitDepth: number | undefined, colorType: number | undefined): boolean {
|
|
1290
|
+
if (bitDepth === undefined || colorType === undefined) return false;
|
|
1291
|
+
switch (colorType) {
|
|
1292
|
+
case 0:
|
|
1293
|
+
return [1, 2, 4, 8, 16].includes(bitDepth);
|
|
1294
|
+
case 2:
|
|
1295
|
+
case 4:
|
|
1296
|
+
case 6:
|
|
1297
|
+
return bitDepth === 8 || bitDepth === 16;
|
|
1298
|
+
case 3:
|
|
1299
|
+
return [1, 2, 4, 8].includes(bitDepth);
|
|
1300
|
+
default:
|
|
1301
|
+
return false;
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
const CRC32_TABLE = Uint32Array.from({ length: 256 }, (_, value) => {
|
|
1306
|
+
let current = value;
|
|
1307
|
+
for (let bit = 0; bit < 8; bit += 1) {
|
|
1308
|
+
current = (current & 1) !== 0 ? 0xedb88320 ^ (current >>> 1) : current >>> 1;
|
|
1309
|
+
}
|
|
1310
|
+
return current >>> 0;
|
|
1311
|
+
});
|
|
1312
|
+
|
|
1313
|
+
function crc32(bytes: Uint8Array): number {
|
|
1314
|
+
let crc = 0xffffffff;
|
|
1315
|
+
for (const byte of bytes) crc = CRC32_TABLE[(crc ^ byte) & 0xff]! ^ (crc >>> 8);
|
|
1316
|
+
return (crc ^ 0xffffffff) >>> 0;
|
|
1317
|
+
}
|