@opengeni/worker-bundle 0.16.29 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +85 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +662 -17374
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +2 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1043 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1470 -246
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +6 -6
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +45 -42
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +77 -4
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +48 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +199 -4
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +19 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +3 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
|
@@ -0,0 +1,956 @@
|
|
|
1
|
+
import { once } from "node:events";
|
|
2
|
+
import { isAbsolute } from "node:path";
|
|
3
|
+
import type { ChildProcessWithoutNullStreams } from "node:child_process";
|
|
4
|
+
|
|
5
|
+
import { EDITABLE_ARTIFACT_KERNEL_VERSION_MAX_BYTES } from "@opengeni/contracts/editable-artifacts";
|
|
6
|
+
|
|
7
|
+
import {
|
|
8
|
+
EditableArtifactMaterializerPermanentError,
|
|
9
|
+
mimeTypeForFormat,
|
|
10
|
+
type ClaimedEditableArtifactMaterialization,
|
|
11
|
+
type EditableArtifactMaterializerFailureDiagnostic,
|
|
12
|
+
type NativeEditableArtifactMaterializationResult,
|
|
13
|
+
type NativeEditableArtifactMaterializerIdentity,
|
|
14
|
+
type NativeEditableArtifactMaterializerPort,
|
|
15
|
+
} from "./editable-artifact-materializer";
|
|
16
|
+
|
|
17
|
+
const INPUT_MAGIC = new TextEncoder().encode("OGAMI001");
|
|
18
|
+
const VERIFY_INPUT_MAGIC = new TextEncoder().encode("OGAVI001");
|
|
19
|
+
const OUTPUT_MAGIC = new TextEncoder().encode("OGAMO001");
|
|
20
|
+
const VERIFY_OUTPUT_MAGIC = new TextEncoder().encode("OGAVO001");
|
|
21
|
+
const ERROR_MAGIC = new TextEncoder().encode("OGAME001");
|
|
22
|
+
const IDENTITY_ARGUMENT = "--opengeni-materializer-identity-v1";
|
|
23
|
+
const MATERIALIZE_ARGUMENT = "--opengeni-materialize-v1";
|
|
24
|
+
const VERIFY_ARGUMENT = "--opengeni-verify-materialization-v1";
|
|
25
|
+
const MAX_IDENTITY_BYTES = 64 * 1024;
|
|
26
|
+
const MAX_METADATA_BYTES = 256 * 1024;
|
|
27
|
+
const MAX_STDERR_BYTES = 64 * 1024;
|
|
28
|
+
const STREAM_CHUNK_BYTES = 1024 * 1024;
|
|
29
|
+
|
|
30
|
+
export type NativeEditableArtifactSubprocessOptions = Readonly<{
|
|
31
|
+
/** Absolute path to the exact codec executable. Isolation belongs to launcher. */
|
|
32
|
+
executable: string;
|
|
33
|
+
launcher: VerifiedEditableArtifactProcessLauncher;
|
|
34
|
+
childEnvironment?: Readonly<Record<string, string>>;
|
|
35
|
+
/** Explicitly permits the current-host local-dev launcher; never set in production. */
|
|
36
|
+
allowUnsandboxedDevelopment?: boolean;
|
|
37
|
+
wallTimeoutMs: number;
|
|
38
|
+
maxSourceBytes: number;
|
|
39
|
+
maxOutputBytes: number;
|
|
40
|
+
probeTimeoutMs?: number;
|
|
41
|
+
}>;
|
|
42
|
+
|
|
43
|
+
export type VerifiedEditableArtifactProcessLauncher = Readonly<{
|
|
44
|
+
identity: Readonly<{
|
|
45
|
+
platform: string;
|
|
46
|
+
isolation: "subprocess";
|
|
47
|
+
network: "denied" | "host";
|
|
48
|
+
officeAutomation: false;
|
|
49
|
+
sandboxEnforced: boolean;
|
|
50
|
+
memoryLimitBytes: number;
|
|
51
|
+
cpuTimeLimitMs: number;
|
|
52
|
+
fileDescriptorLimit: number;
|
|
53
|
+
processLimit: number;
|
|
54
|
+
fileSizeLimitBytes: number;
|
|
55
|
+
}>;
|
|
56
|
+
spawn(input: {
|
|
57
|
+
executable: string;
|
|
58
|
+
args: readonly string[];
|
|
59
|
+
environment: Readonly<Record<string, string>>;
|
|
60
|
+
}): ChildProcessWithoutNullStreams;
|
|
61
|
+
}>;
|
|
62
|
+
|
|
63
|
+
type IdentityEnvelope = Readonly<{
|
|
64
|
+
protocol: "OGAMC001";
|
|
65
|
+
runtimeKind: "native";
|
|
66
|
+
runtimeTarget: string;
|
|
67
|
+
kernelVersion: string;
|
|
68
|
+
codecVersions: Readonly<Record<string, string>>;
|
|
69
|
+
fontRegistryHash: string;
|
|
70
|
+
policyHash: string;
|
|
71
|
+
maxOutputBytes: number;
|
|
72
|
+
supportedModelSchemaVersions: readonly number[];
|
|
73
|
+
supportedOperationProtocolVersions: readonly number[];
|
|
74
|
+
supportedSnapshotProtocolVersions: readonly number[];
|
|
75
|
+
}>;
|
|
76
|
+
|
|
77
|
+
export type NativeEditableArtifactSubprocessPort = NativeEditableArtifactMaterializerPort &
|
|
78
|
+
Readonly<{
|
|
79
|
+
verifyMaterialization(input: {
|
|
80
|
+
format: string;
|
|
81
|
+
codecId: string;
|
|
82
|
+
codecVersion: string;
|
|
83
|
+
expectedSemanticHash: string;
|
|
84
|
+
byteSize: number;
|
|
85
|
+
chunks: AsyncIterable<Uint8Array>;
|
|
86
|
+
signal: AbortSignal;
|
|
87
|
+
}): Promise<void>;
|
|
88
|
+
}>;
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Probe exact codec/kernel capabilities and combine them with independently
|
|
92
|
+
* verified parent-launcher isolation. The child is never trusted to attest its
|
|
93
|
+
* own network namespace, resource limits, or platform sandbox.
|
|
94
|
+
*/
|
|
95
|
+
export async function createNativeEditableArtifactSubprocessPort(
|
|
96
|
+
options: NativeEditableArtifactSubprocessOptions,
|
|
97
|
+
): Promise<NativeEditableArtifactSubprocessPort> {
|
|
98
|
+
validateOptions(options);
|
|
99
|
+
const childIdentity = await probeIdentity(options);
|
|
100
|
+
validateIdentity(childIdentity);
|
|
101
|
+
const identity = compositeIdentity(childIdentity, options);
|
|
102
|
+
return Object.freeze({
|
|
103
|
+
identity,
|
|
104
|
+
materialize: async (
|
|
105
|
+
input: Parameters<NativeEditableArtifactMaterializerPort["materialize"]>[0],
|
|
106
|
+
) => await executeMaterialization(options, childIdentity, input),
|
|
107
|
+
verifyMaterialization: async (
|
|
108
|
+
input: Parameters<NativeEditableArtifactSubprocessPort["verifyMaterialization"]>[0],
|
|
109
|
+
) => await executeVerification(options, childIdentity, input),
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async function executeMaterialization(
|
|
114
|
+
options: NativeEditableArtifactSubprocessOptions,
|
|
115
|
+
identity: IdentityEnvelope,
|
|
116
|
+
input: Parameters<NativeEditableArtifactMaterializerPort["materialize"]>[0],
|
|
117
|
+
): Promise<NativeEditableArtifactMaterializationResult> {
|
|
118
|
+
throwIfAborted(input.signal);
|
|
119
|
+
if (input.job.sourceByteSize > options.maxSourceBytes) {
|
|
120
|
+
throw new EditableArtifactMaterializerPermanentError("source_size_limit");
|
|
121
|
+
}
|
|
122
|
+
const manifest = encodeManifest(input.job, input.normalizedOptions);
|
|
123
|
+
const child = spawnNative(options, [MATERIALIZE_ARGUMENT]);
|
|
124
|
+
const terminate = () => terminateChild(child);
|
|
125
|
+
const timeout = setTimeout(terminate, options.wallTimeoutMs);
|
|
126
|
+
timeout.unref?.();
|
|
127
|
+
const abort = () => terminate();
|
|
128
|
+
input.signal.addEventListener("abort", abort, { once: true });
|
|
129
|
+
|
|
130
|
+
const stderrDone = drainStderr(child, terminate);
|
|
131
|
+
const exit = waitForExit(child);
|
|
132
|
+
const cancellation = createAbortWait(input.signal);
|
|
133
|
+
const inputDone = pumpInput(
|
|
134
|
+
child,
|
|
135
|
+
manifest,
|
|
136
|
+
input.job.sourceByteSize,
|
|
137
|
+
input.snapshot,
|
|
138
|
+
input.signal,
|
|
139
|
+
exit,
|
|
140
|
+
cancellation.promise,
|
|
141
|
+
);
|
|
142
|
+
void inputDone.catch(terminate);
|
|
143
|
+
const reader = new BoundedProcessReader(child, terminate, cancellation.promise);
|
|
144
|
+
|
|
145
|
+
try {
|
|
146
|
+
const magic = await reader.readExactly(OUTPUT_MAGIC.byteLength, input.signal);
|
|
147
|
+
if (equalBytes(magic, ERROR_MAGIC)) {
|
|
148
|
+
throw await readTypedError(reader, input.signal);
|
|
149
|
+
}
|
|
150
|
+
if (!equalBytes(magic, OUTPUT_MAGIC)) {
|
|
151
|
+
throw new Error("native materializer returned an invalid output protocol");
|
|
152
|
+
}
|
|
153
|
+
const header = await reader.readExactly(12, input.signal);
|
|
154
|
+
const headerView = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
155
|
+
const metadataBytes = headerView.getUint32(0, true);
|
|
156
|
+
const outputBytes = safeU64(headerView, 4, "native materializer output length");
|
|
157
|
+
if (metadataBytes < 2 || metadataBytes > MAX_METADATA_BYTES) {
|
|
158
|
+
throw new Error("native materializer metadata exceeds its bound");
|
|
159
|
+
}
|
|
160
|
+
if (
|
|
161
|
+
outputBytes <= 0 ||
|
|
162
|
+
outputBytes > options.maxOutputBytes ||
|
|
163
|
+
outputBytes > identity.maxOutputBytes
|
|
164
|
+
) {
|
|
165
|
+
throw new EditableArtifactMaterializerPermanentError("output_size_limit");
|
|
166
|
+
}
|
|
167
|
+
const metadataText = new TextDecoder("utf-8", { fatal: true }).decode(
|
|
168
|
+
await reader.readExactly(metadataBytes, input.signal),
|
|
169
|
+
);
|
|
170
|
+
const result = decodeOutputMetadata(metadataText, input.job, outputBytes);
|
|
171
|
+
let consumed = false;
|
|
172
|
+
return Object.freeze({
|
|
173
|
+
...result,
|
|
174
|
+
chunks: Object.freeze({
|
|
175
|
+
async *[Symbol.asyncIterator](): AsyncIterableIterator<Uint8Array> {
|
|
176
|
+
if (consumed) throw new Error("native materializer output is single-use");
|
|
177
|
+
consumed = true;
|
|
178
|
+
try {
|
|
179
|
+
yield* reader.streamExactly(outputBytes, input.signal);
|
|
180
|
+
await reader.assertEof(input.signal);
|
|
181
|
+
await inputDone;
|
|
182
|
+
await stderrDone;
|
|
183
|
+
const status = await exit;
|
|
184
|
+
if (status.code !== 0 || status.signal !== null) {
|
|
185
|
+
throw new Error("native materializer exited unsuccessfully");
|
|
186
|
+
}
|
|
187
|
+
} finally {
|
|
188
|
+
cleanup();
|
|
189
|
+
}
|
|
190
|
+
},
|
|
191
|
+
}),
|
|
192
|
+
});
|
|
193
|
+
} catch (error) {
|
|
194
|
+
terminate();
|
|
195
|
+
await Promise.allSettled([inputDone, stderrDone, exit]);
|
|
196
|
+
cleanup();
|
|
197
|
+
throw mapSubprocessFailure(error, input.signal);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function cleanup(): void {
|
|
201
|
+
clearTimeout(timeout);
|
|
202
|
+
input.signal.removeEventListener("abort", abort);
|
|
203
|
+
cancellation.dispose();
|
|
204
|
+
if (child.exitCode === null && child.signalCode === null) terminate();
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
async function executeVerification(
|
|
209
|
+
options: NativeEditableArtifactSubprocessOptions,
|
|
210
|
+
identity: IdentityEnvelope,
|
|
211
|
+
input: Parameters<NativeEditableArtifactSubprocessPort["verifyMaterialization"]>[0],
|
|
212
|
+
): Promise<void> {
|
|
213
|
+
throwIfAborted(input.signal);
|
|
214
|
+
if (
|
|
215
|
+
!isHash(input.expectedSemanticHash) ||
|
|
216
|
+
!Number.isSafeInteger(input.byteSize) ||
|
|
217
|
+
input.byteSize <= 0 ||
|
|
218
|
+
input.byteSize > options.maxOutputBytes ||
|
|
219
|
+
identity.codecVersions[input.codecId] !== input.codecVersion
|
|
220
|
+
) {
|
|
221
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
222
|
+
}
|
|
223
|
+
const manifest = new TextEncoder().encode(
|
|
224
|
+
JSON.stringify({
|
|
225
|
+
codecId: input.codecId,
|
|
226
|
+
codecVersion: input.codecVersion,
|
|
227
|
+
expectedSemanticHash: input.expectedSemanticHash,
|
|
228
|
+
format: input.format,
|
|
229
|
+
protocol: "OGAVJ001",
|
|
230
|
+
}),
|
|
231
|
+
);
|
|
232
|
+
const child = spawnNative(options, [VERIFY_ARGUMENT]);
|
|
233
|
+
const terminate = () => terminateChild(child);
|
|
234
|
+
const timeout = setTimeout(terminate, options.wallTimeoutMs);
|
|
235
|
+
timeout.unref?.();
|
|
236
|
+
const abort = () => terminate();
|
|
237
|
+
input.signal.addEventListener("abort", abort, { once: true });
|
|
238
|
+
const stderrDone = drainStderr(child, terminate);
|
|
239
|
+
const exit = waitForExit(child);
|
|
240
|
+
const cancellation = createAbortWait(input.signal);
|
|
241
|
+
const inputDone = pumpFrame(
|
|
242
|
+
child,
|
|
243
|
+
VERIFY_INPUT_MAGIC,
|
|
244
|
+
manifest,
|
|
245
|
+
input.byteSize,
|
|
246
|
+
input.chunks,
|
|
247
|
+
input.signal,
|
|
248
|
+
exit,
|
|
249
|
+
cancellation.promise,
|
|
250
|
+
"output_verification_failed",
|
|
251
|
+
);
|
|
252
|
+
void inputDone.catch(terminate);
|
|
253
|
+
const reader = new BoundedProcessReader(child, terminate, cancellation.promise);
|
|
254
|
+
try {
|
|
255
|
+
const magic = await reader.readExactly(VERIFY_OUTPUT_MAGIC.byteLength, input.signal);
|
|
256
|
+
if (equalBytes(magic, ERROR_MAGIC)) throw await readTypedError(reader, input.signal);
|
|
257
|
+
if (!equalBytes(magic, VERIFY_OUTPUT_MAGIC)) {
|
|
258
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
259
|
+
}
|
|
260
|
+
const header = await reader.readExactly(12, input.signal);
|
|
261
|
+
const view = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
262
|
+
const metadataBytes = view.getUint32(0, true);
|
|
263
|
+
const payloadBytes = safeU64(view, 4, "verification payload length");
|
|
264
|
+
if (metadataBytes < 2 || metadataBytes > MAX_METADATA_BYTES || payloadBytes !== 0) {
|
|
265
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
266
|
+
}
|
|
267
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(
|
|
268
|
+
await reader.readExactly(metadataBytes, input.signal),
|
|
269
|
+
);
|
|
270
|
+
const value = JSON.parse(text) as unknown;
|
|
271
|
+
if (
|
|
272
|
+
JSON.stringify(value) !== text ||
|
|
273
|
+
typeof value !== "object" ||
|
|
274
|
+
value === null ||
|
|
275
|
+
Array.isArray(value) ||
|
|
276
|
+
Object.keys(value).sort().join("\0") !== "protocol\0semanticHash" ||
|
|
277
|
+
(value as Record<string, unknown>).protocol !== "OGAVR001" ||
|
|
278
|
+
(value as Record<string, unknown>).semanticHash !== input.expectedSemanticHash
|
|
279
|
+
) {
|
|
280
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
281
|
+
}
|
|
282
|
+
await reader.assertEof(input.signal);
|
|
283
|
+
await inputDone;
|
|
284
|
+
await stderrDone;
|
|
285
|
+
const status = await exit;
|
|
286
|
+
if (status.code !== 0 || status.signal !== null) {
|
|
287
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
288
|
+
}
|
|
289
|
+
} catch (error) {
|
|
290
|
+
terminate();
|
|
291
|
+
await Promise.allSettled([inputDone, stderrDone, exit]);
|
|
292
|
+
if (error instanceof EditableArtifactMaterializerPermanentError) throw error;
|
|
293
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
294
|
+
} finally {
|
|
295
|
+
clearTimeout(timeout);
|
|
296
|
+
input.signal.removeEventListener("abort", abort);
|
|
297
|
+
cancellation.dispose();
|
|
298
|
+
if (child.exitCode === null && child.signalCode === null) terminate();
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
async function readTypedError(
|
|
303
|
+
reader: BoundedProcessReader,
|
|
304
|
+
signal: AbortSignal,
|
|
305
|
+
): Promise<EditableArtifactMaterializerPermanentError> {
|
|
306
|
+
const header = await reader.readExactly(12, signal);
|
|
307
|
+
const view = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
308
|
+
const metadataBytes = view.getUint32(0, true);
|
|
309
|
+
const payloadBytes = safeU64(view, 4, "error payload length");
|
|
310
|
+
if (metadataBytes < 2 || metadataBytes > MAX_METADATA_BYTES || payloadBytes !== 0) {
|
|
311
|
+
return new EditableArtifactMaterializerPermanentError("kernel_result_mismatch");
|
|
312
|
+
}
|
|
313
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(
|
|
314
|
+
await reader.readExactly(metadataBytes, signal),
|
|
315
|
+
);
|
|
316
|
+
const value = JSON.parse(text) as unknown;
|
|
317
|
+
const keys =
|
|
318
|
+
typeof value === "object" && value !== null && !Array.isArray(value)
|
|
319
|
+
? Object.keys(value).sort().join("\0")
|
|
320
|
+
: "";
|
|
321
|
+
if (
|
|
322
|
+
JSON.stringify(value) !== text ||
|
|
323
|
+
typeof value !== "object" ||
|
|
324
|
+
value === null ||
|
|
325
|
+
Array.isArray(value) ||
|
|
326
|
+
(keys !== "code\0protocol" && keys !== "code\0protocol\0subcode") ||
|
|
327
|
+
(value as Record<string, unknown>).protocol !== "OGAMERR1"
|
|
328
|
+
) {
|
|
329
|
+
return new EditableArtifactMaterializerPermanentError("kernel_result_mismatch");
|
|
330
|
+
}
|
|
331
|
+
const code = (value as Record<string, unknown>).code;
|
|
332
|
+
const subcode = (value as Record<string, unknown>).subcode;
|
|
333
|
+
if (
|
|
334
|
+
code === "unsupported_semantics" ||
|
|
335
|
+
code === "source_identity_mismatch" ||
|
|
336
|
+
code === "kernel_incompatible" ||
|
|
337
|
+
code === "output_verification_failed"
|
|
338
|
+
) {
|
|
339
|
+
const diagnostic = nativeFailureDiagnostic(code, subcode);
|
|
340
|
+
if (subcode !== undefined && !diagnostic) {
|
|
341
|
+
return new EditableArtifactMaterializerPermanentError("kernel_result_mismatch");
|
|
342
|
+
}
|
|
343
|
+
return new EditableArtifactMaterializerPermanentError(code, diagnostic);
|
|
344
|
+
}
|
|
345
|
+
return new EditableArtifactMaterializerPermanentError("kernel_result_mismatch");
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function nativeFailureDiagnostic(
|
|
349
|
+
code: unknown,
|
|
350
|
+
subcode: unknown,
|
|
351
|
+
): EditableArtifactMaterializerFailureDiagnostic | undefined {
|
|
352
|
+
if (code !== "source_identity_mismatch") return undefined;
|
|
353
|
+
if (subcode === "input_framing" || subcode === "snapshot_open" || subcode === "state_mismatch") {
|
|
354
|
+
return { stage: "native", subcode };
|
|
355
|
+
}
|
|
356
|
+
return undefined;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
async function probeIdentity(
|
|
360
|
+
options: NativeEditableArtifactSubprocessOptions,
|
|
361
|
+
): Promise<IdentityEnvelope> {
|
|
362
|
+
const child = spawnNative(options, [IDENTITY_ARGUMENT]);
|
|
363
|
+
const terminate = () => terminateChild(child);
|
|
364
|
+
const timeout = setTimeout(terminate, options.probeTimeoutMs ?? 5_000);
|
|
365
|
+
timeout.unref?.();
|
|
366
|
+
const stdout = collectBounded(child.stdout, MAX_IDENTITY_BYTES, terminate);
|
|
367
|
+
const stderr = drainStderr(child, terminate);
|
|
368
|
+
try {
|
|
369
|
+
const [bytes, , status] = await Promise.all([stdout, stderr, waitForExit(child)]);
|
|
370
|
+
if (status.code !== 0 || status.signal !== null) {
|
|
371
|
+
throw new Error("native materializer identity probe failed");
|
|
372
|
+
}
|
|
373
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
374
|
+
const parsed = JSON.parse(text) as unknown;
|
|
375
|
+
if (JSON.stringify(parsed) !== text) {
|
|
376
|
+
throw new Error("native materializer identity is not canonical JSON");
|
|
377
|
+
}
|
|
378
|
+
return parsed as IdentityEnvelope;
|
|
379
|
+
} finally {
|
|
380
|
+
clearTimeout(timeout);
|
|
381
|
+
if (child.exitCode === null && child.signalCode === null) terminate();
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
function spawnNative(
|
|
386
|
+
options: NativeEditableArtifactSubprocessOptions,
|
|
387
|
+
args: readonly string[],
|
|
388
|
+
): ChildProcessWithoutNullStreams {
|
|
389
|
+
return options.launcher.spawn({
|
|
390
|
+
executable: options.executable,
|
|
391
|
+
args,
|
|
392
|
+
environment: Object.freeze({
|
|
393
|
+
LANG: "C",
|
|
394
|
+
LC_ALL: "C",
|
|
395
|
+
TZ: "UTC",
|
|
396
|
+
...(options.childEnvironment ?? {}),
|
|
397
|
+
}),
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
async function pumpInput(
|
|
402
|
+
child: ChildProcessWithoutNullStreams,
|
|
403
|
+
manifest: Uint8Array,
|
|
404
|
+
expectedSourceBytes: number,
|
|
405
|
+
source: AsyncIterable<Uint8Array>,
|
|
406
|
+
signal: AbortSignal,
|
|
407
|
+
exit: Promise<{ code: number | null; signal: NodeJS.Signals | null }>,
|
|
408
|
+
cancellation: Promise<never>,
|
|
409
|
+
): Promise<void> {
|
|
410
|
+
await pumpFrame(
|
|
411
|
+
child,
|
|
412
|
+
INPUT_MAGIC,
|
|
413
|
+
manifest,
|
|
414
|
+
expectedSourceBytes,
|
|
415
|
+
source,
|
|
416
|
+
signal,
|
|
417
|
+
exit,
|
|
418
|
+
cancellation,
|
|
419
|
+
"source_identity_mismatch",
|
|
420
|
+
);
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
async function pumpFrame(
|
|
424
|
+
child: ChildProcessWithoutNullStreams,
|
|
425
|
+
magic: Uint8Array,
|
|
426
|
+
manifest: Uint8Array,
|
|
427
|
+
expectedSourceBytes: number,
|
|
428
|
+
source: AsyncIterable<Uint8Array>,
|
|
429
|
+
signal: AbortSignal,
|
|
430
|
+
exit: Promise<{ code: number | null; signal: NodeJS.Signals | null }>,
|
|
431
|
+
cancellation: Promise<never>,
|
|
432
|
+
mismatchCode: "source_identity_mismatch" | "output_verification_failed",
|
|
433
|
+
): Promise<void> {
|
|
434
|
+
const header = new Uint8Array(12);
|
|
435
|
+
const view = new DataView(header.buffer);
|
|
436
|
+
view.setUint32(0, manifest.byteLength, true);
|
|
437
|
+
view.setBigUint64(4, BigInt(expectedSourceBytes), true);
|
|
438
|
+
await write(child, magic, signal, exit, cancellation);
|
|
439
|
+
await write(child, header, signal, exit, cancellation);
|
|
440
|
+
await write(child, manifest, signal, exit, cancellation);
|
|
441
|
+
let sourceBytes = 0;
|
|
442
|
+
for await (const chunk of source) {
|
|
443
|
+
throwIfAborted(signal);
|
|
444
|
+
if (
|
|
445
|
+
!(chunk instanceof Uint8Array) ||
|
|
446
|
+
chunk.byteLength === 0 ||
|
|
447
|
+
chunk.byteLength > 8 * 1024 * 1024
|
|
448
|
+
) {
|
|
449
|
+
throw new EditableArtifactMaterializerPermanentError(mismatchCode);
|
|
450
|
+
}
|
|
451
|
+
sourceBytes += chunk.byteLength;
|
|
452
|
+
if (sourceBytes > expectedSourceBytes) {
|
|
453
|
+
throw new EditableArtifactMaterializerPermanentError(mismatchCode);
|
|
454
|
+
}
|
|
455
|
+
await write(child, chunk, signal, exit, cancellation);
|
|
456
|
+
}
|
|
457
|
+
if (sourceBytes !== expectedSourceBytes) {
|
|
458
|
+
throw new EditableArtifactMaterializerPermanentError(mismatchCode);
|
|
459
|
+
}
|
|
460
|
+
child.stdin.end();
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
async function write(
|
|
464
|
+
child: ChildProcessWithoutNullStreams,
|
|
465
|
+
bytes: Uint8Array,
|
|
466
|
+
signal: AbortSignal,
|
|
467
|
+
exit: Promise<{ code: number | null; signal: NodeJS.Signals | null }>,
|
|
468
|
+
cancellation: Promise<never>,
|
|
469
|
+
): Promise<void> {
|
|
470
|
+
throwIfAborted(signal);
|
|
471
|
+
if (!child.stdin.write(bytes)) {
|
|
472
|
+
await Promise.race([
|
|
473
|
+
once(child.stdin, "drain"),
|
|
474
|
+
cancellation,
|
|
475
|
+
exit.then(() => {
|
|
476
|
+
throw new Error("native materializer closed input early");
|
|
477
|
+
}),
|
|
478
|
+
]);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
class BoundedProcessReader {
|
|
483
|
+
readonly #iterator: AsyncIterator<Buffer | string>;
|
|
484
|
+
#buffer = new Uint8Array(0);
|
|
485
|
+
#done = false;
|
|
486
|
+
|
|
487
|
+
constructor(
|
|
488
|
+
child: ChildProcessWithoutNullStreams,
|
|
489
|
+
private readonly terminate: () => void,
|
|
490
|
+
private readonly cancellation: Promise<never>,
|
|
491
|
+
) {
|
|
492
|
+
this.#iterator = child.stdout[Symbol.asyncIterator]();
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
async readExactly(length: number, signal: AbortSignal): Promise<Uint8Array> {
|
|
496
|
+
if (!Number.isSafeInteger(length) || length < 0) throw new Error("invalid framed length");
|
|
497
|
+
const output = new Uint8Array(length);
|
|
498
|
+
let offset = 0;
|
|
499
|
+
while (offset < length) {
|
|
500
|
+
throwIfAborted(signal);
|
|
501
|
+
if (this.#buffer.byteLength === 0) await this.#fill(signal);
|
|
502
|
+
if (this.#buffer.byteLength === 0) throw new Error("truncated native materializer output");
|
|
503
|
+
const count = Math.min(length - offset, this.#buffer.byteLength);
|
|
504
|
+
output.set(this.#buffer.subarray(0, count), offset);
|
|
505
|
+
this.#buffer = this.#buffer.subarray(count);
|
|
506
|
+
offset += count;
|
|
507
|
+
}
|
|
508
|
+
return output;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
async *streamExactly(length: number, signal: AbortSignal): AsyncIterableIterator<Uint8Array> {
|
|
512
|
+
let remaining = length;
|
|
513
|
+
while (remaining > 0) {
|
|
514
|
+
throwIfAborted(signal);
|
|
515
|
+
const count = Math.min(remaining, STREAM_CHUNK_BYTES);
|
|
516
|
+
const chunk = await this.readExactly(count, signal);
|
|
517
|
+
remaining -= chunk.byteLength;
|
|
518
|
+
yield chunk;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
async assertEof(signal: AbortSignal): Promise<void> {
|
|
523
|
+
throwIfAborted(signal);
|
|
524
|
+
if (this.#buffer.byteLength > 0) throw new Error("trailing native materializer output");
|
|
525
|
+
if (!this.#done) await this.#fill(signal);
|
|
526
|
+
if (this.#buffer.byteLength > 0 || !this.#done) {
|
|
527
|
+
throw new Error("trailing native materializer output");
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
async #fill(signal: AbortSignal): Promise<void> {
|
|
532
|
+
if (this.#done) return;
|
|
533
|
+
throwIfAborted(signal);
|
|
534
|
+
let next: IteratorResult<Buffer | string>;
|
|
535
|
+
try {
|
|
536
|
+
next = await Promise.race([this.#iterator.next(), this.cancellation]);
|
|
537
|
+
} catch (error) {
|
|
538
|
+
this.terminate();
|
|
539
|
+
throw error;
|
|
540
|
+
}
|
|
541
|
+
if (next.done) {
|
|
542
|
+
this.#done = true;
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
545
|
+
if (typeof next.value === "string") throw new Error("native materializer emitted text stream");
|
|
546
|
+
this.#buffer = new Uint8Array(
|
|
547
|
+
next.value.buffer,
|
|
548
|
+
next.value.byteOffset,
|
|
549
|
+
next.value.byteLength,
|
|
550
|
+
).slice();
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
function encodeManifest(
|
|
555
|
+
job: ClaimedEditableArtifactMaterialization,
|
|
556
|
+
normalizedOptions: Uint8Array,
|
|
557
|
+
): Uint8Array {
|
|
558
|
+
const options = new TextDecoder("utf-8", { fatal: true }).decode(normalizedOptions);
|
|
559
|
+
const manifest = {
|
|
560
|
+
protocol: "OGAMJ001",
|
|
561
|
+
artifactId: job.artifactId,
|
|
562
|
+
jobId: job.jobId,
|
|
563
|
+
versionId: job.versionId,
|
|
564
|
+
modality: job.modality,
|
|
565
|
+
inputSnapshotId: job.inputSnapshotId,
|
|
566
|
+
targetHeadSequence: job.targetHeadSequence,
|
|
567
|
+
stateHash: job.stateHash,
|
|
568
|
+
sourceByteSize: job.sourceByteSize,
|
|
569
|
+
sourceContentHash: job.sourceContentHash,
|
|
570
|
+
modelSchemaVersion: job.modelSchemaVersion,
|
|
571
|
+
operationProtocolVersion: job.operationProtocolVersion,
|
|
572
|
+
snapshotProtocolVersion: job.snapshotProtocolVersion,
|
|
573
|
+
format: job.format,
|
|
574
|
+
codecId: job.codecId,
|
|
575
|
+
normalizedOptions: JSON.parse(options) as unknown,
|
|
576
|
+
optionsHash: job.optionsHash,
|
|
577
|
+
codecVersion: job.codecVersion,
|
|
578
|
+
kernelVersion: job.kernelVersion,
|
|
579
|
+
fontRegistryHash: job.fontRegistryHash,
|
|
580
|
+
policyHash: job.policyHash,
|
|
581
|
+
};
|
|
582
|
+
const bytes = new TextEncoder().encode(JSON.stringify(manifest));
|
|
583
|
+
if (bytes.byteLength > MAX_METADATA_BYTES) {
|
|
584
|
+
throw new EditableArtifactMaterializerPermanentError("invalid_job_manifest");
|
|
585
|
+
}
|
|
586
|
+
return bytes;
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
function decodeOutputMetadata(
|
|
590
|
+
text: string,
|
|
591
|
+
job: ClaimedEditableArtifactMaterialization,
|
|
592
|
+
outputBytes: number,
|
|
593
|
+
): Omit<NativeEditableArtifactMaterializationResult, "chunks"> {
|
|
594
|
+
const value = JSON.parse(text) as unknown;
|
|
595
|
+
if (JSON.stringify(value) !== text || typeof value !== "object" || value === null) {
|
|
596
|
+
throw new Error("native materializer metadata is not canonical JSON");
|
|
597
|
+
}
|
|
598
|
+
const record = value as Record<string, unknown>;
|
|
599
|
+
const expectedKeys = [
|
|
600
|
+
"byteSize",
|
|
601
|
+
"codecId",
|
|
602
|
+
"codecVersion",
|
|
603
|
+
"contentHash",
|
|
604
|
+
"fontRegistryHash",
|
|
605
|
+
"format",
|
|
606
|
+
"headSequence",
|
|
607
|
+
"kernelVersion",
|
|
608
|
+
"mimeType",
|
|
609
|
+
"policyHash",
|
|
610
|
+
"protocol",
|
|
611
|
+
"semanticHash",
|
|
612
|
+
"stateHash",
|
|
613
|
+
].sort();
|
|
614
|
+
if (Object.keys(record).sort().join("\0") !== expectedKeys.join("\0")) {
|
|
615
|
+
throw new Error("native materializer metadata fields are invalid");
|
|
616
|
+
}
|
|
617
|
+
if (record.protocol !== "OGAMR001" || record.byteSize !== outputBytes) {
|
|
618
|
+
throw new Error("native materializer output length disagrees with metadata");
|
|
619
|
+
}
|
|
620
|
+
const expectedMimeType = mimeTypeForFormat(job.format);
|
|
621
|
+
if (
|
|
622
|
+
!Number.isSafeInteger(record.headSequence) ||
|
|
623
|
+
record.headSequence !== job.targetHeadSequence ||
|
|
624
|
+
!isHash(record.stateHash) ||
|
|
625
|
+
record.stateHash !== job.stateHash ||
|
|
626
|
+
record.format !== job.format ||
|
|
627
|
+
record.mimeType !== expectedMimeType ||
|
|
628
|
+
!isBoundedVersion(record.codecId) ||
|
|
629
|
+
record.codecId !== job.codecId ||
|
|
630
|
+
!isBoundedVersion(record.codecVersion) ||
|
|
631
|
+
record.codecVersion !== job.codecVersion ||
|
|
632
|
+
!isBoundedKernelVersion(record.kernelVersion) ||
|
|
633
|
+
record.kernelVersion !== job.kernelVersion ||
|
|
634
|
+
!isHash(record.fontRegistryHash) ||
|
|
635
|
+
record.fontRegistryHash !== job.fontRegistryHash ||
|
|
636
|
+
!isHash(record.policyHash) ||
|
|
637
|
+
record.policyHash !== job.policyHash ||
|
|
638
|
+
!isHash(record.contentHash) ||
|
|
639
|
+
!isHash(record.semanticHash)
|
|
640
|
+
) {
|
|
641
|
+
throw new Error("native materializer output metadata is incompatible");
|
|
642
|
+
}
|
|
643
|
+
return Object.freeze({
|
|
644
|
+
headSequence: record.headSequence,
|
|
645
|
+
stateHash: record.stateHash,
|
|
646
|
+
format: job.format,
|
|
647
|
+
mimeType: expectedMimeType,
|
|
648
|
+
codecId: record.codecId,
|
|
649
|
+
codecVersion: record.codecVersion,
|
|
650
|
+
kernelVersion: record.kernelVersion,
|
|
651
|
+
fontRegistryHash: record.fontRegistryHash,
|
|
652
|
+
policyHash: record.policyHash,
|
|
653
|
+
byteSize: outputBytes,
|
|
654
|
+
contentHash: record.contentHash,
|
|
655
|
+
semanticHash: record.semanticHash,
|
|
656
|
+
});
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
function validateOptions(options: NativeEditableArtifactSubprocessOptions): void {
|
|
660
|
+
if (!isAbsolute(options.executable) || options.executable.length > 2_048) {
|
|
661
|
+
throw new TypeError("native materializer executable must be an absolute path");
|
|
662
|
+
}
|
|
663
|
+
for (const [label, value] of [
|
|
664
|
+
["wall timeout", options.wallTimeoutMs],
|
|
665
|
+
["source limit", options.maxSourceBytes],
|
|
666
|
+
["output limit", options.maxOutputBytes],
|
|
667
|
+
] as const) {
|
|
668
|
+
if (!Number.isSafeInteger(value) || value <= 0) throw new TypeError(`${label} is invalid`);
|
|
669
|
+
}
|
|
670
|
+
const launcher = options.launcher.identity;
|
|
671
|
+
const sandboxed =
|
|
672
|
+
launcher.sandboxEnforced === true &&
|
|
673
|
+
launcher.network === "denied" &&
|
|
674
|
+
Number.isSafeInteger(launcher.memoryLimitBytes) &&
|
|
675
|
+
launcher.memoryLimitBytes > 0 &&
|
|
676
|
+
Number.isSafeInteger(launcher.cpuTimeLimitMs) &&
|
|
677
|
+
launcher.cpuTimeLimitMs > 0 &&
|
|
678
|
+
Number.isSafeInteger(launcher.fileDescriptorLimit) &&
|
|
679
|
+
launcher.fileDescriptorLimit > 0 &&
|
|
680
|
+
Number.isSafeInteger(launcher.processLimit) &&
|
|
681
|
+
launcher.processLimit > 0 &&
|
|
682
|
+
Number.isSafeInteger(launcher.fileSizeLimitBytes) &&
|
|
683
|
+
launcher.fileSizeLimitBytes > 0;
|
|
684
|
+
const developmentUnsandboxed =
|
|
685
|
+
options.allowUnsandboxedDevelopment === true &&
|
|
686
|
+
launcher.sandboxEnforced === false &&
|
|
687
|
+
launcher.network === "host" &&
|
|
688
|
+
launcher.memoryLimitBytes === 0 &&
|
|
689
|
+
launcher.cpuTimeLimitMs === 0 &&
|
|
690
|
+
launcher.fileDescriptorLimit === 0 &&
|
|
691
|
+
launcher.processLimit === 0 &&
|
|
692
|
+
launcher.fileSizeLimitBytes === 0;
|
|
693
|
+
if (
|
|
694
|
+
launcher.isolation !== "subprocess" ||
|
|
695
|
+
launcher.officeAutomation !== false ||
|
|
696
|
+
!isBoundedVersion(launcher.platform) ||
|
|
697
|
+
(!sandboxed && !developmentUnsandboxed)
|
|
698
|
+
) {
|
|
699
|
+
throw new TypeError("materializer launcher isolation is not verified");
|
|
700
|
+
}
|
|
701
|
+
if (options.childEnvironment !== undefined) {
|
|
702
|
+
const keys = Object.keys(options.childEnvironment).sort();
|
|
703
|
+
const productionKeys = ["OPENGENI_ARTIFACT_RUNTIME_MANIFEST", "OPENGENI_ARTIFACT_TOOL_ENTRY"];
|
|
704
|
+
const developmentKeys = [
|
|
705
|
+
"NODE_ENV",
|
|
706
|
+
"OPENGENI_ARTIFACT_DEVELOPMENT_RUNTIME_MANIFEST",
|
|
707
|
+
"OPENGENI_ARTIFACT_TOOL_ENTRY",
|
|
708
|
+
];
|
|
709
|
+
const production = keys.join("\0") === productionKeys.join("\0");
|
|
710
|
+
const development = keys.join("\0") === developmentKeys.join("\0");
|
|
711
|
+
if (!production && !development) {
|
|
712
|
+
throw new TypeError("native materializer child environment is not exact");
|
|
713
|
+
}
|
|
714
|
+
if (development !== developmentUnsandboxed) {
|
|
715
|
+
throw new TypeError("native materializer launcher and runtime authority modes differ");
|
|
716
|
+
}
|
|
717
|
+
if (
|
|
718
|
+
development &&
|
|
719
|
+
(options.allowUnsandboxedDevelopment !== true ||
|
|
720
|
+
options.childEnvironment.NODE_ENV !== "development")
|
|
721
|
+
) {
|
|
722
|
+
throw new TypeError("native materializer development child environment is forbidden");
|
|
723
|
+
}
|
|
724
|
+
for (const [name, value] of Object.entries(options.childEnvironment)) {
|
|
725
|
+
if (name === "NODE_ENV") continue;
|
|
726
|
+
if (!isAbsolute(value) || value.length > 2_048 || /[\u0000-\u001f\u007f]/u.test(value)) {
|
|
727
|
+
throw new TypeError("native materializer child environment path is invalid");
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
function validateIdentity(identity: IdentityEnvelope): void {
|
|
734
|
+
const record = identity as unknown as Record<string, unknown>;
|
|
735
|
+
const expectedKeys = [
|
|
736
|
+
"codecVersions",
|
|
737
|
+
"fontRegistryHash",
|
|
738
|
+
"kernelVersion",
|
|
739
|
+
"maxOutputBytes",
|
|
740
|
+
"policyHash",
|
|
741
|
+
"protocol",
|
|
742
|
+
"runtimeKind",
|
|
743
|
+
"runtimeTarget",
|
|
744
|
+
"supportedModelSchemaVersions",
|
|
745
|
+
"supportedOperationProtocolVersions",
|
|
746
|
+
"supportedSnapshotProtocolVersions",
|
|
747
|
+
].sort();
|
|
748
|
+
if (Object.keys(record).sort().join("\0") !== expectedKeys.join("\0")) {
|
|
749
|
+
throw new Error("native materializer identity fields are invalid");
|
|
750
|
+
}
|
|
751
|
+
if (
|
|
752
|
+
identity.protocol !== "OGAMC001" ||
|
|
753
|
+
identity.runtimeKind !== "native" ||
|
|
754
|
+
!Number.isSafeInteger(identity.maxOutputBytes) ||
|
|
755
|
+
identity.maxOutputBytes <= 0
|
|
756
|
+
) {
|
|
757
|
+
throw new Error("native materializer sandbox identity is incompatible");
|
|
758
|
+
}
|
|
759
|
+
if (
|
|
760
|
+
!isBoundedKernelVersion(identity.kernelVersion) ||
|
|
761
|
+
!isBoundedVersion(identity.runtimeTarget) ||
|
|
762
|
+
!isHash(identity.fontRegistryHash) ||
|
|
763
|
+
!isHash(identity.policyHash) ||
|
|
764
|
+
!isCodecRegistry(identity.codecVersions) ||
|
|
765
|
+
!isCanonicalPositiveVersionList(identity.supportedModelSchemaVersions) ||
|
|
766
|
+
!isCanonicalPositiveVersionList(identity.supportedOperationProtocolVersions) ||
|
|
767
|
+
!isCanonicalPositiveVersionList(identity.supportedSnapshotProtocolVersions)
|
|
768
|
+
) {
|
|
769
|
+
throw new Error("native materializer capability identity is incomplete");
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
function compositeIdentity(
|
|
774
|
+
child: IdentityEnvelope,
|
|
775
|
+
options: NativeEditableArtifactSubprocessOptions,
|
|
776
|
+
): NativeEditableArtifactMaterializerIdentity {
|
|
777
|
+
const launcher = options.launcher.identity;
|
|
778
|
+
return Object.freeze({
|
|
779
|
+
kind: "native",
|
|
780
|
+
isolation: launcher.isolation,
|
|
781
|
+
network: launcher.network,
|
|
782
|
+
officeAutomation: launcher.officeAutomation,
|
|
783
|
+
processProtocolVersion: 1,
|
|
784
|
+
sandboxEnforced: launcher.sandboxEnforced,
|
|
785
|
+
memoryLimitBytes: launcher.memoryLimitBytes,
|
|
786
|
+
cpuTimeLimitMs: launcher.cpuTimeLimitMs,
|
|
787
|
+
fileDescriptorLimit: launcher.fileDescriptorLimit,
|
|
788
|
+
processLimit: launcher.processLimit,
|
|
789
|
+
fileSizeLimitBytes: launcher.fileSizeLimitBytes,
|
|
790
|
+
maxOutputBytes: Math.min(child.maxOutputBytes, options.maxOutputBytes),
|
|
791
|
+
kernelVersion: child.kernelVersion,
|
|
792
|
+
codecVersions: child.codecVersions,
|
|
793
|
+
fontRegistryHash: child.fontRegistryHash,
|
|
794
|
+
policyHash: child.policyHash,
|
|
795
|
+
runtimeTarget: child.runtimeTarget,
|
|
796
|
+
supportedModelSchemaVersions: child.supportedModelSchemaVersions,
|
|
797
|
+
supportedOperationProtocolVersions: child.supportedOperationProtocolVersions,
|
|
798
|
+
supportedSnapshotProtocolVersions: child.supportedSnapshotProtocolVersions,
|
|
799
|
+
});
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
function isCodecRegistry(value: unknown): value is Readonly<Record<string, string>> {
|
|
803
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
804
|
+
const entries = Object.entries(value as Record<string, unknown>);
|
|
805
|
+
if (entries.length === 0 || entries.length > 64) return false;
|
|
806
|
+
let previous = "";
|
|
807
|
+
for (const [codecId, version] of entries) {
|
|
808
|
+
if (
|
|
809
|
+
!/^[a-z][a-z0-9._-]{0,127}$/u.test(codecId) ||
|
|
810
|
+
codecId <= previous ||
|
|
811
|
+
!isBoundedVersion(version)
|
|
812
|
+
) {
|
|
813
|
+
return false;
|
|
814
|
+
}
|
|
815
|
+
previous = codecId;
|
|
816
|
+
}
|
|
817
|
+
return true;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
function isCanonicalPositiveVersionList(value: unknown): value is readonly number[] {
|
|
821
|
+
if (!Array.isArray(value) || value.length === 0 || value.length > 64) return false;
|
|
822
|
+
let previous = 0;
|
|
823
|
+
for (const version of value) {
|
|
824
|
+
if (!Number.isSafeInteger(version) || version <= previous) return false;
|
|
825
|
+
previous = version;
|
|
826
|
+
}
|
|
827
|
+
return true;
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
function isBoundedVersion(value: unknown): value is string {
|
|
831
|
+
return (
|
|
832
|
+
typeof value === "string" &&
|
|
833
|
+
value.length >= 1 &&
|
|
834
|
+
value.length <= 128 &&
|
|
835
|
+
/^[\x21-\x7e]+$/u.test(value)
|
|
836
|
+
);
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
function isBoundedKernelVersion(value: unknown): value is string {
|
|
840
|
+
return (
|
|
841
|
+
typeof value === "string" &&
|
|
842
|
+
value.length >= 1 &&
|
|
843
|
+
new TextEncoder().encode(value).byteLength <= EDITABLE_ARTIFACT_KERNEL_VERSION_MAX_BYTES &&
|
|
844
|
+
/^[\x21-\x7e]+$/u.test(value)
|
|
845
|
+
);
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
function isHash(value: unknown): value is string {
|
|
849
|
+
return typeof value === "string" && /^sha256:[0-9a-f]{64}$/u.test(value);
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
async function collectBounded(
|
|
853
|
+
stream: NodeJS.ReadableStream & AsyncIterable<Buffer | string>,
|
|
854
|
+
maximum: number,
|
|
855
|
+
terminate: () => void,
|
|
856
|
+
): Promise<Uint8Array> {
|
|
857
|
+
const chunks: Uint8Array[] = [];
|
|
858
|
+
let total = 0;
|
|
859
|
+
for await (const value of stream) {
|
|
860
|
+
const chunk = typeof value === "string" ? Buffer.from(value) : value;
|
|
861
|
+
total += chunk.byteLength;
|
|
862
|
+
if (total > maximum) {
|
|
863
|
+
terminate();
|
|
864
|
+
throw new Error("native materializer diagnostic exceeded its bound");
|
|
865
|
+
}
|
|
866
|
+
chunks.push(new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength).slice());
|
|
867
|
+
}
|
|
868
|
+
const result = new Uint8Array(total);
|
|
869
|
+
let offset = 0;
|
|
870
|
+
for (const chunk of chunks) {
|
|
871
|
+
result.set(chunk, offset);
|
|
872
|
+
offset += chunk.byteLength;
|
|
873
|
+
}
|
|
874
|
+
return result;
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
async function drainStderr(
|
|
878
|
+
child: ChildProcessWithoutNullStreams,
|
|
879
|
+
terminate: () => void,
|
|
880
|
+
): Promise<void> {
|
|
881
|
+
await collectBounded(child.stderr, MAX_STDERR_BYTES, terminate);
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
function waitForExit(
|
|
885
|
+
child: ChildProcessWithoutNullStreams,
|
|
886
|
+
): Promise<{ code: number | null; signal: NodeJS.Signals | null }> {
|
|
887
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
888
|
+
return Promise.resolve({ code: child.exitCode, signal: child.signalCode });
|
|
889
|
+
}
|
|
890
|
+
return new Promise((resolve, reject) => {
|
|
891
|
+
const onError = (error: Error) => {
|
|
892
|
+
child.removeListener("exit", onExit);
|
|
893
|
+
reject(error);
|
|
894
|
+
};
|
|
895
|
+
const onExit = (code: number | null, signal: NodeJS.Signals | null) => {
|
|
896
|
+
child.removeListener("error", onError);
|
|
897
|
+
resolve({ code, signal });
|
|
898
|
+
};
|
|
899
|
+
child.once("error", onError);
|
|
900
|
+
child.once("exit", onExit);
|
|
901
|
+
});
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
function terminateChild(child: ChildProcessWithoutNullStreams): void {
|
|
905
|
+
if (child.exitCode !== null || child.signalCode !== null) return;
|
|
906
|
+
child.stdin.destroy();
|
|
907
|
+
child.kill("SIGKILL");
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
function createAbortWait(signal: AbortSignal): {
|
|
911
|
+
promise: Promise<never>;
|
|
912
|
+
dispose(): void;
|
|
913
|
+
} {
|
|
914
|
+
let listener: (() => void) | null = null;
|
|
915
|
+
const promise = new Promise<never>((_resolve, reject) => {
|
|
916
|
+
if (signal.aborted) {
|
|
917
|
+
reject(signal.reason ?? abortError());
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
listener = () => reject(signal.reason ?? abortError());
|
|
921
|
+
signal.addEventListener("abort", listener, { once: true });
|
|
922
|
+
});
|
|
923
|
+
// The promise is deliberately shared by every stream race. Attach a handler
|
|
924
|
+
// once so a late abort after another branch failed cannot be unhandled.
|
|
925
|
+
void promise.catch(() => undefined);
|
|
926
|
+
return {
|
|
927
|
+
promise,
|
|
928
|
+
dispose() {
|
|
929
|
+
if (listener) signal.removeEventListener("abort", listener);
|
|
930
|
+
listener = null;
|
|
931
|
+
},
|
|
932
|
+
};
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
function throwIfAborted(signal: AbortSignal): void {
|
|
936
|
+
if (signal.aborted) throw signal.reason ?? abortError();
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
function abortError(): DOMException {
|
|
940
|
+
return new DOMException("Operation aborted", "AbortError");
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
function mapSubprocessFailure(error: unknown, signal: AbortSignal): unknown {
|
|
944
|
+
if (signal.aborted) return signal.reason ?? abortError();
|
|
945
|
+
return error;
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
function safeU64(view: DataView, offset: number, label: string): number {
|
|
949
|
+
const value = view.getBigUint64(offset, true);
|
|
950
|
+
if (value > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error(`${label} exceeds safe integer`);
|
|
951
|
+
return Number(value);
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
function equalBytes(left: Uint8Array, right: Uint8Array): boolean {
|
|
955
|
+
return left.byteLength === right.byteLength && left.every((byte, index) => byte === right[index]);
|
|
956
|
+
}
|