@opengeni/worker-bundle 0.16.21 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +145 -32
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +3 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -7
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +2 -2
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +129 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +2 -2
- package/dist/activities/run-input.d.ts +5 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +11 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -147
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +675 -16460
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +40 -7
- package/dist/sandbox-routing.d.ts +13 -6
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1044 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1905 -559
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +16 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +47 -6
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +52 -60
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +1317 -0
- package/src/activities/rig-verification.ts +567 -77
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +40 -10
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +96 -7
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +77 -13
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +50 -118
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +202 -7
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +331 -157
- package/src/sandbox-routing.ts +49 -39
- package/src/worker-service-lifecycle.ts +82 -12
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +6 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
- package/src/activities/toolspace-token-renewal.ts +0 -57
|
@@ -0,0 +1,2662 @@
|
|
|
1
|
+
// src/editable-artifact-materializer-service.ts
|
|
2
|
+
import { dbSearchPath, getSettings } from "@opengeni/config";
|
|
3
|
+
import {
|
|
4
|
+
createDb,
|
|
5
|
+
dbSql,
|
|
6
|
+
PostgresEditableArtifactMaterializationRepository
|
|
7
|
+
} from "@opengeni/db";
|
|
8
|
+
import { createObservability } from "@opengeni/observability";
|
|
9
|
+
import { createObjectStorage } from "@opengeni/storage";
|
|
10
|
+
import { dirname as dirname2 } from "path";
|
|
11
|
+
|
|
12
|
+
// src/editable-artifact-materializer.ts
|
|
13
|
+
import { createHash } from "crypto";
|
|
14
|
+
import { EDITABLE_ARTIFACT_KERNEL_VERSION_MAX_BYTES } from "@opengeni/contracts/editable-artifacts";
|
|
15
|
+
import { BoundedObjectReadError, BoundedObjectWriteError } from "@opengeni/storage";
|
|
16
|
+
var EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE = "opengeni_artifact_materializer";
|
|
17
|
+
var EDITABLE_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES = 512 * 1024 * 1024;
|
|
18
|
+
var EDITABLE_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES = 512 * 1024 * 1024;
|
|
19
|
+
var EDITABLE_ARTIFACT_MATERIALIZER_MAX_OPTIONS_BYTES = 256 * 1024;
|
|
20
|
+
var EDITABLE_ARTIFACT_MATERIALIZER_MAX_ATTEMPTS = 20;
|
|
21
|
+
var DEFAULT_OPTIONS = Object.freeze({
|
|
22
|
+
batchSize: 8,
|
|
23
|
+
concurrency: 2,
|
|
24
|
+
leaseDurationMs: 12e4,
|
|
25
|
+
leaseRenewIntervalMs: 3e4,
|
|
26
|
+
pollIntervalMs: 1e3,
|
|
27
|
+
claimBackoffMs: 5e3,
|
|
28
|
+
maxAttempts: EDITABLE_ARTIFACT_MATERIALIZER_MAX_ATTEMPTS,
|
|
29
|
+
maxSourceBytes: EDITABLE_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES,
|
|
30
|
+
maxOutputBytes: EDITABLE_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES
|
|
31
|
+
});
|
|
32
|
+
var EditableArtifactMaterializerPermanentError = class extends Error {
|
|
33
|
+
constructor(code, diagnostic) {
|
|
34
|
+
super(`Editable artifact materialization cannot continue: ${code}`);
|
|
35
|
+
this.code = code;
|
|
36
|
+
this.diagnostic = diagnostic;
|
|
37
|
+
this.name = "EditableArtifactMaterializerPermanentError";
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
var EditableArtifactMaterializerLeaseLostError = class extends Error {
|
|
41
|
+
constructor() {
|
|
42
|
+
super("Editable artifact materialization lease was lost");
|
|
43
|
+
this.name = "EditableArtifactMaterializerLeaseLostError";
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
var EditableArtifactMaterializer = class {
|
|
47
|
+
constructor(dependencies, options, securityPolicy = {}) {
|
|
48
|
+
this.dependencies = dependencies;
|
|
49
|
+
this.#options = normalizeOptions(options);
|
|
50
|
+
assertNativeKernelIdentity(
|
|
51
|
+
dependencies.kernel.identity,
|
|
52
|
+
securityPolicy.allowUnsandboxedDevelopment === true
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
#options;
|
|
56
|
+
#activePass = null;
|
|
57
|
+
async run(signal) {
|
|
58
|
+
let claimFailed = false;
|
|
59
|
+
while (!signal.aborted) {
|
|
60
|
+
const summary = await this.dispatchOnce(signal);
|
|
61
|
+
claimFailed = summary.claimFailed;
|
|
62
|
+
if (signal.aborted) break;
|
|
63
|
+
const delay = claimFailed ? this.#options.claimBackoffMs : summary.claimed === 0 ? this.#options.pollIntervalMs : 0;
|
|
64
|
+
if (delay > 0 && !await sleepOrAbort(this.dependencies.scheduler, delay, signal)) break;
|
|
65
|
+
}
|
|
66
|
+
await this.drain();
|
|
67
|
+
}
|
|
68
|
+
/** Concurrent triggers share one bounded claim pass. */
|
|
69
|
+
dispatchOnce(signal) {
|
|
70
|
+
if (this.#activePass) return this.#activePass;
|
|
71
|
+
const pass = this.#dispatchBatch(signal);
|
|
72
|
+
this.#activePass = pass;
|
|
73
|
+
void pass.then(
|
|
74
|
+
() => {
|
|
75
|
+
if (this.#activePass === pass) this.#activePass = null;
|
|
76
|
+
},
|
|
77
|
+
() => {
|
|
78
|
+
if (this.#activePass === pass) this.#activePass = null;
|
|
79
|
+
}
|
|
80
|
+
);
|
|
81
|
+
return pass;
|
|
82
|
+
}
|
|
83
|
+
async drain() {
|
|
84
|
+
await this.#activePass;
|
|
85
|
+
}
|
|
86
|
+
async #dispatchBatch(parentSignal) {
|
|
87
|
+
const summary = mutableSummary();
|
|
88
|
+
if (parentSignal?.aborted) {
|
|
89
|
+
return freezeSummary(summary);
|
|
90
|
+
}
|
|
91
|
+
let jobs;
|
|
92
|
+
try {
|
|
93
|
+
jobs = await this.dependencies.store.claim({
|
|
94
|
+
owner: this.#options.owner,
|
|
95
|
+
leaseDurationMs: this.#options.leaseDurationMs,
|
|
96
|
+
limit: this.#options.batchSize
|
|
97
|
+
});
|
|
98
|
+
if (!Array.isArray(jobs) || jobs.length > this.#options.batchSize) {
|
|
99
|
+
throw new TypeError("Materialization store returned an invalid claim batch");
|
|
100
|
+
}
|
|
101
|
+
} catch (error) {
|
|
102
|
+
summary.claimFailed = true;
|
|
103
|
+
this.#metric("claim_failed");
|
|
104
|
+
this.#warn("Editable artifact materialization claim failed", "claim_failed", error);
|
|
105
|
+
return freezeSummary(summary);
|
|
106
|
+
}
|
|
107
|
+
summary.claimed = jobs.length;
|
|
108
|
+
if (jobs.length > 0) this.#metric("claimed", jobs.length);
|
|
109
|
+
let cursor = 0;
|
|
110
|
+
const worker = async () => {
|
|
111
|
+
for (; ; ) {
|
|
112
|
+
const index = cursor;
|
|
113
|
+
cursor += 1;
|
|
114
|
+
const job = jobs[index];
|
|
115
|
+
if (!job) return;
|
|
116
|
+
await this.#processJob(job, summary, parentSignal);
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
await Promise.all(
|
|
120
|
+
Array.from({ length: Math.min(jobs.length, this.#options.concurrency) }, worker)
|
|
121
|
+
);
|
|
122
|
+
return freezeSummary(summary);
|
|
123
|
+
}
|
|
124
|
+
async #processJob(job, summary, parentSignal) {
|
|
125
|
+
let lease;
|
|
126
|
+
try {
|
|
127
|
+
lease = validateClaimedJob(job, this.#options.owner, this.dependencies.kernel.identity);
|
|
128
|
+
} catch (error) {
|
|
129
|
+
await this.#deadLetterBestEffort(job, "invalid_job_manifest", summary, error);
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
if (job.attemptCount >= this.#options.maxAttempts) {
|
|
133
|
+
await this.#settlePermanent(lease, "retry_exhausted", summary, null);
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
const operationAbort = new AbortController();
|
|
137
|
+
const renewalAbort = new AbortController();
|
|
138
|
+
const combined = combineAbortSignals(parentSignal, operationAbort.signal);
|
|
139
|
+
let leaseLost = false;
|
|
140
|
+
const renewal = this.#renewLease(lease, renewalAbort.signal, operationAbort, () => {
|
|
141
|
+
leaseLost = true;
|
|
142
|
+
});
|
|
143
|
+
let source = null;
|
|
144
|
+
try {
|
|
145
|
+
throwIfAborted(combined.signal);
|
|
146
|
+
assertJobKernelCompatibility(job, this.dependencies.kernel.identity);
|
|
147
|
+
const optionsBytes = canonicalOptionsBytes(job.normalizedOptions, job.optionsHash);
|
|
148
|
+
if (job.sourceByteSize > this.#options.maxSourceBytes) {
|
|
149
|
+
throw new EditableArtifactMaterializerPermanentError("source_size_limit");
|
|
150
|
+
}
|
|
151
|
+
try {
|
|
152
|
+
source = await this.dependencies.sourceReader.open({
|
|
153
|
+
opaqueReference: job.sourceObjectReference,
|
|
154
|
+
maxBytes: this.#options.maxSourceBytes,
|
|
155
|
+
expectedByteSize: job.sourceByteSize,
|
|
156
|
+
signal: combined.signal
|
|
157
|
+
});
|
|
158
|
+
} catch (error) {
|
|
159
|
+
throw diagnoseSourceReaderFailure(error, "open");
|
|
160
|
+
}
|
|
161
|
+
if (source.contentType !== void 0 && source.contentType !== job.sourceMimeType) {
|
|
162
|
+
throw new EditableArtifactMaterializerPermanentError("source_identity_mismatch", {
|
|
163
|
+
stage: "source_reader",
|
|
164
|
+
subcode: "content_type"
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
let sourceChunks;
|
|
168
|
+
try {
|
|
169
|
+
sourceChunks = source.chunks({ signal: combined.signal });
|
|
170
|
+
} catch (error) {
|
|
171
|
+
throw diagnoseSourceReaderFailure(error, "stream_identity");
|
|
172
|
+
}
|
|
173
|
+
const trackedSource = trackExactStream(
|
|
174
|
+
sourceChunks,
|
|
175
|
+
job.sourceByteSize,
|
|
176
|
+
job.sourceContentHash,
|
|
177
|
+
"source_identity_mismatch",
|
|
178
|
+
{ stage: "source_reader", subcode: "stream_identity" }
|
|
179
|
+
);
|
|
180
|
+
let output;
|
|
181
|
+
try {
|
|
182
|
+
output = await this.dependencies.kernel.materialize({
|
|
183
|
+
job,
|
|
184
|
+
normalizedOptions: optionsBytes,
|
|
185
|
+
snapshot: trackedSource.chunks,
|
|
186
|
+
signal: combined.signal
|
|
187
|
+
});
|
|
188
|
+
await trackedSource.done;
|
|
189
|
+
} catch (error) {
|
|
190
|
+
void trackedSource.done.catch(() => void 0);
|
|
191
|
+
throw error;
|
|
192
|
+
}
|
|
193
|
+
try {
|
|
194
|
+
await source.assertUnchanged(combined.signal);
|
|
195
|
+
} catch (error) {
|
|
196
|
+
throw diagnoseSourceReaderFailure(error, "revalidate");
|
|
197
|
+
}
|
|
198
|
+
validateKernelResult(job, output);
|
|
199
|
+
const written = await this.dependencies.outputWriter.write({
|
|
200
|
+
chunks: output.chunks,
|
|
201
|
+
contentType: output.mimeType,
|
|
202
|
+
maxBytes: this.#options.maxOutputBytes,
|
|
203
|
+
...output.byteSize !== void 0 ? { expectedByteSize: output.byteSize } : {},
|
|
204
|
+
...output.contentHash !== void 0 ? { expectedContentHash: output.contentHash } : {},
|
|
205
|
+
signal: combined.signal
|
|
206
|
+
});
|
|
207
|
+
let verified;
|
|
208
|
+
try {
|
|
209
|
+
verified = await this.dependencies.outputVerifier.verify({
|
|
210
|
+
objectReference: written.opaqueReference,
|
|
211
|
+
expectedByteSize: written.byteSize,
|
|
212
|
+
expectedContentHash: written.contentHash,
|
|
213
|
+
expectedMimeType: output.mimeType,
|
|
214
|
+
format: job.format,
|
|
215
|
+
codecId: job.codecId,
|
|
216
|
+
codecVersion: job.codecVersion,
|
|
217
|
+
expectedSemanticHash: output.semanticHash,
|
|
218
|
+
maxBytes: this.#options.maxOutputBytes,
|
|
219
|
+
signal: combined.signal
|
|
220
|
+
});
|
|
221
|
+
} catch (error) {
|
|
222
|
+
if (error instanceof BoundedObjectReadError) {
|
|
223
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
224
|
+
}
|
|
225
|
+
throw error;
|
|
226
|
+
}
|
|
227
|
+
validateIndependentVerification(job, written, verified);
|
|
228
|
+
renewalAbort.abort();
|
|
229
|
+
await renewal;
|
|
230
|
+
if (leaseLost) throw new EditableArtifactMaterializerLeaseLostError();
|
|
231
|
+
throwIfAborted(combined.signal);
|
|
232
|
+
await this.dependencies.store.renew({
|
|
233
|
+
...lease,
|
|
234
|
+
leaseDurationMs: this.#options.leaseDurationMs
|
|
235
|
+
});
|
|
236
|
+
const ids = deterministicResultIds(job, written.contentHash);
|
|
237
|
+
await this.dependencies.store.succeed({
|
|
238
|
+
...lease,
|
|
239
|
+
...ids,
|
|
240
|
+
objectReference: written.opaqueReference,
|
|
241
|
+
byteSize: written.byteSize,
|
|
242
|
+
contentHash: written.contentHash,
|
|
243
|
+
mimeType: output.mimeType,
|
|
244
|
+
verifiedAt: this.dependencies.clock.now().toISOString()
|
|
245
|
+
});
|
|
246
|
+
summary.succeeded += 1;
|
|
247
|
+
this.#metric("succeeded");
|
|
248
|
+
} catch (error) {
|
|
249
|
+
renewalAbort.abort();
|
|
250
|
+
await renewal;
|
|
251
|
+
if (leaseLost || isLeaseFence(error)) {
|
|
252
|
+
summary.leaseLost += 1;
|
|
253
|
+
this.#metric("lease_lost");
|
|
254
|
+
this.#warn("Editable artifact materialization lease was lost", "lease_lost", error);
|
|
255
|
+
} else if (combined.signal.aborted || isAbortError(error)) {
|
|
256
|
+
summary.cancelled += 1;
|
|
257
|
+
this.#metric("cancelled");
|
|
258
|
+
} else {
|
|
259
|
+
const permanent = permanentDependencyFailure(error);
|
|
260
|
+
if (permanent) {
|
|
261
|
+
await this.#settlePermanent(lease, permanent, summary, error);
|
|
262
|
+
} else {
|
|
263
|
+
summary.retryDeferred += 1;
|
|
264
|
+
this.#metric("retry_deferred");
|
|
265
|
+
this.#warn(
|
|
266
|
+
"Editable artifact materialization will retry after lease expiry",
|
|
267
|
+
"retry_deferred",
|
|
268
|
+
error
|
|
269
|
+
);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
} finally {
|
|
273
|
+
renewalAbort.abort();
|
|
274
|
+
combined.dispose();
|
|
275
|
+
if (source) await source.close();
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
async #renewLease(lease, signal, operationAbort, onLeaseLost) {
|
|
279
|
+
while (await sleepOrAbort(this.dependencies.scheduler, this.#options.leaseRenewIntervalMs, signal)) {
|
|
280
|
+
try {
|
|
281
|
+
await this.dependencies.store.renew({
|
|
282
|
+
...lease,
|
|
283
|
+
leaseDurationMs: this.#options.leaseDurationMs
|
|
284
|
+
});
|
|
285
|
+
} catch (error) {
|
|
286
|
+
onLeaseLost();
|
|
287
|
+
operationAbort.abort(error);
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
async #settlePermanent(lease, code, summary, cause) {
|
|
293
|
+
try {
|
|
294
|
+
await this.dependencies.store.renew({
|
|
295
|
+
...lease,
|
|
296
|
+
leaseDurationMs: this.#options.leaseDurationMs
|
|
297
|
+
});
|
|
298
|
+
await this.dependencies.store.fail({ ...lease, errorCode: code });
|
|
299
|
+
summary.deadLettered += 1;
|
|
300
|
+
this.#metric("dead_lettered");
|
|
301
|
+
if (cause) this.#warn("Editable artifact materialization was dead-lettered", code, cause);
|
|
302
|
+
} catch (error) {
|
|
303
|
+
summary.leaseLost += 1;
|
|
304
|
+
this.#metric("lease_lost");
|
|
305
|
+
this.#warn("Editable artifact dead-letter settlement was fenced", "lease_lost", error);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
async #deadLetterBestEffort(job, code, summary, cause) {
|
|
309
|
+
try {
|
|
310
|
+
const lease = rawLease(job, this.#options.owner);
|
|
311
|
+
await this.#settlePermanent(lease, code, summary, cause);
|
|
312
|
+
} catch (error) {
|
|
313
|
+
summary.leaseLost += 1;
|
|
314
|
+
this.#metric("lease_lost");
|
|
315
|
+
this.#warn("Invalid materialization claim could not be settled", "lease_lost", error);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
#metric(outcome, count = 1) {
|
|
319
|
+
try {
|
|
320
|
+
this.dependencies.metrics?.increment(outcome, count);
|
|
321
|
+
} catch {
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
#warn(message, code, error) {
|
|
325
|
+
try {
|
|
326
|
+
this.dependencies.logger?.warn(message, {
|
|
327
|
+
code,
|
|
328
|
+
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
329
|
+
...error instanceof EditableArtifactMaterializerPermanentError && error.diagnostic ? {
|
|
330
|
+
failureStage: error.diagnostic.stage,
|
|
331
|
+
failureSubcode: error.diagnostic.subcode
|
|
332
|
+
} : {}
|
|
333
|
+
});
|
|
334
|
+
} catch {
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
};
|
|
338
|
+
function createConfiguredEditableArtifactMaterializer(input) {
|
|
339
|
+
if (!input.enabled) return null;
|
|
340
|
+
if (input.databaseRole !== EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE) {
|
|
341
|
+
throw new Error("Editable artifact materializer requires its dedicated database role");
|
|
342
|
+
}
|
|
343
|
+
if (!input.objectStorageConfigured) {
|
|
344
|
+
throw new Error("Editable artifact materializer requires immutable object storage");
|
|
345
|
+
}
|
|
346
|
+
const dependencies = input.dependencies;
|
|
347
|
+
if (!dependencies?.store || !dependencies.sourceReader || !dependencies.outputWriter || !dependencies.outputVerifier || !dependencies.kernel || !dependencies.scheduler || !dependencies.clock) {
|
|
348
|
+
throw new Error("Editable artifact materializer dependencies are incomplete");
|
|
349
|
+
}
|
|
350
|
+
assertNativeKernelIdentity(
|
|
351
|
+
dependencies.kernel.identity,
|
|
352
|
+
input.allowUnsandboxedDevelopment === true
|
|
353
|
+
);
|
|
354
|
+
if (!input.options) throw new Error("Editable artifact materializer options are required");
|
|
355
|
+
return new EditableArtifactMaterializer(
|
|
356
|
+
dependencies,
|
|
357
|
+
input.options,
|
|
358
|
+
input.allowUnsandboxedDevelopment === void 0 ? {} : { allowUnsandboxedDevelopment: input.allowUnsandboxedDevelopment }
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
var systemEditableArtifactMaterializerScheduler = Object.freeze({
|
|
362
|
+
sleep(milliseconds, signal) {
|
|
363
|
+
return new Promise((resolve, reject) => {
|
|
364
|
+
if (signal.aborted) {
|
|
365
|
+
reject(signal.reason ?? abortError());
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
const complete = () => {
|
|
369
|
+
signal.removeEventListener("abort", abort);
|
|
370
|
+
resolve();
|
|
371
|
+
};
|
|
372
|
+
const timer = setTimeout(complete, milliseconds);
|
|
373
|
+
timer.unref?.();
|
|
374
|
+
const abort = () => {
|
|
375
|
+
clearTimeout(timer);
|
|
376
|
+
signal.removeEventListener("abort", abort);
|
|
377
|
+
reject(signal.reason ?? abortError());
|
|
378
|
+
};
|
|
379
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
});
|
|
383
|
+
var systemEditableArtifactMaterializerClock = Object.freeze({ now: () => /* @__PURE__ */ new Date() });
|
|
384
|
+
function normalizeOptions(options) {
|
|
385
|
+
const normalized = Object.freeze({
|
|
386
|
+
owner: validateOwner(options.owner),
|
|
387
|
+
batchSize: options.batchSize ?? DEFAULT_OPTIONS.batchSize,
|
|
388
|
+
concurrency: options.concurrency ?? DEFAULT_OPTIONS.concurrency,
|
|
389
|
+
leaseDurationMs: options.leaseDurationMs ?? DEFAULT_OPTIONS.leaseDurationMs,
|
|
390
|
+
leaseRenewIntervalMs: options.leaseRenewIntervalMs ?? DEFAULT_OPTIONS.leaseRenewIntervalMs,
|
|
391
|
+
pollIntervalMs: options.pollIntervalMs ?? DEFAULT_OPTIONS.pollIntervalMs,
|
|
392
|
+
claimBackoffMs: options.claimBackoffMs ?? DEFAULT_OPTIONS.claimBackoffMs,
|
|
393
|
+
maxAttempts: options.maxAttempts ?? DEFAULT_OPTIONS.maxAttempts,
|
|
394
|
+
maxSourceBytes: options.maxSourceBytes ?? DEFAULT_OPTIONS.maxSourceBytes,
|
|
395
|
+
maxOutputBytes: options.maxOutputBytes ?? DEFAULT_OPTIONS.maxOutputBytes
|
|
396
|
+
});
|
|
397
|
+
for (const [label, value] of Object.entries(normalized)) {
|
|
398
|
+
if (label === "owner") continue;
|
|
399
|
+
positiveInteger(value, label);
|
|
400
|
+
}
|
|
401
|
+
if (normalized.batchSize > 64) throw new TypeError("materializer batch size exceeds 64");
|
|
402
|
+
if (normalized.concurrency > normalized.batchSize) {
|
|
403
|
+
throw new TypeError("materializer concurrency exceeds its batch size");
|
|
404
|
+
}
|
|
405
|
+
if (normalized.leaseDurationMs < 1e4 || normalized.leaseDurationMs > 864e5) {
|
|
406
|
+
throw new TypeError("materializer lease duration must be 10 seconds to 24 hours");
|
|
407
|
+
}
|
|
408
|
+
if (normalized.leaseRenewIntervalMs * 2 >= normalized.leaseDurationMs) {
|
|
409
|
+
throw new TypeError("materializer renewal interval must be less than half its lease");
|
|
410
|
+
}
|
|
411
|
+
if (normalized.maxAttempts > 1e3) throw new TypeError("materializer attempts exceed 1000");
|
|
412
|
+
if (normalized.maxSourceBytes > EDITABLE_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES) {
|
|
413
|
+
throw new TypeError("materializer source limit exceeds the protocol maximum");
|
|
414
|
+
}
|
|
415
|
+
if (normalized.maxOutputBytes > EDITABLE_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES) {
|
|
416
|
+
throw new TypeError("materializer output limit exceeds the protocol maximum");
|
|
417
|
+
}
|
|
418
|
+
return normalized;
|
|
419
|
+
}
|
|
420
|
+
function validateClaimedJob(job, owner, kernel) {
|
|
421
|
+
const lease = rawLease(job, owner);
|
|
422
|
+
if (job.leaseOwner !== owner) throw new TypeError("claimed job belongs to another owner");
|
|
423
|
+
positiveInteger(job.targetHeadSequence + 1, "target head sequence + 1");
|
|
424
|
+
positiveInteger(job.sourceByteSize, "source byte size");
|
|
425
|
+
positiveInteger(job.modelSchemaVersion, "model schema version");
|
|
426
|
+
positiveInteger(job.operationProtocolVersion, "operation protocol version");
|
|
427
|
+
positiveInteger(job.snapshotProtocolVersion, "snapshot protocol version");
|
|
428
|
+
positiveInteger(job.attemptCount, "attempt count");
|
|
429
|
+
stableId(job.artifactId, "artifact id");
|
|
430
|
+
stableId(job.jobId, "job id");
|
|
431
|
+
stableId(job.inputSnapshotId, "input snapshot id");
|
|
432
|
+
if (job.versionId !== null) stableId(job.versionId, "version id");
|
|
433
|
+
canonicalHash(job.stateHash, "state hash");
|
|
434
|
+
canonicalHash(job.sourceContentHash, "source content hash");
|
|
435
|
+
canonicalHash(job.optionsHash, "options hash");
|
|
436
|
+
canonicalHash(job.fontRegistryHash, "font registry hash");
|
|
437
|
+
canonicalHash(job.policyHash, "policy hash");
|
|
438
|
+
boundedText(job.sourceObjectReference, "source object reference", 2048);
|
|
439
|
+
boundedText(job.sourceMimeType, "source mime type", 256);
|
|
440
|
+
boundedText(job.codecId, "codec id", 128);
|
|
441
|
+
boundedText(job.codecVersion, "codec version", 128);
|
|
442
|
+
boundedText(job.kernelVersion, "kernel version", EDITABLE_ARTIFACT_KERNEL_VERSION_MAX_BYTES);
|
|
443
|
+
validateFormatForModality(job.modality, job.format);
|
|
444
|
+
assertJobKernelCompatibility(job, kernel);
|
|
445
|
+
return lease;
|
|
446
|
+
}
|
|
447
|
+
function rawLease(job, owner) {
|
|
448
|
+
boundedText(job.scope?.accountId, "account id", 128);
|
|
449
|
+
boundedText(job.scope?.workspaceId, "workspace id", 128);
|
|
450
|
+
stableId(job.artifactId, "artifact id");
|
|
451
|
+
stableId(job.jobId, "job id");
|
|
452
|
+
positiveInteger(job.attemptCount, "attempt count");
|
|
453
|
+
validateOwner(owner);
|
|
454
|
+
return Object.freeze({
|
|
455
|
+
scope: Object.freeze({ ...job.scope }),
|
|
456
|
+
artifactId: job.artifactId,
|
|
457
|
+
jobId: job.jobId,
|
|
458
|
+
owner,
|
|
459
|
+
attemptCount: job.attemptCount
|
|
460
|
+
});
|
|
461
|
+
}
|
|
462
|
+
function assertNativeKernelIdentity(identity, allowUnsandboxedDevelopment = false) {
|
|
463
|
+
if (identity.kind !== "native" || identity.isolation !== "subprocess" || identity.officeAutomation !== false) {
|
|
464
|
+
throw new Error("Editable artifact materializer requires an isolated native kernel");
|
|
465
|
+
}
|
|
466
|
+
if (identity.processProtocolVersion !== 1) {
|
|
467
|
+
throw new Error("Editable artifact materializer subprocess protocol is not enforced");
|
|
468
|
+
}
|
|
469
|
+
if (identity.sandboxEnforced) {
|
|
470
|
+
if (identity.network !== "denied") {
|
|
471
|
+
throw new Error("Editable artifact materializer sandbox network policy is invalid");
|
|
472
|
+
}
|
|
473
|
+
positiveInteger(identity.memoryLimitBytes, "kernel memory limit");
|
|
474
|
+
positiveInteger(identity.cpuTimeLimitMs, "kernel CPU time limit");
|
|
475
|
+
positiveInteger(identity.fileDescriptorLimit, "kernel file descriptor limit");
|
|
476
|
+
positiveInteger(identity.processLimit, "kernel process limit");
|
|
477
|
+
positiveInteger(identity.fileSizeLimitBytes, "kernel file size limit");
|
|
478
|
+
} else if (!allowUnsandboxedDevelopment || identity.network !== "host" || identity.memoryLimitBytes !== 0 || identity.cpuTimeLimitMs !== 0 || identity.fileDescriptorLimit !== 0 || identity.processLimit !== 0 || identity.fileSizeLimitBytes !== 0) {
|
|
479
|
+
throw new Error("Editable artifact materializer subprocess protocol is not enforced");
|
|
480
|
+
}
|
|
481
|
+
positiveInteger(identity.maxOutputBytes, "kernel output limit");
|
|
482
|
+
boundedText(identity.kernelVersion, "kernel version", EDITABLE_ARTIFACT_KERNEL_VERSION_MAX_BYTES);
|
|
483
|
+
boundedText(identity.runtimeTarget, "kernel runtime target", 128);
|
|
484
|
+
canonicalHash(identity.fontRegistryHash, "font registry hash");
|
|
485
|
+
canonicalHash(identity.policyHash, "policy hash");
|
|
486
|
+
if (!identity.codecVersions || typeof identity.codecVersions !== "object") {
|
|
487
|
+
throw new Error("Editable artifact materializer codec registry is unavailable");
|
|
488
|
+
}
|
|
489
|
+
for (const [codecId, version] of Object.entries(identity.codecVersions)) {
|
|
490
|
+
boundedText(codecId, "codec id", 128);
|
|
491
|
+
boundedText(version, "codec version", 128);
|
|
492
|
+
}
|
|
493
|
+
for (const [label, values] of [
|
|
494
|
+
["model schema versions", identity.supportedModelSchemaVersions],
|
|
495
|
+
["operation protocol versions", identity.supportedOperationProtocolVersions],
|
|
496
|
+
["snapshot protocol versions", identity.supportedSnapshotProtocolVersions]
|
|
497
|
+
]) {
|
|
498
|
+
if (!Array.isArray(values) || values.length === 0 || values.length > 64) {
|
|
499
|
+
throw new Error(`Editable artifact kernel ${label} are invalid`);
|
|
500
|
+
}
|
|
501
|
+
const unique = /* @__PURE__ */ new Set();
|
|
502
|
+
for (const value of values) {
|
|
503
|
+
positiveInteger(value, label);
|
|
504
|
+
if (unique.has(value)) throw new Error(`Editable artifact kernel ${label} repeat`);
|
|
505
|
+
unique.add(value);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
function assertJobKernelCompatibility(job, identity) {
|
|
510
|
+
const compatible = identity.kernelVersion === job.kernelVersion && identity.codecVersions[job.codecId] === job.codecVersion && identity.fontRegistryHash === job.fontRegistryHash && identity.policyHash === job.policyHash && identity.supportedModelSchemaVersions.includes(job.modelSchemaVersion) && identity.supportedOperationProtocolVersions.includes(job.operationProtocolVersion) && identity.supportedSnapshotProtocolVersions.includes(job.snapshotProtocolVersion);
|
|
511
|
+
if (!compatible) throw new EditableArtifactMaterializerPermanentError("kernel_incompatible");
|
|
512
|
+
}
|
|
513
|
+
function validateKernelResult(job, result) {
|
|
514
|
+
if (result.headSequence !== job.targetHeadSequence || result.stateHash !== job.stateHash || result.format !== job.format || result.mimeType !== mimeTypeForFormat(job.format) || result.codecId !== job.codecId || result.codecVersion !== job.codecVersion || result.kernelVersion !== job.kernelVersion || result.fontRegistryHash !== job.fontRegistryHash || result.policyHash !== job.policyHash || !result.chunks || typeof result.chunks[Symbol.asyncIterator] !== "function") {
|
|
515
|
+
throw new EditableArtifactMaterializerPermanentError("kernel_result_mismatch");
|
|
516
|
+
}
|
|
517
|
+
if (result.byteSize !== void 0) positiveInteger(result.byteSize, "kernel output bytes");
|
|
518
|
+
if (result.contentHash !== void 0) canonicalHash(result.contentHash, "kernel output hash");
|
|
519
|
+
canonicalHash(result.semanticHash, "kernel semantic hash");
|
|
520
|
+
}
|
|
521
|
+
function validateIndependentVerification(job, written, verified) {
|
|
522
|
+
if (verified.objectReference !== written.opaqueReference || verified.byteSize !== written.byteSize || verified.contentHash !== written.contentHash || verified.mimeType !== written.contentType || verified.format !== job.format || verified.codecId !== job.codecId || verified.codecVersion !== job.codecVersion) {
|
|
523
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
function canonicalOptionsBytes(value, expectedHash) {
|
|
527
|
+
if (typeof value !== "string") {
|
|
528
|
+
throw new EditableArtifactMaterializerPermanentError("invalid_job_manifest");
|
|
529
|
+
}
|
|
530
|
+
const bytes = new TextEncoder().encode(value);
|
|
531
|
+
if (bytes.byteLength > EDITABLE_ARTIFACT_MATERIALIZER_MAX_OPTIONS_BYTES) {
|
|
532
|
+
throw new EditableArtifactMaterializerPermanentError("invalid_job_manifest");
|
|
533
|
+
}
|
|
534
|
+
try {
|
|
535
|
+
const parsed = JSON.parse(value);
|
|
536
|
+
if (JSON.stringify(parsed) !== value) {
|
|
537
|
+
throw new Error("non-canonical options");
|
|
538
|
+
}
|
|
539
|
+
} catch {
|
|
540
|
+
throw new EditableArtifactMaterializerPermanentError("invalid_job_manifest");
|
|
541
|
+
}
|
|
542
|
+
if (hashBytes(bytes) !== expectedHash) {
|
|
543
|
+
throw new EditableArtifactMaterializerPermanentError("invalid_job_manifest");
|
|
544
|
+
}
|
|
545
|
+
return bytes;
|
|
546
|
+
}
|
|
547
|
+
function trackExactStream(source, expectedByteSize, expectedContentHash, code, diagnostic) {
|
|
548
|
+
let settle;
|
|
549
|
+
let reject;
|
|
550
|
+
const done = new Promise((resolve, rejectPromise) => {
|
|
551
|
+
settle = resolve;
|
|
552
|
+
reject = rejectPromise;
|
|
553
|
+
});
|
|
554
|
+
let claimed = false;
|
|
555
|
+
const chunks = Object.freeze({
|
|
556
|
+
async *[Symbol.asyncIterator]() {
|
|
557
|
+
if (claimed) {
|
|
558
|
+
const error = new EditableArtifactMaterializerPermanentError(code, diagnostic);
|
|
559
|
+
reject(error);
|
|
560
|
+
throw error;
|
|
561
|
+
}
|
|
562
|
+
claimed = true;
|
|
563
|
+
const digest = createHash("sha256");
|
|
564
|
+
let byteSize = 0;
|
|
565
|
+
try {
|
|
566
|
+
for await (const chunk of source) {
|
|
567
|
+
if (!(chunk instanceof Uint8Array) || chunk.byteLength === 0) {
|
|
568
|
+
throw new EditableArtifactMaterializerPermanentError(code, diagnostic);
|
|
569
|
+
}
|
|
570
|
+
byteSize += chunk.byteLength;
|
|
571
|
+
if (byteSize > expectedByteSize) {
|
|
572
|
+
throw new EditableArtifactMaterializerPermanentError(code, diagnostic);
|
|
573
|
+
}
|
|
574
|
+
digest.update(chunk);
|
|
575
|
+
yield chunk.slice();
|
|
576
|
+
}
|
|
577
|
+
if (byteSize !== expectedByteSize) {
|
|
578
|
+
throw new EditableArtifactMaterializerPermanentError(code, diagnostic);
|
|
579
|
+
}
|
|
580
|
+
const contentHash = `sha256:${digest.digest("hex")}`;
|
|
581
|
+
if (contentHash !== expectedContentHash) {
|
|
582
|
+
throw new EditableArtifactMaterializerPermanentError(code, diagnostic);
|
|
583
|
+
}
|
|
584
|
+
settle();
|
|
585
|
+
} catch (error) {
|
|
586
|
+
const diagnosed = diagnoseSourceReaderFailure(error, "stream_identity");
|
|
587
|
+
reject(diagnosed);
|
|
588
|
+
throw diagnosed;
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
});
|
|
592
|
+
return { chunks, done };
|
|
593
|
+
}
|
|
594
|
+
function diagnoseSourceReaderFailure(error, subcode) {
|
|
595
|
+
const code = permanentDependencyFailure(error);
|
|
596
|
+
return code ? new EditableArtifactMaterializerPermanentError(code, { stage: "source_reader", subcode }) : error;
|
|
597
|
+
}
|
|
598
|
+
function deterministicResultIds(job, contentHash) {
|
|
599
|
+
return {
|
|
600
|
+
resultId: deterministicStableId("result", job, contentHash),
|
|
601
|
+
blobRefId: deterministicStableId("blob", job, contentHash)
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
function deterministicStableId(kind, job, contentHash) {
|
|
605
|
+
const digest = createHash("sha256");
|
|
606
|
+
digest.update(`OpenGeni editable artifact materialization ${kind}\0v1\0`, "utf8");
|
|
607
|
+
for (const value of [
|
|
608
|
+
job.scope.accountId,
|
|
609
|
+
job.scope.workspaceId,
|
|
610
|
+
job.artifactId,
|
|
611
|
+
job.jobId,
|
|
612
|
+
contentHash
|
|
613
|
+
]) {
|
|
614
|
+
const bytes = Buffer.from(value, "utf8");
|
|
615
|
+
const length = Buffer.allocUnsafe(4);
|
|
616
|
+
length.writeUInt32BE(bytes.byteLength);
|
|
617
|
+
digest.update(length);
|
|
618
|
+
digest.update(bytes);
|
|
619
|
+
}
|
|
620
|
+
const id = digest.digest("hex").slice(0, 32);
|
|
621
|
+
return /^0+$/u.test(id) ? `1${id.slice(1)}` : id;
|
|
622
|
+
}
|
|
623
|
+
function hashBytes(bytes) {
|
|
624
|
+
return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
625
|
+
}
|
|
626
|
+
function validateFormatForModality(modality, format) {
|
|
627
|
+
const allowed = {
|
|
628
|
+
spreadsheet: ["xlsx", "pdf", "png", "webp"],
|
|
629
|
+
presentation: ["pptx", "pdf", "png", "webp"],
|
|
630
|
+
document: ["docx", "pdf", "png", "webp"]
|
|
631
|
+
};
|
|
632
|
+
if (!allowed[modality]?.includes(format)) {
|
|
633
|
+
throw new TypeError("materialization format does not match artifact modality");
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
function mimeTypeForFormat(format) {
|
|
637
|
+
switch (format) {
|
|
638
|
+
case "xlsx":
|
|
639
|
+
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
|
640
|
+
case "pptx":
|
|
641
|
+
return "application/vnd.openxmlformats-officedocument.presentationml.presentation";
|
|
642
|
+
case "docx":
|
|
643
|
+
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
|
|
644
|
+
case "pdf":
|
|
645
|
+
return "application/pdf";
|
|
646
|
+
case "png":
|
|
647
|
+
return "image/png";
|
|
648
|
+
case "webp":
|
|
649
|
+
return "image/webp";
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
function validateOwner(value) {
|
|
653
|
+
if (typeof value !== "string" || value.length < 1 || value.length > 256 || value.trim() !== value || !/^[A-Za-z0-9._:-]+$/u.test(value)) {
|
|
654
|
+
throw new TypeError("materializer owner is invalid");
|
|
655
|
+
}
|
|
656
|
+
return value;
|
|
657
|
+
}
|
|
658
|
+
function stableId(value, label) {
|
|
659
|
+
if (typeof value !== "string" || !/^[0-9a-f]{32}$/u.test(value) || /^0+$/u.test(value)) {
|
|
660
|
+
throw new TypeError(`${label} is invalid`);
|
|
661
|
+
}
|
|
662
|
+
return value;
|
|
663
|
+
}
|
|
664
|
+
function canonicalHash(value, label) {
|
|
665
|
+
if (typeof value !== "string" || !/^sha256:[0-9a-f]{64}$/u.test(value)) {
|
|
666
|
+
throw new TypeError(`${label} is invalid`);
|
|
667
|
+
}
|
|
668
|
+
return value;
|
|
669
|
+
}
|
|
670
|
+
function boundedText(value, label, maxBytes) {
|
|
671
|
+
if (typeof value !== "string" || value.length === 0 || new TextEncoder().encode(value).byteLength > maxBytes || value.trim() !== value || /[\u0000-\u001f\u007f]/u.test(value)) {
|
|
672
|
+
throw new TypeError(`${label} is invalid`);
|
|
673
|
+
}
|
|
674
|
+
return value;
|
|
675
|
+
}
|
|
676
|
+
function positiveInteger(value, label) {
|
|
677
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) {
|
|
678
|
+
throw new TypeError(`${label} must be a positive safe integer`);
|
|
679
|
+
}
|
|
680
|
+
return value;
|
|
681
|
+
}
|
|
682
|
+
function isLeaseFence(error) {
|
|
683
|
+
return error instanceof EditableArtifactMaterializerLeaseLostError || typeof error === "object" && error !== null && "code" in error && error.code === "lease_fenced";
|
|
684
|
+
}
|
|
685
|
+
function permanentDependencyFailure(error) {
|
|
686
|
+
if (error instanceof EditableArtifactMaterializerPermanentError) return error.code;
|
|
687
|
+
if (error instanceof BoundedObjectReadError) {
|
|
688
|
+
if (error.code === "size_limit") return "source_size_limit";
|
|
689
|
+
if (error.code === "object_changed" || error.code === "truncated" || error.code === "object_missing" || error.code === "invalid_request") {
|
|
690
|
+
return "source_identity_mismatch";
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
if (error instanceof BoundedObjectWriteError) {
|
|
694
|
+
if (error.code === "size_limit") return "output_size_limit";
|
|
695
|
+
if (error.code === "content_hash_mismatch" || error.code === "readback_mismatch" || error.code === "truncated" || error.code === "invalid_request") {
|
|
696
|
+
return "output_verification_failed";
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
return null;
|
|
700
|
+
}
|
|
701
|
+
function isAbortError(error) {
|
|
702
|
+
return error instanceof DOMException ? error.name === "AbortError" : error instanceof Error && error.name === "AbortError";
|
|
703
|
+
}
|
|
704
|
+
function throwIfAborted(signal) {
|
|
705
|
+
if (signal.aborted) throw signal.reason ?? abortError();
|
|
706
|
+
}
|
|
707
|
+
function abortError() {
|
|
708
|
+
return new DOMException("Operation aborted", "AbortError");
|
|
709
|
+
}
|
|
710
|
+
function combineAbortSignals(first, second) {
|
|
711
|
+
const controller = new AbortController();
|
|
712
|
+
const sources = first ? [first, second] : [second];
|
|
713
|
+
const listeners = [];
|
|
714
|
+
for (const source of sources) {
|
|
715
|
+
const listener = () => controller.abort(source.reason);
|
|
716
|
+
if (source.aborted) listener();
|
|
717
|
+
else {
|
|
718
|
+
source.addEventListener("abort", listener, { once: true });
|
|
719
|
+
listeners.push([source, listener]);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
return {
|
|
723
|
+
signal: controller.signal,
|
|
724
|
+
dispose() {
|
|
725
|
+
for (const [source, listener] of listeners) {
|
|
726
|
+
source.removeEventListener("abort", listener);
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
async function sleepOrAbort(scheduler, milliseconds, signal) {
|
|
732
|
+
try {
|
|
733
|
+
await scheduler.sleep(milliseconds, signal);
|
|
734
|
+
return true;
|
|
735
|
+
} catch (error) {
|
|
736
|
+
if (signal.aborted || isAbortError(error)) return false;
|
|
737
|
+
throw error;
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
function mutableSummary() {
|
|
741
|
+
return {
|
|
742
|
+
claimed: 0,
|
|
743
|
+
succeeded: 0,
|
|
744
|
+
deadLettered: 0,
|
|
745
|
+
retryDeferred: 0,
|
|
746
|
+
leaseLost: 0,
|
|
747
|
+
cancelled: 0,
|
|
748
|
+
claimFailed: false
|
|
749
|
+
};
|
|
750
|
+
}
|
|
751
|
+
function freezeSummary(summary) {
|
|
752
|
+
return Object.freeze({ ...summary });
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
// src/editable-artifact-materializer-storage.ts
|
|
756
|
+
import { createHash as createHash2 } from "crypto";
|
|
757
|
+
import { chmod, mkdtemp, open, rm } from "fs/promises";
|
|
758
|
+
import { tmpdir } from "os";
|
|
759
|
+
import { join } from "path";
|
|
760
|
+
import {
|
|
761
|
+
BoundedObjectReadError as BoundedObjectReadError2,
|
|
762
|
+
createObjectStorageBoundedPorts
|
|
763
|
+
} from "@opengeni/storage";
|
|
764
|
+
var SHA256 = /^sha256:[0-9a-f]{64}$/u;
|
|
765
|
+
var SNAPSHOT_KEY_PREFIX = "editable-artifacts/snapshots/sha256/";
|
|
766
|
+
var MATERIALIZATION_KEY_PREFIX = "editable-artifacts/materializations/v1/sha256/";
|
|
767
|
+
var SEMANTIC_VERIFIER_CHUNK_BYTES = 1024 * 1024;
|
|
768
|
+
var ZIP_EOCD_SEARCH_BYTES = 65557;
|
|
769
|
+
var MAX_ZIP_ENTRY_NAME_BYTES = 4096;
|
|
770
|
+
var MAX_ZIP_TOTAL_NAME_BYTES = 16 * 1024 * 1024;
|
|
771
|
+
function createEditableArtifactObjectStorageReader(storage) {
|
|
772
|
+
return createObjectStorageBoundedPorts(storage, { keyPrefix: SNAPSHOT_KEY_PREFIX }).read;
|
|
773
|
+
}
|
|
774
|
+
function createEditableArtifactObjectStorageWriter(storage) {
|
|
775
|
+
return createObjectStorageBoundedPorts(storage, {
|
|
776
|
+
keyPrefix: MATERIALIZATION_KEY_PREFIX
|
|
777
|
+
}).write;
|
|
778
|
+
}
|
|
779
|
+
function createEditableArtifactObjectStorageVerifier(storage, semanticVerifier) {
|
|
780
|
+
const read = createObjectStorageBoundedPorts(storage, {
|
|
781
|
+
keyPrefix: MATERIALIZATION_KEY_PREFIX
|
|
782
|
+
}).read;
|
|
783
|
+
return Object.freeze({
|
|
784
|
+
async verify(input) {
|
|
785
|
+
validateReference(input.objectReference);
|
|
786
|
+
validateLimit(input.maxBytes);
|
|
787
|
+
if (!Number.isSafeInteger(input.expectedByteSize) || input.expectedByteSize <= 0 || input.expectedByteSize > input.maxBytes || !SHA256.test(input.expectedContentHash)) {
|
|
788
|
+
throw new BoundedObjectReadError2("invalid_request");
|
|
789
|
+
}
|
|
790
|
+
throwIfAborted2(input.signal);
|
|
791
|
+
const stagingDirectory = await mkdtemp(join(tmpdir(), "opengeni-artifact-verify-"));
|
|
792
|
+
const stagingPath = join(stagingDirectory, "payload");
|
|
793
|
+
let stagingHandle = null;
|
|
794
|
+
try {
|
|
795
|
+
await chmod(stagingDirectory, 448);
|
|
796
|
+
stagingHandle = await open(stagingPath, "wx", 384);
|
|
797
|
+
const reader = await read.open({
|
|
798
|
+
opaqueReference: input.objectReference,
|
|
799
|
+
maxBytes: input.maxBytes,
|
|
800
|
+
expectedByteSize: input.expectedByteSize,
|
|
801
|
+
signal: input.signal
|
|
802
|
+
});
|
|
803
|
+
try {
|
|
804
|
+
if (reader.contentType !== void 0 && reader.contentType !== input.expectedMimeType) {
|
|
805
|
+
throw new BoundedObjectReadError2("object_changed");
|
|
806
|
+
}
|
|
807
|
+
const digest = createHash2("sha256");
|
|
808
|
+
let offset = 0;
|
|
809
|
+
for await (const chunk of reader.chunks({ signal: input.signal })) {
|
|
810
|
+
if (offset + chunk.byteLength > input.expectedByteSize) {
|
|
811
|
+
throw new BoundedObjectReadError2("object_changed");
|
|
812
|
+
}
|
|
813
|
+
await writeAll(stagingHandle, chunk, offset);
|
|
814
|
+
digest.update(chunk);
|
|
815
|
+
offset += chunk.byteLength;
|
|
816
|
+
}
|
|
817
|
+
if (offset !== input.expectedByteSize || `sha256:${digest.digest("hex")}` !== input.expectedContentHash) {
|
|
818
|
+
throw new BoundedObjectReadError2("object_changed");
|
|
819
|
+
}
|
|
820
|
+
await reader.assertUnchanged(input.signal);
|
|
821
|
+
} finally {
|
|
822
|
+
await reader.close();
|
|
823
|
+
}
|
|
824
|
+
await stagingHandle.close();
|
|
825
|
+
stagingHandle = null;
|
|
826
|
+
await verifyFormat(
|
|
827
|
+
stagingPath,
|
|
828
|
+
input.expectedByteSize,
|
|
829
|
+
input.format,
|
|
830
|
+
input.expectedMimeType,
|
|
831
|
+
input.signal
|
|
832
|
+
);
|
|
833
|
+
try {
|
|
834
|
+
await semanticVerifier.verify({
|
|
835
|
+
format: input.format,
|
|
836
|
+
codecId: input.codecId,
|
|
837
|
+
codecVersion: input.codecVersion,
|
|
838
|
+
expectedSemanticHash: input.expectedSemanticHash,
|
|
839
|
+
byteSize: input.expectedByteSize,
|
|
840
|
+
chunks: fileChunks(stagingPath, input.expectedByteSize, input.signal),
|
|
841
|
+
signal: input.signal
|
|
842
|
+
});
|
|
843
|
+
} catch (error) {
|
|
844
|
+
throwIfAborted2(input.signal);
|
|
845
|
+
preserveKernelCompatibilityFailure(error);
|
|
846
|
+
throw new BoundedObjectReadError2("object_changed");
|
|
847
|
+
}
|
|
848
|
+
throwIfAborted2(input.signal);
|
|
849
|
+
return Object.freeze({
|
|
850
|
+
objectReference: input.objectReference,
|
|
851
|
+
byteSize: input.expectedByteSize,
|
|
852
|
+
contentHash: input.expectedContentHash,
|
|
853
|
+
mimeType: input.expectedMimeType,
|
|
854
|
+
format: input.format,
|
|
855
|
+
codecId: input.codecId,
|
|
856
|
+
codecVersion: input.codecVersion
|
|
857
|
+
});
|
|
858
|
+
} finally {
|
|
859
|
+
if (stagingHandle) await stagingHandle.close().catch(() => void 0);
|
|
860
|
+
await rm(stagingDirectory, { recursive: true, force: true });
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
});
|
|
864
|
+
}
|
|
865
|
+
async function verifyFormat(path, byteSize, format, mimeType, signal) {
|
|
866
|
+
if (mimeType !== mimeTypeForFormat(format)) {
|
|
867
|
+
throw new BoundedObjectReadError2("object_changed");
|
|
868
|
+
}
|
|
869
|
+
const handle = await open(path, "r");
|
|
870
|
+
try {
|
|
871
|
+
throwIfAborted2(signal);
|
|
872
|
+
switch (format) {
|
|
873
|
+
case "xlsx":
|
|
874
|
+
await assertOfficePackage(handle, byteSize, "xl/workbook.xml", signal);
|
|
875
|
+
break;
|
|
876
|
+
case "pptx":
|
|
877
|
+
await assertOfficePackage(handle, byteSize, "ppt/presentation.xml", signal);
|
|
878
|
+
break;
|
|
879
|
+
case "docx":
|
|
880
|
+
await assertOfficePackage(handle, byteSize, "word/document.xml", signal);
|
|
881
|
+
break;
|
|
882
|
+
case "pdf": {
|
|
883
|
+
const start = await readExact(handle, 0, 5);
|
|
884
|
+
const end = await readExact(handle, byteSize - 5, 5);
|
|
885
|
+
if (!startsWithAscii(start, "%PDF-") || !startsWithAscii(end, "%%EOF")) {
|
|
886
|
+
throw new Error("invalid PDF framing");
|
|
887
|
+
}
|
|
888
|
+
break;
|
|
889
|
+
}
|
|
890
|
+
case "png":
|
|
891
|
+
if (!equalPrefix(
|
|
892
|
+
await readExact(handle, 0, 8),
|
|
893
|
+
[137, 80, 78, 71, 13, 10, 26, 10]
|
|
894
|
+
)) {
|
|
895
|
+
throw new Error("invalid PNG signature");
|
|
896
|
+
}
|
|
897
|
+
break;
|
|
898
|
+
case "webp": {
|
|
899
|
+
const header = await readExact(handle, 0, 12);
|
|
900
|
+
if (!startsWithAscii(header, "RIFF") || !asciiAt(header, 8, "WEBP")) {
|
|
901
|
+
throw new Error("invalid WebP signature");
|
|
902
|
+
}
|
|
903
|
+
break;
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
} catch {
|
|
907
|
+
throwIfAborted2(signal);
|
|
908
|
+
throw new BoundedObjectReadError2("object_changed");
|
|
909
|
+
} finally {
|
|
910
|
+
await handle.close();
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
async function assertOfficePackage(handle, byteSize, requiredPart, signal) {
|
|
914
|
+
if (!equalPrefix(await readExact(handle, 0, 4), [80, 75, 3, 4])) {
|
|
915
|
+
throw new Error("invalid ZIP");
|
|
916
|
+
}
|
|
917
|
+
const tailSize = Math.min(byteSize, ZIP_EOCD_SEARCH_BYTES);
|
|
918
|
+
const tail = await readExact(handle, byteSize - tailSize, tailSize);
|
|
919
|
+
const eocdInTail = findEndOfCentralDirectory(tail);
|
|
920
|
+
const eocd = byteSize - tailSize + eocdInTail;
|
|
921
|
+
const view = new DataView(tail.buffer, tail.byteOffset, tail.byteLength);
|
|
922
|
+
const disk = view.getUint16(eocdInTail + 4, true);
|
|
923
|
+
const centralDisk = view.getUint16(eocdInTail + 6, true);
|
|
924
|
+
const diskEntries = view.getUint16(eocdInTail + 8, true);
|
|
925
|
+
const entries = view.getUint16(eocdInTail + 10, true);
|
|
926
|
+
const centralSize = view.getUint32(eocdInTail + 12, true);
|
|
927
|
+
const centralOffset = view.getUint32(eocdInTail + 16, true);
|
|
928
|
+
const commentLength = view.getUint16(eocdInTail + 20, true);
|
|
929
|
+
if (disk !== 0 || centralDisk !== 0 || diskEntries !== entries || entries === 0 || centralSize === 4294967295 || centralOffset === 4294967295 || eocd + 22 + commentLength !== byteSize || centralOffset + centralSize !== eocd || centralOffset > byteSize) {
|
|
930
|
+
throw new Error("invalid ZIP central directory");
|
|
931
|
+
}
|
|
932
|
+
const names = /* @__PURE__ */ new Set();
|
|
933
|
+
let offset = centralOffset;
|
|
934
|
+
let totalNameBytes = 0;
|
|
935
|
+
for (let index = 0; index < entries; index += 1) {
|
|
936
|
+
throwIfAborted2(signal);
|
|
937
|
+
if (offset + 46 > eocd) {
|
|
938
|
+
throw new Error("invalid ZIP entry");
|
|
939
|
+
}
|
|
940
|
+
const header = await readExact(handle, offset, 46);
|
|
941
|
+
const headerView = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
942
|
+
if (headerView.getUint32(0, true) !== 33639248) throw new Error("invalid ZIP entry");
|
|
943
|
+
if ((headerView.getUint16(8, true) & 1) !== 0) throw new Error("encrypted ZIP entry");
|
|
944
|
+
const nameLength = headerView.getUint16(28, true);
|
|
945
|
+
const extraLength = headerView.getUint16(30, true);
|
|
946
|
+
const commentBytes = headerView.getUint16(32, true);
|
|
947
|
+
const next = offset + 46 + nameLength + extraLength + commentBytes;
|
|
948
|
+
totalNameBytes += nameLength;
|
|
949
|
+
if (nameLength === 0 || nameLength > MAX_ZIP_ENTRY_NAME_BYTES || totalNameBytes > MAX_ZIP_TOTAL_NAME_BYTES || next > eocd) {
|
|
950
|
+
throw new Error("truncated ZIP entry");
|
|
951
|
+
}
|
|
952
|
+
const name = new TextDecoder("utf-8", { fatal: true }).decode(
|
|
953
|
+
await readExact(handle, offset + 46, nameLength)
|
|
954
|
+
);
|
|
955
|
+
if (!isSafeZipEntryName(name) || names.has(name)) {
|
|
956
|
+
throw new Error("unsafe ZIP entry name");
|
|
957
|
+
}
|
|
958
|
+
names.add(name);
|
|
959
|
+
offset = next;
|
|
960
|
+
}
|
|
961
|
+
if (offset !== eocd || !names.has("[Content_Types].xml") || !names.has(requiredPart)) {
|
|
962
|
+
throw new Error("Office package is missing required parts");
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
function isSafeZipEntryName(name) {
|
|
966
|
+
if (name.startsWith("/") || name.includes("\\")) return false;
|
|
967
|
+
const segments = name.split("/");
|
|
968
|
+
if (segments.at(-1) === "") segments.pop();
|
|
969
|
+
return segments.length > 0 && segments.every((segment) => segment.length > 0 && segment !== "." && segment !== "..");
|
|
970
|
+
}
|
|
971
|
+
function findEndOfCentralDirectory(bytes) {
|
|
972
|
+
const minimum = Math.max(0, bytes.byteLength - 65557);
|
|
973
|
+
for (let offset = bytes.byteLength - 22; offset >= minimum; offset -= 1) {
|
|
974
|
+
if (bytes[offset] === 80 && bytes[offset + 1] === 75 && bytes[offset + 2] === 5 && bytes[offset + 3] === 6)
|
|
975
|
+
return offset;
|
|
976
|
+
}
|
|
977
|
+
throw new Error("missing ZIP EOCD");
|
|
978
|
+
}
|
|
979
|
+
async function* fileChunks(path, byteSize, signal) {
|
|
980
|
+
const handle = await open(path, "r");
|
|
981
|
+
try {
|
|
982
|
+
let offset = 0;
|
|
983
|
+
const buffer = new Uint8Array(SEMANTIC_VERIFIER_CHUNK_BYTES);
|
|
984
|
+
while (offset < byteSize) {
|
|
985
|
+
throwIfAborted2(signal);
|
|
986
|
+
const length = Math.min(buffer.byteLength, byteSize - offset);
|
|
987
|
+
const { bytesRead } = await handle.read(buffer, 0, length, offset);
|
|
988
|
+
if (bytesRead !== length) throw new BoundedObjectReadError2("object_changed");
|
|
989
|
+
offset += bytesRead;
|
|
990
|
+
yield buffer.slice(0, bytesRead);
|
|
991
|
+
}
|
|
992
|
+
} finally {
|
|
993
|
+
await handle.close();
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
async function writeAll(handle, bytes, position) {
|
|
997
|
+
let offset = 0;
|
|
998
|
+
while (offset < bytes.byteLength) {
|
|
999
|
+
const { bytesWritten } = await handle.write(
|
|
1000
|
+
bytes,
|
|
1001
|
+
offset,
|
|
1002
|
+
bytes.byteLength - offset,
|
|
1003
|
+
position + offset
|
|
1004
|
+
);
|
|
1005
|
+
if (bytesWritten <= 0) throw new BoundedObjectReadError2("object_changed");
|
|
1006
|
+
offset += bytesWritten;
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
async function readExact(handle, position, length) {
|
|
1010
|
+
if (!Number.isSafeInteger(position) || position < 0 || !Number.isSafeInteger(length) || length < 0) {
|
|
1011
|
+
throw new Error("invalid file range");
|
|
1012
|
+
}
|
|
1013
|
+
const result = new Uint8Array(length);
|
|
1014
|
+
let offset = 0;
|
|
1015
|
+
while (offset < length) {
|
|
1016
|
+
const { bytesRead } = await handle.read(result, offset, length - offset, position + offset);
|
|
1017
|
+
if (bytesRead <= 0) throw new Error("truncated file range");
|
|
1018
|
+
offset += bytesRead;
|
|
1019
|
+
}
|
|
1020
|
+
return result;
|
|
1021
|
+
}
|
|
1022
|
+
function validateReference(value) {
|
|
1023
|
+
if (typeof value !== "string" || value.length < 1 || value.length > 2048 || value.trim() !== value || /[\u0000-\u001f\u007f]/u.test(value))
|
|
1024
|
+
throw new BoundedObjectReadError2("invalid_request");
|
|
1025
|
+
}
|
|
1026
|
+
function validateLimit(value) {
|
|
1027
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
1028
|
+
throw new BoundedObjectReadError2("invalid_request");
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
function throwIfAborted2(signal) {
|
|
1032
|
+
if (signal?.aborted) throw new BoundedObjectReadError2("aborted");
|
|
1033
|
+
}
|
|
1034
|
+
function preserveKernelCompatibilityFailure(error) {
|
|
1035
|
+
if (error instanceof EditableArtifactMaterializerPermanentError && error.code === "kernel_incompatible") {
|
|
1036
|
+
throw error;
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
function startsWithAscii(bytes, value) {
|
|
1040
|
+
return asciiAt(bytes, 0, value);
|
|
1041
|
+
}
|
|
1042
|
+
function asciiAt(bytes, offset, value) {
|
|
1043
|
+
if (offset < 0 || offset + value.length > bytes.byteLength) return false;
|
|
1044
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
1045
|
+
if (bytes[offset + index] !== value.charCodeAt(index)) return false;
|
|
1046
|
+
}
|
|
1047
|
+
return true;
|
|
1048
|
+
}
|
|
1049
|
+
function equalPrefix(bytes, expected) {
|
|
1050
|
+
return expected.every((value, index) => bytes[index] === value);
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
// src/editable-artifact-materializer-subprocess.ts
|
|
1054
|
+
import { once } from "events";
|
|
1055
|
+
import { isAbsolute } from "path";
|
|
1056
|
+
import { EDITABLE_ARTIFACT_KERNEL_VERSION_MAX_BYTES as EDITABLE_ARTIFACT_KERNEL_VERSION_MAX_BYTES2 } from "@opengeni/contracts/editable-artifacts";
|
|
1057
|
+
var INPUT_MAGIC = new TextEncoder().encode("OGAMI001");
|
|
1058
|
+
var VERIFY_INPUT_MAGIC = new TextEncoder().encode("OGAVI001");
|
|
1059
|
+
var OUTPUT_MAGIC = new TextEncoder().encode("OGAMO001");
|
|
1060
|
+
var VERIFY_OUTPUT_MAGIC = new TextEncoder().encode("OGAVO001");
|
|
1061
|
+
var ERROR_MAGIC = new TextEncoder().encode("OGAME001");
|
|
1062
|
+
var IDENTITY_ARGUMENT = "--opengeni-materializer-identity-v1";
|
|
1063
|
+
var MATERIALIZE_ARGUMENT = "--opengeni-materialize-v1";
|
|
1064
|
+
var VERIFY_ARGUMENT = "--opengeni-verify-materialization-v1";
|
|
1065
|
+
var MAX_IDENTITY_BYTES = 64 * 1024;
|
|
1066
|
+
var MAX_METADATA_BYTES = 256 * 1024;
|
|
1067
|
+
var MAX_STDERR_BYTES = 64 * 1024;
|
|
1068
|
+
var STREAM_CHUNK_BYTES = 1024 * 1024;
|
|
1069
|
+
async function createNativeEditableArtifactSubprocessPort(options) {
|
|
1070
|
+
validateOptions(options);
|
|
1071
|
+
const childIdentity = await probeIdentity(options);
|
|
1072
|
+
validateIdentity(childIdentity);
|
|
1073
|
+
const identity = compositeIdentity(childIdentity, options);
|
|
1074
|
+
return Object.freeze({
|
|
1075
|
+
identity,
|
|
1076
|
+
materialize: async (input) => await executeMaterialization(options, childIdentity, input),
|
|
1077
|
+
verifyMaterialization: async (input) => await executeVerification(options, childIdentity, input)
|
|
1078
|
+
});
|
|
1079
|
+
}
|
|
1080
|
+
async function executeMaterialization(options, identity, input) {
|
|
1081
|
+
throwIfAborted3(input.signal);
|
|
1082
|
+
if (input.job.sourceByteSize > options.maxSourceBytes) {
|
|
1083
|
+
throw new EditableArtifactMaterializerPermanentError("source_size_limit");
|
|
1084
|
+
}
|
|
1085
|
+
const manifest = encodeManifest(input.job, input.normalizedOptions);
|
|
1086
|
+
const child = spawnNative(options, [MATERIALIZE_ARGUMENT]);
|
|
1087
|
+
const terminate = () => terminateChild(child);
|
|
1088
|
+
const timeout = setTimeout(terminate, options.wallTimeoutMs);
|
|
1089
|
+
timeout.unref?.();
|
|
1090
|
+
const abort = () => terminate();
|
|
1091
|
+
input.signal.addEventListener("abort", abort, { once: true });
|
|
1092
|
+
const stderrDone = drainStderr(child, terminate);
|
|
1093
|
+
const exit = waitForExit(child);
|
|
1094
|
+
const cancellation = createAbortWait(input.signal);
|
|
1095
|
+
const inputDone = pumpInput(
|
|
1096
|
+
child,
|
|
1097
|
+
manifest,
|
|
1098
|
+
input.job.sourceByteSize,
|
|
1099
|
+
input.snapshot,
|
|
1100
|
+
input.signal,
|
|
1101
|
+
exit,
|
|
1102
|
+
cancellation.promise
|
|
1103
|
+
);
|
|
1104
|
+
void inputDone.catch(terminate);
|
|
1105
|
+
const reader = new BoundedProcessReader(child, terminate, cancellation.promise);
|
|
1106
|
+
try {
|
|
1107
|
+
const magic = await reader.readExactly(OUTPUT_MAGIC.byteLength, input.signal);
|
|
1108
|
+
if (equalBytes(magic, ERROR_MAGIC)) {
|
|
1109
|
+
throw await readTypedError(reader, input.signal);
|
|
1110
|
+
}
|
|
1111
|
+
if (!equalBytes(magic, OUTPUT_MAGIC)) {
|
|
1112
|
+
throw new Error("native materializer returned an invalid output protocol");
|
|
1113
|
+
}
|
|
1114
|
+
const header = await reader.readExactly(12, input.signal);
|
|
1115
|
+
const headerView = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
1116
|
+
const metadataBytes = headerView.getUint32(0, true);
|
|
1117
|
+
const outputBytes = safeU64(headerView, 4, "native materializer output length");
|
|
1118
|
+
if (metadataBytes < 2 || metadataBytes > MAX_METADATA_BYTES) {
|
|
1119
|
+
throw new Error("native materializer metadata exceeds its bound");
|
|
1120
|
+
}
|
|
1121
|
+
if (outputBytes <= 0 || outputBytes > options.maxOutputBytes || outputBytes > identity.maxOutputBytes) {
|
|
1122
|
+
throw new EditableArtifactMaterializerPermanentError("output_size_limit");
|
|
1123
|
+
}
|
|
1124
|
+
const metadataText = new TextDecoder("utf-8", { fatal: true }).decode(
|
|
1125
|
+
await reader.readExactly(metadataBytes, input.signal)
|
|
1126
|
+
);
|
|
1127
|
+
const result = decodeOutputMetadata(metadataText, input.job, outputBytes);
|
|
1128
|
+
let consumed = false;
|
|
1129
|
+
return Object.freeze({
|
|
1130
|
+
...result,
|
|
1131
|
+
chunks: Object.freeze({
|
|
1132
|
+
async *[Symbol.asyncIterator]() {
|
|
1133
|
+
if (consumed) throw new Error("native materializer output is single-use");
|
|
1134
|
+
consumed = true;
|
|
1135
|
+
try {
|
|
1136
|
+
yield* reader.streamExactly(outputBytes, input.signal);
|
|
1137
|
+
await reader.assertEof(input.signal);
|
|
1138
|
+
await inputDone;
|
|
1139
|
+
await stderrDone;
|
|
1140
|
+
const status = await exit;
|
|
1141
|
+
if (status.code !== 0 || status.signal !== null) {
|
|
1142
|
+
throw new Error("native materializer exited unsuccessfully");
|
|
1143
|
+
}
|
|
1144
|
+
} finally {
|
|
1145
|
+
cleanup();
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
})
|
|
1149
|
+
});
|
|
1150
|
+
} catch (error) {
|
|
1151
|
+
terminate();
|
|
1152
|
+
await Promise.allSettled([inputDone, stderrDone, exit]);
|
|
1153
|
+
cleanup();
|
|
1154
|
+
throw mapSubprocessFailure(error, input.signal);
|
|
1155
|
+
}
|
|
1156
|
+
function cleanup() {
|
|
1157
|
+
clearTimeout(timeout);
|
|
1158
|
+
input.signal.removeEventListener("abort", abort);
|
|
1159
|
+
cancellation.dispose();
|
|
1160
|
+
if (child.exitCode === null && child.signalCode === null) terminate();
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
async function executeVerification(options, identity, input) {
|
|
1164
|
+
throwIfAborted3(input.signal);
|
|
1165
|
+
if (!isHash(input.expectedSemanticHash) || !Number.isSafeInteger(input.byteSize) || input.byteSize <= 0 || input.byteSize > options.maxOutputBytes || identity.codecVersions[input.codecId] !== input.codecVersion) {
|
|
1166
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
1167
|
+
}
|
|
1168
|
+
const manifest = new TextEncoder().encode(
|
|
1169
|
+
JSON.stringify({
|
|
1170
|
+
codecId: input.codecId,
|
|
1171
|
+
codecVersion: input.codecVersion,
|
|
1172
|
+
expectedSemanticHash: input.expectedSemanticHash,
|
|
1173
|
+
format: input.format,
|
|
1174
|
+
protocol: "OGAVJ001"
|
|
1175
|
+
})
|
|
1176
|
+
);
|
|
1177
|
+
const child = spawnNative(options, [VERIFY_ARGUMENT]);
|
|
1178
|
+
const terminate = () => terminateChild(child);
|
|
1179
|
+
const timeout = setTimeout(terminate, options.wallTimeoutMs);
|
|
1180
|
+
timeout.unref?.();
|
|
1181
|
+
const abort = () => terminate();
|
|
1182
|
+
input.signal.addEventListener("abort", abort, { once: true });
|
|
1183
|
+
const stderrDone = drainStderr(child, terminate);
|
|
1184
|
+
const exit = waitForExit(child);
|
|
1185
|
+
const cancellation = createAbortWait(input.signal);
|
|
1186
|
+
const inputDone = pumpFrame(
|
|
1187
|
+
child,
|
|
1188
|
+
VERIFY_INPUT_MAGIC,
|
|
1189
|
+
manifest,
|
|
1190
|
+
input.byteSize,
|
|
1191
|
+
input.chunks,
|
|
1192
|
+
input.signal,
|
|
1193
|
+
exit,
|
|
1194
|
+
cancellation.promise,
|
|
1195
|
+
"output_verification_failed"
|
|
1196
|
+
);
|
|
1197
|
+
void inputDone.catch(terminate);
|
|
1198
|
+
const reader = new BoundedProcessReader(child, terminate, cancellation.promise);
|
|
1199
|
+
try {
|
|
1200
|
+
const magic = await reader.readExactly(VERIFY_OUTPUT_MAGIC.byteLength, input.signal);
|
|
1201
|
+
if (equalBytes(magic, ERROR_MAGIC)) throw await readTypedError(reader, input.signal);
|
|
1202
|
+
if (!equalBytes(magic, VERIFY_OUTPUT_MAGIC)) {
|
|
1203
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
1204
|
+
}
|
|
1205
|
+
const header = await reader.readExactly(12, input.signal);
|
|
1206
|
+
const view = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
1207
|
+
const metadataBytes = view.getUint32(0, true);
|
|
1208
|
+
const payloadBytes = safeU64(view, 4, "verification payload length");
|
|
1209
|
+
if (metadataBytes < 2 || metadataBytes > MAX_METADATA_BYTES || payloadBytes !== 0) {
|
|
1210
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
1211
|
+
}
|
|
1212
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(
|
|
1213
|
+
await reader.readExactly(metadataBytes, input.signal)
|
|
1214
|
+
);
|
|
1215
|
+
const value = JSON.parse(text);
|
|
1216
|
+
if (JSON.stringify(value) !== text || typeof value !== "object" || value === null || Array.isArray(value) || Object.keys(value).sort().join("\0") !== "protocol\0semanticHash" || value.protocol !== "OGAVR001" || value.semanticHash !== input.expectedSemanticHash) {
|
|
1217
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
1218
|
+
}
|
|
1219
|
+
await reader.assertEof(input.signal);
|
|
1220
|
+
await inputDone;
|
|
1221
|
+
await stderrDone;
|
|
1222
|
+
const status = await exit;
|
|
1223
|
+
if (status.code !== 0 || status.signal !== null) {
|
|
1224
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
1225
|
+
}
|
|
1226
|
+
} catch (error) {
|
|
1227
|
+
terminate();
|
|
1228
|
+
await Promise.allSettled([inputDone, stderrDone, exit]);
|
|
1229
|
+
if (error instanceof EditableArtifactMaterializerPermanentError) throw error;
|
|
1230
|
+
throw new EditableArtifactMaterializerPermanentError("output_verification_failed");
|
|
1231
|
+
} finally {
|
|
1232
|
+
clearTimeout(timeout);
|
|
1233
|
+
input.signal.removeEventListener("abort", abort);
|
|
1234
|
+
cancellation.dispose();
|
|
1235
|
+
if (child.exitCode === null && child.signalCode === null) terminate();
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
async function readTypedError(reader, signal) {
|
|
1239
|
+
const header = await reader.readExactly(12, signal);
|
|
1240
|
+
const view = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
1241
|
+
const metadataBytes = view.getUint32(0, true);
|
|
1242
|
+
const payloadBytes = safeU64(view, 4, "error payload length");
|
|
1243
|
+
if (metadataBytes < 2 || metadataBytes > MAX_METADATA_BYTES || payloadBytes !== 0) {
|
|
1244
|
+
return new EditableArtifactMaterializerPermanentError("kernel_result_mismatch");
|
|
1245
|
+
}
|
|
1246
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(
|
|
1247
|
+
await reader.readExactly(metadataBytes, signal)
|
|
1248
|
+
);
|
|
1249
|
+
const value = JSON.parse(text);
|
|
1250
|
+
const keys = typeof value === "object" && value !== null && !Array.isArray(value) ? Object.keys(value).sort().join("\0") : "";
|
|
1251
|
+
if (JSON.stringify(value) !== text || typeof value !== "object" || value === null || Array.isArray(value) || keys !== "code\0protocol" && keys !== "code\0protocol\0subcode" || value.protocol !== "OGAMERR1") {
|
|
1252
|
+
return new EditableArtifactMaterializerPermanentError("kernel_result_mismatch");
|
|
1253
|
+
}
|
|
1254
|
+
const code = value.code;
|
|
1255
|
+
const subcode = value.subcode;
|
|
1256
|
+
if (code === "unsupported_semantics" || code === "source_identity_mismatch" || code === "kernel_incompatible" || code === "output_verification_failed") {
|
|
1257
|
+
const diagnostic = nativeFailureDiagnostic(code, subcode);
|
|
1258
|
+
if (subcode !== void 0 && !diagnostic) {
|
|
1259
|
+
return new EditableArtifactMaterializerPermanentError("kernel_result_mismatch");
|
|
1260
|
+
}
|
|
1261
|
+
return new EditableArtifactMaterializerPermanentError(code, diagnostic);
|
|
1262
|
+
}
|
|
1263
|
+
return new EditableArtifactMaterializerPermanentError("kernel_result_mismatch");
|
|
1264
|
+
}
|
|
1265
|
+
function nativeFailureDiagnostic(code, subcode) {
|
|
1266
|
+
if (code !== "source_identity_mismatch") return void 0;
|
|
1267
|
+
if (subcode === "input_framing" || subcode === "snapshot_open" || subcode === "state_mismatch") {
|
|
1268
|
+
return { stage: "native", subcode };
|
|
1269
|
+
}
|
|
1270
|
+
return void 0;
|
|
1271
|
+
}
|
|
1272
|
+
async function probeIdentity(options) {
|
|
1273
|
+
const child = spawnNative(options, [IDENTITY_ARGUMENT]);
|
|
1274
|
+
const terminate = () => terminateChild(child);
|
|
1275
|
+
const timeout = setTimeout(terminate, options.probeTimeoutMs ?? 5e3);
|
|
1276
|
+
timeout.unref?.();
|
|
1277
|
+
const stdout = collectBounded(child.stdout, MAX_IDENTITY_BYTES, terminate);
|
|
1278
|
+
const stderr = drainStderr(child, terminate);
|
|
1279
|
+
try {
|
|
1280
|
+
const [bytes, , status] = await Promise.all([stdout, stderr, waitForExit(child)]);
|
|
1281
|
+
if (status.code !== 0 || status.signal !== null) {
|
|
1282
|
+
throw new Error("native materializer identity probe failed");
|
|
1283
|
+
}
|
|
1284
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
1285
|
+
const parsed = JSON.parse(text);
|
|
1286
|
+
if (JSON.stringify(parsed) !== text) {
|
|
1287
|
+
throw new Error("native materializer identity is not canonical JSON");
|
|
1288
|
+
}
|
|
1289
|
+
return parsed;
|
|
1290
|
+
} finally {
|
|
1291
|
+
clearTimeout(timeout);
|
|
1292
|
+
if (child.exitCode === null && child.signalCode === null) terminate();
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
function spawnNative(options, args) {
|
|
1296
|
+
return options.launcher.spawn({
|
|
1297
|
+
executable: options.executable,
|
|
1298
|
+
args,
|
|
1299
|
+
environment: Object.freeze({
|
|
1300
|
+
LANG: "C",
|
|
1301
|
+
LC_ALL: "C",
|
|
1302
|
+
TZ: "UTC",
|
|
1303
|
+
...options.childEnvironment ?? {}
|
|
1304
|
+
})
|
|
1305
|
+
});
|
|
1306
|
+
}
|
|
1307
|
+
async function pumpInput(child, manifest, expectedSourceBytes, source, signal, exit, cancellation) {
|
|
1308
|
+
await pumpFrame(
|
|
1309
|
+
child,
|
|
1310
|
+
INPUT_MAGIC,
|
|
1311
|
+
manifest,
|
|
1312
|
+
expectedSourceBytes,
|
|
1313
|
+
source,
|
|
1314
|
+
signal,
|
|
1315
|
+
exit,
|
|
1316
|
+
cancellation,
|
|
1317
|
+
"source_identity_mismatch"
|
|
1318
|
+
);
|
|
1319
|
+
}
|
|
1320
|
+
async function pumpFrame(child, magic, manifest, expectedSourceBytes, source, signal, exit, cancellation, mismatchCode) {
|
|
1321
|
+
const header = new Uint8Array(12);
|
|
1322
|
+
const view = new DataView(header.buffer);
|
|
1323
|
+
view.setUint32(0, manifest.byteLength, true);
|
|
1324
|
+
view.setBigUint64(4, BigInt(expectedSourceBytes), true);
|
|
1325
|
+
await write(child, magic, signal, exit, cancellation);
|
|
1326
|
+
await write(child, header, signal, exit, cancellation);
|
|
1327
|
+
await write(child, manifest, signal, exit, cancellation);
|
|
1328
|
+
let sourceBytes = 0;
|
|
1329
|
+
for await (const chunk of source) {
|
|
1330
|
+
throwIfAborted3(signal);
|
|
1331
|
+
if (!(chunk instanceof Uint8Array) || chunk.byteLength === 0 || chunk.byteLength > 8 * 1024 * 1024) {
|
|
1332
|
+
throw new EditableArtifactMaterializerPermanentError(mismatchCode);
|
|
1333
|
+
}
|
|
1334
|
+
sourceBytes += chunk.byteLength;
|
|
1335
|
+
if (sourceBytes > expectedSourceBytes) {
|
|
1336
|
+
throw new EditableArtifactMaterializerPermanentError(mismatchCode);
|
|
1337
|
+
}
|
|
1338
|
+
await write(child, chunk, signal, exit, cancellation);
|
|
1339
|
+
}
|
|
1340
|
+
if (sourceBytes !== expectedSourceBytes) {
|
|
1341
|
+
throw new EditableArtifactMaterializerPermanentError(mismatchCode);
|
|
1342
|
+
}
|
|
1343
|
+
child.stdin.end();
|
|
1344
|
+
}
|
|
1345
|
+
async function write(child, bytes, signal, exit, cancellation) {
|
|
1346
|
+
throwIfAborted3(signal);
|
|
1347
|
+
if (!child.stdin.write(bytes)) {
|
|
1348
|
+
await Promise.race([
|
|
1349
|
+
once(child.stdin, "drain"),
|
|
1350
|
+
cancellation,
|
|
1351
|
+
exit.then(() => {
|
|
1352
|
+
throw new Error("native materializer closed input early");
|
|
1353
|
+
})
|
|
1354
|
+
]);
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
var BoundedProcessReader = class {
|
|
1358
|
+
constructor(child, terminate, cancellation) {
|
|
1359
|
+
this.terminate = terminate;
|
|
1360
|
+
this.cancellation = cancellation;
|
|
1361
|
+
this.#iterator = child.stdout[Symbol.asyncIterator]();
|
|
1362
|
+
}
|
|
1363
|
+
#iterator;
|
|
1364
|
+
#buffer = new Uint8Array(0);
|
|
1365
|
+
#done = false;
|
|
1366
|
+
async readExactly(length, signal) {
|
|
1367
|
+
if (!Number.isSafeInteger(length) || length < 0) throw new Error("invalid framed length");
|
|
1368
|
+
const output = new Uint8Array(length);
|
|
1369
|
+
let offset = 0;
|
|
1370
|
+
while (offset < length) {
|
|
1371
|
+
throwIfAborted3(signal);
|
|
1372
|
+
if (this.#buffer.byteLength === 0) await this.#fill(signal);
|
|
1373
|
+
if (this.#buffer.byteLength === 0) throw new Error("truncated native materializer output");
|
|
1374
|
+
const count = Math.min(length - offset, this.#buffer.byteLength);
|
|
1375
|
+
output.set(this.#buffer.subarray(0, count), offset);
|
|
1376
|
+
this.#buffer = this.#buffer.subarray(count);
|
|
1377
|
+
offset += count;
|
|
1378
|
+
}
|
|
1379
|
+
return output;
|
|
1380
|
+
}
|
|
1381
|
+
async *streamExactly(length, signal) {
|
|
1382
|
+
let remaining = length;
|
|
1383
|
+
while (remaining > 0) {
|
|
1384
|
+
throwIfAborted3(signal);
|
|
1385
|
+
const count = Math.min(remaining, STREAM_CHUNK_BYTES);
|
|
1386
|
+
const chunk = await this.readExactly(count, signal);
|
|
1387
|
+
remaining -= chunk.byteLength;
|
|
1388
|
+
yield chunk;
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
async assertEof(signal) {
|
|
1392
|
+
throwIfAborted3(signal);
|
|
1393
|
+
if (this.#buffer.byteLength > 0) throw new Error("trailing native materializer output");
|
|
1394
|
+
if (!this.#done) await this.#fill(signal);
|
|
1395
|
+
if (this.#buffer.byteLength > 0 || !this.#done) {
|
|
1396
|
+
throw new Error("trailing native materializer output");
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
async #fill(signal) {
|
|
1400
|
+
if (this.#done) return;
|
|
1401
|
+
throwIfAborted3(signal);
|
|
1402
|
+
let next;
|
|
1403
|
+
try {
|
|
1404
|
+
next = await Promise.race([this.#iterator.next(), this.cancellation]);
|
|
1405
|
+
} catch (error) {
|
|
1406
|
+
this.terminate();
|
|
1407
|
+
throw error;
|
|
1408
|
+
}
|
|
1409
|
+
if (next.done) {
|
|
1410
|
+
this.#done = true;
|
|
1411
|
+
return;
|
|
1412
|
+
}
|
|
1413
|
+
if (typeof next.value === "string") throw new Error("native materializer emitted text stream");
|
|
1414
|
+
this.#buffer = new Uint8Array(
|
|
1415
|
+
next.value.buffer,
|
|
1416
|
+
next.value.byteOffset,
|
|
1417
|
+
next.value.byteLength
|
|
1418
|
+
).slice();
|
|
1419
|
+
}
|
|
1420
|
+
};
|
|
1421
|
+
function encodeManifest(job, normalizedOptions) {
|
|
1422
|
+
const options = new TextDecoder("utf-8", { fatal: true }).decode(normalizedOptions);
|
|
1423
|
+
const manifest = {
|
|
1424
|
+
protocol: "OGAMJ001",
|
|
1425
|
+
artifactId: job.artifactId,
|
|
1426
|
+
jobId: job.jobId,
|
|
1427
|
+
versionId: job.versionId,
|
|
1428
|
+
modality: job.modality,
|
|
1429
|
+
inputSnapshotId: job.inputSnapshotId,
|
|
1430
|
+
targetHeadSequence: job.targetHeadSequence,
|
|
1431
|
+
stateHash: job.stateHash,
|
|
1432
|
+
sourceByteSize: job.sourceByteSize,
|
|
1433
|
+
sourceContentHash: job.sourceContentHash,
|
|
1434
|
+
modelSchemaVersion: job.modelSchemaVersion,
|
|
1435
|
+
operationProtocolVersion: job.operationProtocolVersion,
|
|
1436
|
+
snapshotProtocolVersion: job.snapshotProtocolVersion,
|
|
1437
|
+
format: job.format,
|
|
1438
|
+
codecId: job.codecId,
|
|
1439
|
+
normalizedOptions: JSON.parse(options),
|
|
1440
|
+
optionsHash: job.optionsHash,
|
|
1441
|
+
codecVersion: job.codecVersion,
|
|
1442
|
+
kernelVersion: job.kernelVersion,
|
|
1443
|
+
fontRegistryHash: job.fontRegistryHash,
|
|
1444
|
+
policyHash: job.policyHash
|
|
1445
|
+
};
|
|
1446
|
+
const bytes = new TextEncoder().encode(JSON.stringify(manifest));
|
|
1447
|
+
if (bytes.byteLength > MAX_METADATA_BYTES) {
|
|
1448
|
+
throw new EditableArtifactMaterializerPermanentError("invalid_job_manifest");
|
|
1449
|
+
}
|
|
1450
|
+
return bytes;
|
|
1451
|
+
}
|
|
1452
|
+
function decodeOutputMetadata(text, job, outputBytes) {
|
|
1453
|
+
const value = JSON.parse(text);
|
|
1454
|
+
if (JSON.stringify(value) !== text || typeof value !== "object" || value === null) {
|
|
1455
|
+
throw new Error("native materializer metadata is not canonical JSON");
|
|
1456
|
+
}
|
|
1457
|
+
const record = value;
|
|
1458
|
+
const expectedKeys = [
|
|
1459
|
+
"byteSize",
|
|
1460
|
+
"codecId",
|
|
1461
|
+
"codecVersion",
|
|
1462
|
+
"contentHash",
|
|
1463
|
+
"fontRegistryHash",
|
|
1464
|
+
"format",
|
|
1465
|
+
"headSequence",
|
|
1466
|
+
"kernelVersion",
|
|
1467
|
+
"mimeType",
|
|
1468
|
+
"policyHash",
|
|
1469
|
+
"protocol",
|
|
1470
|
+
"semanticHash",
|
|
1471
|
+
"stateHash"
|
|
1472
|
+
].sort();
|
|
1473
|
+
if (Object.keys(record).sort().join("\0") !== expectedKeys.join("\0")) {
|
|
1474
|
+
throw new Error("native materializer metadata fields are invalid");
|
|
1475
|
+
}
|
|
1476
|
+
if (record.protocol !== "OGAMR001" || record.byteSize !== outputBytes) {
|
|
1477
|
+
throw new Error("native materializer output length disagrees with metadata");
|
|
1478
|
+
}
|
|
1479
|
+
const expectedMimeType = mimeTypeForFormat(job.format);
|
|
1480
|
+
if (!Number.isSafeInteger(record.headSequence) || record.headSequence !== job.targetHeadSequence || !isHash(record.stateHash) || record.stateHash !== job.stateHash || record.format !== job.format || record.mimeType !== expectedMimeType || !isBoundedVersion(record.codecId) || record.codecId !== job.codecId || !isBoundedVersion(record.codecVersion) || record.codecVersion !== job.codecVersion || !isBoundedKernelVersion(record.kernelVersion) || record.kernelVersion !== job.kernelVersion || !isHash(record.fontRegistryHash) || record.fontRegistryHash !== job.fontRegistryHash || !isHash(record.policyHash) || record.policyHash !== job.policyHash || !isHash(record.contentHash) || !isHash(record.semanticHash)) {
|
|
1481
|
+
throw new Error("native materializer output metadata is incompatible");
|
|
1482
|
+
}
|
|
1483
|
+
return Object.freeze({
|
|
1484
|
+
headSequence: record.headSequence,
|
|
1485
|
+
stateHash: record.stateHash,
|
|
1486
|
+
format: job.format,
|
|
1487
|
+
mimeType: expectedMimeType,
|
|
1488
|
+
codecId: record.codecId,
|
|
1489
|
+
codecVersion: record.codecVersion,
|
|
1490
|
+
kernelVersion: record.kernelVersion,
|
|
1491
|
+
fontRegistryHash: record.fontRegistryHash,
|
|
1492
|
+
policyHash: record.policyHash,
|
|
1493
|
+
byteSize: outputBytes,
|
|
1494
|
+
contentHash: record.contentHash,
|
|
1495
|
+
semanticHash: record.semanticHash
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1498
|
+
function validateOptions(options) {
|
|
1499
|
+
if (!isAbsolute(options.executable) || options.executable.length > 2048) {
|
|
1500
|
+
throw new TypeError("native materializer executable must be an absolute path");
|
|
1501
|
+
}
|
|
1502
|
+
for (const [label, value] of [
|
|
1503
|
+
["wall timeout", options.wallTimeoutMs],
|
|
1504
|
+
["source limit", options.maxSourceBytes],
|
|
1505
|
+
["output limit", options.maxOutputBytes]
|
|
1506
|
+
]) {
|
|
1507
|
+
if (!Number.isSafeInteger(value) || value <= 0) throw new TypeError(`${label} is invalid`);
|
|
1508
|
+
}
|
|
1509
|
+
const launcher = options.launcher.identity;
|
|
1510
|
+
const sandboxed = launcher.sandboxEnforced === true && launcher.network === "denied" && Number.isSafeInteger(launcher.memoryLimitBytes) && launcher.memoryLimitBytes > 0 && Number.isSafeInteger(launcher.cpuTimeLimitMs) && launcher.cpuTimeLimitMs > 0 && Number.isSafeInteger(launcher.fileDescriptorLimit) && launcher.fileDescriptorLimit > 0 && Number.isSafeInteger(launcher.processLimit) && launcher.processLimit > 0 && Number.isSafeInteger(launcher.fileSizeLimitBytes) && launcher.fileSizeLimitBytes > 0;
|
|
1511
|
+
const developmentUnsandboxed = options.allowUnsandboxedDevelopment === true && launcher.sandboxEnforced === false && launcher.network === "host" && launcher.memoryLimitBytes === 0 && launcher.cpuTimeLimitMs === 0 && launcher.fileDescriptorLimit === 0 && launcher.processLimit === 0 && launcher.fileSizeLimitBytes === 0;
|
|
1512
|
+
if (launcher.isolation !== "subprocess" || launcher.officeAutomation !== false || !isBoundedVersion(launcher.platform) || !sandboxed && !developmentUnsandboxed) {
|
|
1513
|
+
throw new TypeError("materializer launcher isolation is not verified");
|
|
1514
|
+
}
|
|
1515
|
+
if (options.childEnvironment !== void 0) {
|
|
1516
|
+
const keys = Object.keys(options.childEnvironment).sort();
|
|
1517
|
+
const productionKeys = ["OPENGENI_ARTIFACT_RUNTIME_MANIFEST", "OPENGENI_ARTIFACT_TOOL_ENTRY"];
|
|
1518
|
+
const developmentKeys = [
|
|
1519
|
+
"NODE_ENV",
|
|
1520
|
+
"OPENGENI_ARTIFACT_DEVELOPMENT_RUNTIME_MANIFEST",
|
|
1521
|
+
"OPENGENI_ARTIFACT_TOOL_ENTRY"
|
|
1522
|
+
];
|
|
1523
|
+
const production = keys.join("\0") === productionKeys.join("\0");
|
|
1524
|
+
const development = keys.join("\0") === developmentKeys.join("\0");
|
|
1525
|
+
if (!production && !development) {
|
|
1526
|
+
throw new TypeError("native materializer child environment is not exact");
|
|
1527
|
+
}
|
|
1528
|
+
if (development !== developmentUnsandboxed) {
|
|
1529
|
+
throw new TypeError("native materializer launcher and runtime authority modes differ");
|
|
1530
|
+
}
|
|
1531
|
+
if (development && (options.allowUnsandboxedDevelopment !== true || options.childEnvironment.NODE_ENV !== "development")) {
|
|
1532
|
+
throw new TypeError("native materializer development child environment is forbidden");
|
|
1533
|
+
}
|
|
1534
|
+
for (const [name, value] of Object.entries(options.childEnvironment)) {
|
|
1535
|
+
if (name === "NODE_ENV") continue;
|
|
1536
|
+
if (!isAbsolute(value) || value.length > 2048 || /[\u0000-\u001f\u007f]/u.test(value)) {
|
|
1537
|
+
throw new TypeError("native materializer child environment path is invalid");
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
function validateIdentity(identity) {
|
|
1543
|
+
const record = identity;
|
|
1544
|
+
const expectedKeys = [
|
|
1545
|
+
"codecVersions",
|
|
1546
|
+
"fontRegistryHash",
|
|
1547
|
+
"kernelVersion",
|
|
1548
|
+
"maxOutputBytes",
|
|
1549
|
+
"policyHash",
|
|
1550
|
+
"protocol",
|
|
1551
|
+
"runtimeKind",
|
|
1552
|
+
"runtimeTarget",
|
|
1553
|
+
"supportedModelSchemaVersions",
|
|
1554
|
+
"supportedOperationProtocolVersions",
|
|
1555
|
+
"supportedSnapshotProtocolVersions"
|
|
1556
|
+
].sort();
|
|
1557
|
+
if (Object.keys(record).sort().join("\0") !== expectedKeys.join("\0")) {
|
|
1558
|
+
throw new Error("native materializer identity fields are invalid");
|
|
1559
|
+
}
|
|
1560
|
+
if (identity.protocol !== "OGAMC001" || identity.runtimeKind !== "native" || !Number.isSafeInteger(identity.maxOutputBytes) || identity.maxOutputBytes <= 0) {
|
|
1561
|
+
throw new Error("native materializer sandbox identity is incompatible");
|
|
1562
|
+
}
|
|
1563
|
+
if (!isBoundedKernelVersion(identity.kernelVersion) || !isBoundedVersion(identity.runtimeTarget) || !isHash(identity.fontRegistryHash) || !isHash(identity.policyHash) || !isCodecRegistry(identity.codecVersions) || !isCanonicalPositiveVersionList(identity.supportedModelSchemaVersions) || !isCanonicalPositiveVersionList(identity.supportedOperationProtocolVersions) || !isCanonicalPositiveVersionList(identity.supportedSnapshotProtocolVersions)) {
|
|
1564
|
+
throw new Error("native materializer capability identity is incomplete");
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
function compositeIdentity(child, options) {
|
|
1568
|
+
const launcher = options.launcher.identity;
|
|
1569
|
+
return Object.freeze({
|
|
1570
|
+
kind: "native",
|
|
1571
|
+
isolation: launcher.isolation,
|
|
1572
|
+
network: launcher.network,
|
|
1573
|
+
officeAutomation: launcher.officeAutomation,
|
|
1574
|
+
processProtocolVersion: 1,
|
|
1575
|
+
sandboxEnforced: launcher.sandboxEnforced,
|
|
1576
|
+
memoryLimitBytes: launcher.memoryLimitBytes,
|
|
1577
|
+
cpuTimeLimitMs: launcher.cpuTimeLimitMs,
|
|
1578
|
+
fileDescriptorLimit: launcher.fileDescriptorLimit,
|
|
1579
|
+
processLimit: launcher.processLimit,
|
|
1580
|
+
fileSizeLimitBytes: launcher.fileSizeLimitBytes,
|
|
1581
|
+
maxOutputBytes: Math.min(child.maxOutputBytes, options.maxOutputBytes),
|
|
1582
|
+
kernelVersion: child.kernelVersion,
|
|
1583
|
+
codecVersions: child.codecVersions,
|
|
1584
|
+
fontRegistryHash: child.fontRegistryHash,
|
|
1585
|
+
policyHash: child.policyHash,
|
|
1586
|
+
runtimeTarget: child.runtimeTarget,
|
|
1587
|
+
supportedModelSchemaVersions: child.supportedModelSchemaVersions,
|
|
1588
|
+
supportedOperationProtocolVersions: child.supportedOperationProtocolVersions,
|
|
1589
|
+
supportedSnapshotProtocolVersions: child.supportedSnapshotProtocolVersions
|
|
1590
|
+
});
|
|
1591
|
+
}
|
|
1592
|
+
function isCodecRegistry(value) {
|
|
1593
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
1594
|
+
const entries = Object.entries(value);
|
|
1595
|
+
if (entries.length === 0 || entries.length > 64) return false;
|
|
1596
|
+
let previous = "";
|
|
1597
|
+
for (const [codecId, version] of entries) {
|
|
1598
|
+
if (!/^[a-z][a-z0-9._-]{0,127}$/u.test(codecId) || codecId <= previous || !isBoundedVersion(version)) {
|
|
1599
|
+
return false;
|
|
1600
|
+
}
|
|
1601
|
+
previous = codecId;
|
|
1602
|
+
}
|
|
1603
|
+
return true;
|
|
1604
|
+
}
|
|
1605
|
+
function isCanonicalPositiveVersionList(value) {
|
|
1606
|
+
if (!Array.isArray(value) || value.length === 0 || value.length > 64) return false;
|
|
1607
|
+
let previous = 0;
|
|
1608
|
+
for (const version of value) {
|
|
1609
|
+
if (!Number.isSafeInteger(version) || version <= previous) return false;
|
|
1610
|
+
previous = version;
|
|
1611
|
+
}
|
|
1612
|
+
return true;
|
|
1613
|
+
}
|
|
1614
|
+
function isBoundedVersion(value) {
|
|
1615
|
+
return typeof value === "string" && value.length >= 1 && value.length <= 128 && /^[\x21-\x7e]+$/u.test(value);
|
|
1616
|
+
}
|
|
1617
|
+
function isBoundedKernelVersion(value) {
|
|
1618
|
+
return typeof value === "string" && value.length >= 1 && new TextEncoder().encode(value).byteLength <= EDITABLE_ARTIFACT_KERNEL_VERSION_MAX_BYTES2 && /^[\x21-\x7e]+$/u.test(value);
|
|
1619
|
+
}
|
|
1620
|
+
function isHash(value) {
|
|
1621
|
+
return typeof value === "string" && /^sha256:[0-9a-f]{64}$/u.test(value);
|
|
1622
|
+
}
|
|
1623
|
+
async function collectBounded(stream, maximum, terminate) {
|
|
1624
|
+
const chunks = [];
|
|
1625
|
+
let total = 0;
|
|
1626
|
+
for await (const value of stream) {
|
|
1627
|
+
const chunk = typeof value === "string" ? Buffer.from(value) : value;
|
|
1628
|
+
total += chunk.byteLength;
|
|
1629
|
+
if (total > maximum) {
|
|
1630
|
+
terminate();
|
|
1631
|
+
throw new Error("native materializer diagnostic exceeded its bound");
|
|
1632
|
+
}
|
|
1633
|
+
chunks.push(new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength).slice());
|
|
1634
|
+
}
|
|
1635
|
+
const result = new Uint8Array(total);
|
|
1636
|
+
let offset = 0;
|
|
1637
|
+
for (const chunk of chunks) {
|
|
1638
|
+
result.set(chunk, offset);
|
|
1639
|
+
offset += chunk.byteLength;
|
|
1640
|
+
}
|
|
1641
|
+
return result;
|
|
1642
|
+
}
|
|
1643
|
+
async function drainStderr(child, terminate) {
|
|
1644
|
+
await collectBounded(child.stderr, MAX_STDERR_BYTES, terminate);
|
|
1645
|
+
}
|
|
1646
|
+
function waitForExit(child) {
|
|
1647
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
1648
|
+
return Promise.resolve({ code: child.exitCode, signal: child.signalCode });
|
|
1649
|
+
}
|
|
1650
|
+
return new Promise((resolve, reject) => {
|
|
1651
|
+
const onError = (error) => {
|
|
1652
|
+
child.removeListener("exit", onExit);
|
|
1653
|
+
reject(error);
|
|
1654
|
+
};
|
|
1655
|
+
const onExit = (code, signal) => {
|
|
1656
|
+
child.removeListener("error", onError);
|
|
1657
|
+
resolve({ code, signal });
|
|
1658
|
+
};
|
|
1659
|
+
child.once("error", onError);
|
|
1660
|
+
child.once("exit", onExit);
|
|
1661
|
+
});
|
|
1662
|
+
}
|
|
1663
|
+
function terminateChild(child) {
|
|
1664
|
+
if (child.exitCode !== null || child.signalCode !== null) return;
|
|
1665
|
+
child.stdin.destroy();
|
|
1666
|
+
child.kill("SIGKILL");
|
|
1667
|
+
}
|
|
1668
|
+
function createAbortWait(signal) {
|
|
1669
|
+
let listener = null;
|
|
1670
|
+
const promise = new Promise((_resolve, reject) => {
|
|
1671
|
+
if (signal.aborted) {
|
|
1672
|
+
reject(signal.reason ?? abortError2());
|
|
1673
|
+
return;
|
|
1674
|
+
}
|
|
1675
|
+
listener = () => reject(signal.reason ?? abortError2());
|
|
1676
|
+
signal.addEventListener("abort", listener, { once: true });
|
|
1677
|
+
});
|
|
1678
|
+
void promise.catch(() => void 0);
|
|
1679
|
+
return {
|
|
1680
|
+
promise,
|
|
1681
|
+
dispose() {
|
|
1682
|
+
if (listener) signal.removeEventListener("abort", listener);
|
|
1683
|
+
listener = null;
|
|
1684
|
+
}
|
|
1685
|
+
};
|
|
1686
|
+
}
|
|
1687
|
+
function throwIfAborted3(signal) {
|
|
1688
|
+
if (signal.aborted) throw signal.reason ?? abortError2();
|
|
1689
|
+
}
|
|
1690
|
+
function abortError2() {
|
|
1691
|
+
return new DOMException("Operation aborted", "AbortError");
|
|
1692
|
+
}
|
|
1693
|
+
function mapSubprocessFailure(error, signal) {
|
|
1694
|
+
if (signal.aborted) return signal.reason ?? abortError2();
|
|
1695
|
+
return error;
|
|
1696
|
+
}
|
|
1697
|
+
function safeU64(view, offset, label) {
|
|
1698
|
+
const value = view.getBigUint64(offset, true);
|
|
1699
|
+
if (value > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error(`${label} exceeds safe integer`);
|
|
1700
|
+
return Number(value);
|
|
1701
|
+
}
|
|
1702
|
+
function equalBytes(left, right) {
|
|
1703
|
+
return left.byteLength === right.byteLength && left.every((byte, index) => byte === right[index]);
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1706
|
+
// src/editable-artifact-materializer-launcher.ts
|
|
1707
|
+
import { spawn } from "child_process";
|
|
1708
|
+
import { access, readlink, realpath, stat } from "fs/promises";
|
|
1709
|
+
import { basename, dirname, isAbsolute as isAbsolute2, relative } from "path";
|
|
1710
|
+
var MAX_PROBE_BYTES = 16 * 1024;
|
|
1711
|
+
var PROBE_TIMEOUT_MS = 5e3;
|
|
1712
|
+
async function createDevelopmentEditableArtifactProcessLauncher(options) {
|
|
1713
|
+
if (!options.explicitlyEnabled || options.nodeEnvironment === "production") {
|
|
1714
|
+
throw new Error("Unsandboxed artifact materialization is forbidden outside development");
|
|
1715
|
+
}
|
|
1716
|
+
const configuredMaterializerExecutable = options.materializerExecutable;
|
|
1717
|
+
const materializerExecutable = await developmentExecutable(options.materializerExecutable);
|
|
1718
|
+
return Object.freeze({
|
|
1719
|
+
identity: Object.freeze({
|
|
1720
|
+
platform: `${process.platform}-${process.arch}-development-unsandboxed-v1`,
|
|
1721
|
+
isolation: "subprocess",
|
|
1722
|
+
network: "host",
|
|
1723
|
+
officeAutomation: false,
|
|
1724
|
+
sandboxEnforced: false,
|
|
1725
|
+
memoryLimitBytes: 0,
|
|
1726
|
+
cpuTimeLimitMs: 0,
|
|
1727
|
+
fileDescriptorLimit: 0,
|
|
1728
|
+
processLimit: 0,
|
|
1729
|
+
fileSizeLimitBytes: 0
|
|
1730
|
+
}),
|
|
1731
|
+
spawn(input) {
|
|
1732
|
+
if (input.executable !== configuredMaterializerExecutable && input.executable !== materializerExecutable) {
|
|
1733
|
+
throw new Error("Development materializer differs from its verified executable");
|
|
1734
|
+
}
|
|
1735
|
+
return spawn(materializerExecutable, [...input.args], {
|
|
1736
|
+
cwd: "/",
|
|
1737
|
+
env: { ...input.environment },
|
|
1738
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
1739
|
+
windowsHide: true
|
|
1740
|
+
});
|
|
1741
|
+
}
|
|
1742
|
+
});
|
|
1743
|
+
}
|
|
1744
|
+
async function createLinuxEditableArtifactProcessLauncher(options) {
|
|
1745
|
+
if (process.platform !== "linux") {
|
|
1746
|
+
throw new Error("Linux artifact materializer sandbox is unavailable on this platform");
|
|
1747
|
+
}
|
|
1748
|
+
if (!Number.isSafeInteger(options.memoryLimitBytes) || options.memoryLimitBytes <= 0 || !Number.isSafeInteger(options.cpuTimeLimitMs) || options.cpuTimeLimitMs <= 0 || options.cpuTimeLimitMs % 1e3 !== 0 || !Number.isSafeInteger(options.fileDescriptorLimit) || options.fileDescriptorLimit <= 0 || !Number.isSafeInteger(options.processLimit) || options.processLimit <= 0 || !Number.isSafeInteger(options.fileSizeLimitBytes) || options.fileSizeLimitBytes <= 0) {
|
|
1749
|
+
throw new Error("Artifact materializer launcher limits are invalid");
|
|
1750
|
+
}
|
|
1751
|
+
const bubblewrap = await trustedExecutable(options.bubblewrapExecutable, "bwrap");
|
|
1752
|
+
const prlimit = await trustedExecutable(options.prlimitExecutable, "prlimit");
|
|
1753
|
+
const configuredMaterializerExecutable = options.materializerExecutable;
|
|
1754
|
+
const materializerExecutable = await trustedExecutable(
|
|
1755
|
+
options.materializerExecutable,
|
|
1756
|
+
basename(options.materializerExecutable)
|
|
1757
|
+
);
|
|
1758
|
+
const runtimeRoot = await trustedRuntimeRoot(options.runtimeRoot);
|
|
1759
|
+
const cpuSeconds = options.cpuTimeLimitMs / 1e3;
|
|
1760
|
+
const spawnIsolated = (input) => {
|
|
1761
|
+
if (!isAbsolute2(input.executable)) throw new Error("Sandbox target must be absolute");
|
|
1762
|
+
const sandboxArguments = linuxEditableArtifactSandboxArguments({
|
|
1763
|
+
executable: input.executable,
|
|
1764
|
+
args: input.args,
|
|
1765
|
+
runtimeRoot,
|
|
1766
|
+
scratchLimitBytes: options.fileSizeLimitBytes
|
|
1767
|
+
});
|
|
1768
|
+
return spawn(
|
|
1769
|
+
prlimit,
|
|
1770
|
+
[
|
|
1771
|
+
...linuxEditableArtifactPrlimitArguments({
|
|
1772
|
+
memoryLimitBytes: options.memoryLimitBytes,
|
|
1773
|
+
cpuSeconds,
|
|
1774
|
+
fileDescriptorLimit: options.fileDescriptorLimit,
|
|
1775
|
+
processLimit: options.processLimit,
|
|
1776
|
+
fileSizeLimitBytes: options.fileSizeLimitBytes
|
|
1777
|
+
}),
|
|
1778
|
+
"--",
|
|
1779
|
+
bubblewrap,
|
|
1780
|
+
...sandboxArguments
|
|
1781
|
+
],
|
|
1782
|
+
{
|
|
1783
|
+
cwd: "/",
|
|
1784
|
+
env: { ...input.environment },
|
|
1785
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
1786
|
+
windowsHide: true
|
|
1787
|
+
}
|
|
1788
|
+
);
|
|
1789
|
+
};
|
|
1790
|
+
const spawnMaterializer = (input) => {
|
|
1791
|
+
if (input.executable !== configuredMaterializerExecutable && input.executable !== materializerExecutable) {
|
|
1792
|
+
throw new Error("Sandbox target differs from the verified materializer executable");
|
|
1793
|
+
}
|
|
1794
|
+
assertRuntimeEnvironmentConfined(input.environment, runtimeRoot);
|
|
1795
|
+
return spawnIsolated({ ...input, executable: materializerExecutable });
|
|
1796
|
+
};
|
|
1797
|
+
const probeExecutable = await trustedExecutable(
|
|
1798
|
+
options.probeExecutable ?? process.execPath,
|
|
1799
|
+
basename(process.execPath)
|
|
1800
|
+
);
|
|
1801
|
+
const parentNetworkNamespace = await readlink("/proc/self/ns/net");
|
|
1802
|
+
const probe = spawnIsolated({
|
|
1803
|
+
executable: probeExecutable,
|
|
1804
|
+
args: [
|
|
1805
|
+
"-e",
|
|
1806
|
+
[
|
|
1807
|
+
'const fs=require("node:fs")',
|
|
1808
|
+
'const net=fs.readlinkSync("/proc/self/ns/net")',
|
|
1809
|
+
'const limits=fs.readFileSync("/proc/self/limits","utf8")',
|
|
1810
|
+
'const rootEntries=fs.readdirSync("/").sort()',
|
|
1811
|
+
'let rootWritable=false;try{fs.writeFileSync("/opengeni-root-write-probe","x");rootWritable=true;fs.unlinkSync("/opengeni-root-write-probe")}catch{}',
|
|
1812
|
+
'let scratchWritable=false;try{fs.writeFileSync("/tmp/opengeni-scratch-write-probe","x");scratchWritable=true;fs.unlinkSync("/tmp/opengeni-scratch-write-probe")}catch{}',
|
|
1813
|
+
"process.stdout.write(JSON.stringify({net,limits,rootEntries,rootWritable,scratchWritable}))"
|
|
1814
|
+
].join(";")
|
|
1815
|
+
],
|
|
1816
|
+
environment: Object.freeze({ LANG: "C", LC_ALL: "C", TZ: "UTC" })
|
|
1817
|
+
});
|
|
1818
|
+
probe.stdin.end();
|
|
1819
|
+
const timeout = setTimeout(() => terminateChild2(probe), PROBE_TIMEOUT_MS);
|
|
1820
|
+
timeout.unref?.();
|
|
1821
|
+
let stdout;
|
|
1822
|
+
let stderr;
|
|
1823
|
+
let status;
|
|
1824
|
+
try {
|
|
1825
|
+
[stdout, stderr, status] = await Promise.all([
|
|
1826
|
+
boundedStream(probe.stdout),
|
|
1827
|
+
boundedStream(probe.stderr),
|
|
1828
|
+
waitForExit2(probe)
|
|
1829
|
+
]);
|
|
1830
|
+
} finally {
|
|
1831
|
+
clearTimeout(timeout);
|
|
1832
|
+
terminateChild2(probe);
|
|
1833
|
+
}
|
|
1834
|
+
if (status.code !== 0 || status.signal !== null || stderr.byteLength !== 0) {
|
|
1835
|
+
throw new Error("Artifact materializer sandbox behavioral probe failed");
|
|
1836
|
+
}
|
|
1837
|
+
const parsed = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(stdout));
|
|
1838
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
1839
|
+
throw new Error("Artifact materializer sandbox probe returned invalid evidence");
|
|
1840
|
+
}
|
|
1841
|
+
const evidence = parsed;
|
|
1842
|
+
const expectedRootEntries = linuxEditableArtifactSandboxRootEntries({
|
|
1843
|
+
executable: probeExecutable,
|
|
1844
|
+
runtimeRoot
|
|
1845
|
+
});
|
|
1846
|
+
if (typeof evidence.net !== "string" || evidence.net === parentNetworkNamespace || typeof evidence.limits !== "string" || !hasResourceLimit(evidence.limits, "Max cpu time", String(cpuSeconds)) || !hasResourceLimit(evidence.limits, "Max address space", String(options.memoryLimitBytes)) || !hasResourceLimit(evidence.limits, "Max open files", String(options.fileDescriptorLimit)) || !hasResourceLimit(evidence.limits, "Max processes", String(options.processLimit)) || !hasResourceLimit(evidence.limits, "Max file size", String(options.fileSizeLimitBytes)) || !sameStringList(evidence.rootEntries, expectedRootEntries) || evidence.rootWritable !== false || evidence.scratchWritable !== true) {
|
|
1847
|
+
throw new Error("Artifact materializer sandbox enforcement could not be proved");
|
|
1848
|
+
}
|
|
1849
|
+
return Object.freeze({
|
|
1850
|
+
identity: Object.freeze({
|
|
1851
|
+
platform: `linux-${process.arch}-bwrap-prlimit-v1`,
|
|
1852
|
+
isolation: "subprocess",
|
|
1853
|
+
network: "denied",
|
|
1854
|
+
officeAutomation: false,
|
|
1855
|
+
sandboxEnforced: true,
|
|
1856
|
+
memoryLimitBytes: options.memoryLimitBytes,
|
|
1857
|
+
cpuTimeLimitMs: options.cpuTimeLimitMs,
|
|
1858
|
+
fileDescriptorLimit: options.fileDescriptorLimit,
|
|
1859
|
+
processLimit: options.processLimit,
|
|
1860
|
+
fileSizeLimitBytes: options.fileSizeLimitBytes
|
|
1861
|
+
}),
|
|
1862
|
+
spawn: spawnMaterializer
|
|
1863
|
+
});
|
|
1864
|
+
}
|
|
1865
|
+
function linuxEditableArtifactSandboxArguments(input) {
|
|
1866
|
+
if (!isAbsolute2(input.executable) || !isAbsolute2(input.runtimeRoot) || !Number.isSafeInteger(input.scratchLimitBytes) || input.scratchLimitBytes <= 0) {
|
|
1867
|
+
throw new Error("Artifact materializer sandbox paths must be absolute");
|
|
1868
|
+
}
|
|
1869
|
+
const parentDirectories = sandboxParentDirectories([input.executable, input.runtimeRoot]);
|
|
1870
|
+
return Object.freeze([
|
|
1871
|
+
"--die-with-parent",
|
|
1872
|
+
"--new-session",
|
|
1873
|
+
"--unshare-all",
|
|
1874
|
+
...parentDirectories.flatMap((path) => ["--dir", path]),
|
|
1875
|
+
"--ro-bind",
|
|
1876
|
+
"/usr/lib",
|
|
1877
|
+
"/usr/lib",
|
|
1878
|
+
"--ro-bind-try",
|
|
1879
|
+
"/usr/lib64",
|
|
1880
|
+
"/usr/lib64",
|
|
1881
|
+
"--ro-bind-try",
|
|
1882
|
+
"/usr/share",
|
|
1883
|
+
"/usr/share",
|
|
1884
|
+
"--symlink",
|
|
1885
|
+
"usr/lib",
|
|
1886
|
+
"/lib",
|
|
1887
|
+
"--symlink",
|
|
1888
|
+
"usr/lib64",
|
|
1889
|
+
"/lib64",
|
|
1890
|
+
"--dir",
|
|
1891
|
+
"/etc",
|
|
1892
|
+
"--ro-bind-try",
|
|
1893
|
+
"/etc/ld.so.cache",
|
|
1894
|
+
"/etc/ld.so.cache",
|
|
1895
|
+
"--ro-bind-try",
|
|
1896
|
+
"/etc/ld.so.conf",
|
|
1897
|
+
"/etc/ld.so.conf",
|
|
1898
|
+
"--ro-bind-try",
|
|
1899
|
+
"/etc/ld.so.conf.d",
|
|
1900
|
+
"/etc/ld.so.conf.d",
|
|
1901
|
+
"--ro-bind",
|
|
1902
|
+
input.runtimeRoot,
|
|
1903
|
+
input.runtimeRoot,
|
|
1904
|
+
"--ro-bind",
|
|
1905
|
+
input.executable,
|
|
1906
|
+
input.executable,
|
|
1907
|
+
"--proc",
|
|
1908
|
+
"/proc",
|
|
1909
|
+
"--dev",
|
|
1910
|
+
"/dev",
|
|
1911
|
+
"--size",
|
|
1912
|
+
String(input.scratchLimitBytes),
|
|
1913
|
+
"--tmpfs",
|
|
1914
|
+
"/tmp",
|
|
1915
|
+
"--remount-ro",
|
|
1916
|
+
"/",
|
|
1917
|
+
"--chdir",
|
|
1918
|
+
"/",
|
|
1919
|
+
"--",
|
|
1920
|
+
input.executable,
|
|
1921
|
+
...input.args
|
|
1922
|
+
]);
|
|
1923
|
+
}
|
|
1924
|
+
function linuxEditableArtifactPrlimitArguments(input) {
|
|
1925
|
+
return Object.freeze([
|
|
1926
|
+
`--as=${input.memoryLimitBytes}`,
|
|
1927
|
+
`--cpu=${input.cpuSeconds}`,
|
|
1928
|
+
`--nofile=${input.fileDescriptorLimit}`,
|
|
1929
|
+
`--nproc=${input.processLimit}`,
|
|
1930
|
+
`--fsize=${input.fileSizeLimitBytes}`
|
|
1931
|
+
]);
|
|
1932
|
+
}
|
|
1933
|
+
function linuxEditableArtifactSandboxRootEntries(input) {
|
|
1934
|
+
const entries = /* @__PURE__ */ new Set(["dev", "etc", "lib", "lib64", "proc", "tmp", "usr"]);
|
|
1935
|
+
for (const path of [input.executable, input.runtimeRoot]) {
|
|
1936
|
+
const topLevel = path.split("/").find((part) => part.length > 0);
|
|
1937
|
+
if (topLevel) entries.add(topLevel);
|
|
1938
|
+
}
|
|
1939
|
+
return Object.freeze([...entries].sort());
|
|
1940
|
+
}
|
|
1941
|
+
async function trustedExecutable(input, expectedBasename) {
|
|
1942
|
+
if (!isAbsolute2(input) || basename(input) !== expectedBasename) {
|
|
1943
|
+
throw new Error(`Artifact materializer ${expectedBasename} path is invalid`);
|
|
1944
|
+
}
|
|
1945
|
+
const canonical = await realpath(input);
|
|
1946
|
+
const metadata = await stat(canonical);
|
|
1947
|
+
await access(canonical, 1);
|
|
1948
|
+
if (!metadata.isFile() || metadata.uid !== 0 || (metadata.mode & 18) !== 0) {
|
|
1949
|
+
throw new Error(`Artifact materializer ${expectedBasename} is not a trusted executable`);
|
|
1950
|
+
}
|
|
1951
|
+
return canonical;
|
|
1952
|
+
}
|
|
1953
|
+
async function trustedRuntimeRoot(input) {
|
|
1954
|
+
if (!isAbsolute2(input) || broadRuntimeRoot(input)) {
|
|
1955
|
+
throw new Error("Artifact materializer runtime root is invalid");
|
|
1956
|
+
}
|
|
1957
|
+
const canonical = await realpath(input);
|
|
1958
|
+
const metadata = await stat(canonical);
|
|
1959
|
+
if (canonical !== input || !metadata.isDirectory() || metadata.uid !== 0 || (metadata.mode & 18) !== 0) {
|
|
1960
|
+
throw new Error("Artifact materializer runtime root is not a trusted directory");
|
|
1961
|
+
}
|
|
1962
|
+
return canonical;
|
|
1963
|
+
}
|
|
1964
|
+
function broadRuntimeRoot(input) {
|
|
1965
|
+
if (input.split("/").filter(Boolean).length < 2) return true;
|
|
1966
|
+
return (/* @__PURE__ */ new Set([
|
|
1967
|
+
"/",
|
|
1968
|
+
"/app",
|
|
1969
|
+
"/bin",
|
|
1970
|
+
"/dev",
|
|
1971
|
+
"/etc",
|
|
1972
|
+
"/home",
|
|
1973
|
+
"/lib",
|
|
1974
|
+
"/lib64",
|
|
1975
|
+
"/media",
|
|
1976
|
+
"/mnt",
|
|
1977
|
+
"/opt",
|
|
1978
|
+
"/proc",
|
|
1979
|
+
"/root",
|
|
1980
|
+
"/run",
|
|
1981
|
+
"/sbin",
|
|
1982
|
+
"/srv",
|
|
1983
|
+
"/sys",
|
|
1984
|
+
"/tmp",
|
|
1985
|
+
"/usr",
|
|
1986
|
+
"/usr/local",
|
|
1987
|
+
"/var",
|
|
1988
|
+
"/workspace"
|
|
1989
|
+
])).has(input);
|
|
1990
|
+
}
|
|
1991
|
+
function assertRuntimeEnvironmentConfined(environment, runtimeRoot) {
|
|
1992
|
+
for (const name of [
|
|
1993
|
+
"OPENGENI_ARTIFACT_RUNTIME_MANIFEST",
|
|
1994
|
+
"OPENGENI_ARTIFACT_TOOL_ENTRY"
|
|
1995
|
+
]) {
|
|
1996
|
+
const path = environment[name];
|
|
1997
|
+
if (!path || !isPathInside(runtimeRoot, path)) {
|
|
1998
|
+
throw new Error("Artifact materializer runtime environment escapes its verified root");
|
|
1999
|
+
}
|
|
2000
|
+
}
|
|
2001
|
+
}
|
|
2002
|
+
function isPathInside(root, path) {
|
|
2003
|
+
if (!isAbsolute2(path)) return false;
|
|
2004
|
+
const fromRoot = relative(root, path);
|
|
2005
|
+
return fromRoot.length > 0 && fromRoot !== ".." && !fromRoot.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) && !isAbsolute2(fromRoot);
|
|
2006
|
+
}
|
|
2007
|
+
function sandboxParentDirectories(paths) {
|
|
2008
|
+
const directories = /* @__PURE__ */ new Set();
|
|
2009
|
+
for (const path of paths) {
|
|
2010
|
+
let parent = dirname(path);
|
|
2011
|
+
while (parent !== "/" && parent !== ".") {
|
|
2012
|
+
directories.add(parent);
|
|
2013
|
+
parent = dirname(parent);
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
return [...directories].sort((left, right) => {
|
|
2017
|
+
const depth = left.split("/").length - right.split("/").length;
|
|
2018
|
+
return depth === 0 ? left.localeCompare(right) : depth;
|
|
2019
|
+
});
|
|
2020
|
+
}
|
|
2021
|
+
function sameStringList(value, expected) {
|
|
2022
|
+
return Array.isArray(value) && value.length === expected.length && value.every((entry, index) => entry === expected[index]);
|
|
2023
|
+
}
|
|
2024
|
+
async function developmentExecutable(input) {
|
|
2025
|
+
if (!isAbsolute2(input) || !/^opengeni-artifact-materializer(?:\.exe)?$/u.test(basename(input))) {
|
|
2026
|
+
throw new Error("Development artifact materializer path is invalid");
|
|
2027
|
+
}
|
|
2028
|
+
const canonical = await realpath(input);
|
|
2029
|
+
const metadata = await stat(canonical);
|
|
2030
|
+
await access(canonical, 1);
|
|
2031
|
+
if (!metadata.isFile()) throw new Error("Development artifact materializer is not executable");
|
|
2032
|
+
return canonical;
|
|
2033
|
+
}
|
|
2034
|
+
function hasResourceLimit(limits, name, expected) {
|
|
2035
|
+
const line = limits.split("\n").find((candidate) => candidate.startsWith(name));
|
|
2036
|
+
if (!line) return false;
|
|
2037
|
+
const fields = line.trim().split(/\s{2,}/u);
|
|
2038
|
+
return fields[1] === expected && fields[2] === expected;
|
|
2039
|
+
}
|
|
2040
|
+
async function boundedStream(stream) {
|
|
2041
|
+
const chunks = [];
|
|
2042
|
+
let total = 0;
|
|
2043
|
+
for await (const value of stream) {
|
|
2044
|
+
const chunk = typeof value === "string" ? Buffer.from(value) : value;
|
|
2045
|
+
total += chunk.byteLength;
|
|
2046
|
+
if (total > MAX_PROBE_BYTES) throw new Error("Artifact materializer sandbox probe overflowed");
|
|
2047
|
+
chunks.push(new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength).slice());
|
|
2048
|
+
}
|
|
2049
|
+
const output = new Uint8Array(total);
|
|
2050
|
+
let offset = 0;
|
|
2051
|
+
for (const chunk of chunks) {
|
|
2052
|
+
output.set(chunk, offset);
|
|
2053
|
+
offset += chunk.byteLength;
|
|
2054
|
+
}
|
|
2055
|
+
return output;
|
|
2056
|
+
}
|
|
2057
|
+
function waitForExit2(child) {
|
|
2058
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
2059
|
+
return Promise.resolve({ code: child.exitCode, signal: child.signalCode });
|
|
2060
|
+
}
|
|
2061
|
+
return new Promise((resolve, reject) => {
|
|
2062
|
+
const onError = (error) => {
|
|
2063
|
+
child.removeListener("exit", onExit);
|
|
2064
|
+
reject(error);
|
|
2065
|
+
};
|
|
2066
|
+
const onExit = (code, signal) => {
|
|
2067
|
+
child.removeListener("error", onError);
|
|
2068
|
+
resolve({ code, signal });
|
|
2069
|
+
};
|
|
2070
|
+
child.once("error", onError);
|
|
2071
|
+
child.once("exit", onExit);
|
|
2072
|
+
});
|
|
2073
|
+
}
|
|
2074
|
+
function terminateChild2(child) {
|
|
2075
|
+
if (child.exitCode !== null || child.signalCode !== null) return;
|
|
2076
|
+
child.stdin.destroy();
|
|
2077
|
+
child.kill("SIGKILL");
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
// src/editable-artifact-materializer-service.ts
|
|
2081
|
+
async function createProductionEditableArtifactMaterializerService(options) {
|
|
2082
|
+
if (options.declaredDatabaseRole !== EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE) {
|
|
2083
|
+
throw new Error("Artifact materializer database role declaration is invalid");
|
|
2084
|
+
}
|
|
2085
|
+
if (typeof options.databaseUrl !== "string" || options.databaseUrl.trim().length === 0) {
|
|
2086
|
+
throw new Error("Artifact materializer requires its dedicated database DSN");
|
|
2087
|
+
}
|
|
2088
|
+
const maxSourceBytes = options.maxSourceBytes ?? EDITABLE_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES;
|
|
2089
|
+
const maxOutputBytes = options.maxOutputBytes ?? EDITABLE_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES;
|
|
2090
|
+
const concurrency = options.concurrency ?? 2;
|
|
2091
|
+
const development = options.runtimeAuthority.mode === "development-current-host";
|
|
2092
|
+
if (development !== (options.allowUnsandboxedLocalDevelopment === true)) {
|
|
2093
|
+
throw new Error("Artifact materializer runtime authority and local-development gate differ");
|
|
2094
|
+
}
|
|
2095
|
+
const launcher = development ? await createDevelopmentEditableArtifactProcessLauncher({
|
|
2096
|
+
materializerExecutable: options.executable,
|
|
2097
|
+
explicitlyEnabled: options.allowUnsandboxedLocalDevelopment === true,
|
|
2098
|
+
nodeEnvironment: options.nodeEnvironment
|
|
2099
|
+
}) : await createLinuxEditableArtifactProcessLauncher({
|
|
2100
|
+
materializerExecutable: options.executable,
|
|
2101
|
+
bubblewrapExecutable: requiredOption(options.bubblewrapExecutable, "bubblewrap"),
|
|
2102
|
+
prlimitExecutable: requiredOption(options.prlimitExecutable, "prlimit"),
|
|
2103
|
+
runtimeRoot: dirname2(options.runtimeAuthority.manifestPath),
|
|
2104
|
+
memoryLimitBytes: options.memoryLimitBytes,
|
|
2105
|
+
cpuTimeLimitMs: options.cpuTimeLimitMs,
|
|
2106
|
+
fileDescriptorLimit: options.fileDescriptorLimit,
|
|
2107
|
+
processLimit: options.processLimit,
|
|
2108
|
+
fileSizeLimitBytes: options.fileSizeLimitBytes
|
|
2109
|
+
});
|
|
2110
|
+
const searchPath = dbSearchPath(options.settings);
|
|
2111
|
+
const dbClient = createDb(options.databaseUrl, {
|
|
2112
|
+
...searchPath ? { searchPath } : {},
|
|
2113
|
+
...options.settings.rlsStrategy ? { rlsStrategy: options.settings.rlsStrategy } : {},
|
|
2114
|
+
max: Math.max(2, concurrency + 1)
|
|
2115
|
+
});
|
|
2116
|
+
let created = false;
|
|
2117
|
+
try {
|
|
2118
|
+
await assertDedicatedMaterializerDatabaseRole(dbClient.db);
|
|
2119
|
+
const objectStorage = options.objectStorage ?? createObjectStorage(options.settings);
|
|
2120
|
+
if (!objectStorage) throw new Error("Artifact materializer object storage is unavailable");
|
|
2121
|
+
const subprocessOptions = {
|
|
2122
|
+
executable: options.executable,
|
|
2123
|
+
launcher,
|
|
2124
|
+
childEnvironment: options.runtimeAuthority.mode === "production" ? Object.freeze({
|
|
2125
|
+
OPENGENI_ARTIFACT_RUNTIME_MANIFEST: options.runtimeAuthority.manifestPath,
|
|
2126
|
+
OPENGENI_ARTIFACT_TOOL_ENTRY: options.skillFacadeEntrypoint
|
|
2127
|
+
}) : Object.freeze({
|
|
2128
|
+
NODE_ENV: "development",
|
|
2129
|
+
OPENGENI_ARTIFACT_DEVELOPMENT_RUNTIME_MANIFEST: options.runtimeAuthority.manifestPath,
|
|
2130
|
+
OPENGENI_ARTIFACT_TOOL_ENTRY: options.skillFacadeEntrypoint
|
|
2131
|
+
}),
|
|
2132
|
+
...development ? { allowUnsandboxedDevelopment: true } : {},
|
|
2133
|
+
wallTimeoutMs: options.wallTimeoutMs,
|
|
2134
|
+
maxSourceBytes,
|
|
2135
|
+
maxOutputBytes
|
|
2136
|
+
};
|
|
2137
|
+
const kernel = await createNativeEditableArtifactSubprocessPort(subprocessOptions);
|
|
2138
|
+
const repository = new PostgresEditableArtifactMaterializationRepository(dbClient.db, {
|
|
2139
|
+
...options.settings.dbSchema.trim() ? { dataSchema: options.settings.dbSchema.trim() } : {}
|
|
2140
|
+
});
|
|
2141
|
+
const observability = createObservability(options.settings, {
|
|
2142
|
+
component: "artifact-materializer"
|
|
2143
|
+
});
|
|
2144
|
+
if (development) {
|
|
2145
|
+
observability.warn("Artifact materializer is running without an OS sandbox", {
|
|
2146
|
+
errorClass: "DevelopmentSecurityPosture",
|
|
2147
|
+
errorCode: "artifact_materializer_unsandboxed_development",
|
|
2148
|
+
origin: "worker",
|
|
2149
|
+
outcome: "degraded"
|
|
2150
|
+
});
|
|
2151
|
+
}
|
|
2152
|
+
const materializer = createConfiguredEditableArtifactMaterializer({
|
|
2153
|
+
enabled: true,
|
|
2154
|
+
databaseRole: options.declaredDatabaseRole,
|
|
2155
|
+
objectStorageConfigured: true,
|
|
2156
|
+
...development ? { allowUnsandboxedDevelopment: true } : {},
|
|
2157
|
+
dependencies: {
|
|
2158
|
+
store: repository,
|
|
2159
|
+
sourceReader: createEditableArtifactObjectStorageReader(objectStorage),
|
|
2160
|
+
outputWriter: createEditableArtifactObjectStorageWriter(objectStorage),
|
|
2161
|
+
outputVerifier: createEditableArtifactObjectStorageVerifier(objectStorage, {
|
|
2162
|
+
verify: kernel.verifyMaterialization
|
|
2163
|
+
}),
|
|
2164
|
+
kernel,
|
|
2165
|
+
scheduler: systemEditableArtifactMaterializerScheduler,
|
|
2166
|
+
clock: systemEditableArtifactMaterializerClock,
|
|
2167
|
+
logger: {
|
|
2168
|
+
warn(message, attributes) {
|
|
2169
|
+
observability.warn(message, {
|
|
2170
|
+
errorClass: "WorkerOperationError",
|
|
2171
|
+
errorCode: materializerFailureErrorCode(attributes) ?? "worker_operation_failed",
|
|
2172
|
+
origin: "worker",
|
|
2173
|
+
outcome: typeof attributes.code === "string" ? attributes.code : "unknown"
|
|
2174
|
+
});
|
|
2175
|
+
}
|
|
2176
|
+
},
|
|
2177
|
+
metrics: {
|
|
2178
|
+
increment(outcome, count = 1) {
|
|
2179
|
+
observability.incrementCounter({
|
|
2180
|
+
name: "opengeni_artifact_materializations_total",
|
|
2181
|
+
help: "Editable artifact materialization attempts by closed outcome.",
|
|
2182
|
+
labels: { outcome },
|
|
2183
|
+
amount: count
|
|
2184
|
+
});
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
},
|
|
2188
|
+
options: {
|
|
2189
|
+
owner: options.owner,
|
|
2190
|
+
...options.batchSize !== void 0 ? { batchSize: options.batchSize } : {},
|
|
2191
|
+
concurrency,
|
|
2192
|
+
...options.leaseDurationMs !== void 0 ? { leaseDurationMs: options.leaseDurationMs } : {},
|
|
2193
|
+
...options.leaseRenewIntervalMs !== void 0 ? { leaseRenewIntervalMs: options.leaseRenewIntervalMs } : {},
|
|
2194
|
+
...options.pollIntervalMs !== void 0 ? { pollIntervalMs: options.pollIntervalMs } : {},
|
|
2195
|
+
...options.maxAttempts !== void 0 ? { maxAttempts: options.maxAttempts } : {},
|
|
2196
|
+
maxSourceBytes,
|
|
2197
|
+
maxOutputBytes
|
|
2198
|
+
}
|
|
2199
|
+
});
|
|
2200
|
+
if (!materializer) throw new Error("Artifact materializer is unexpectedly disabled");
|
|
2201
|
+
created = true;
|
|
2202
|
+
return createEditableArtifactMaterializerService({
|
|
2203
|
+
materializer,
|
|
2204
|
+
db: dbClient.db,
|
|
2205
|
+
closeDatabase: dbClient.close,
|
|
2206
|
+
observability,
|
|
2207
|
+
serviceName: options.settings.serviceName,
|
|
2208
|
+
environment: options.settings.environment,
|
|
2209
|
+
deploymentRevision: options.settings.deploymentRevision,
|
|
2210
|
+
runtimeCapability: Object.freeze({
|
|
2211
|
+
mode: options.runtimeAuthority.mode,
|
|
2212
|
+
sandboxEnforced: launcher.identity.sandboxEnforced
|
|
2213
|
+
})
|
|
2214
|
+
});
|
|
2215
|
+
} finally {
|
|
2216
|
+
if (!created) await dbClient.close();
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
function materializerFailureErrorCode(attributes) {
|
|
2220
|
+
const key = `${String(attributes.failureStage)}:${String(attributes.failureSubcode)}`;
|
|
2221
|
+
switch (key) {
|
|
2222
|
+
case "source_reader:open":
|
|
2223
|
+
return "artifact_materializer_source_open_failed";
|
|
2224
|
+
case "source_reader:content_type":
|
|
2225
|
+
return "artifact_materializer_source_content_type_mismatch";
|
|
2226
|
+
case "source_reader:stream_identity":
|
|
2227
|
+
return "artifact_materializer_source_stream_identity_mismatch";
|
|
2228
|
+
case "source_reader:revalidate":
|
|
2229
|
+
return "artifact_materializer_source_revalidation_failed";
|
|
2230
|
+
case "native:input_framing":
|
|
2231
|
+
return "artifact_materializer_native_input_framing_failed";
|
|
2232
|
+
case "native:snapshot_open":
|
|
2233
|
+
return "artifact_materializer_native_snapshot_open_failed";
|
|
2234
|
+
case "native:state_mismatch":
|
|
2235
|
+
return "artifact_materializer_native_state_mismatch";
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
function createEditableArtifactMaterializerService(input) {
|
|
2239
|
+
let state = "starting";
|
|
2240
|
+
let runPromise = null;
|
|
2241
|
+
let closePromise = null;
|
|
2242
|
+
let databaseClosed = false;
|
|
2243
|
+
const abort = new AbortController();
|
|
2244
|
+
const readinessTimeoutMs = input.readinessTimeoutMs ?? 2e3;
|
|
2245
|
+
const ready = async () => {
|
|
2246
|
+
await withTimeout(assertDedicatedMaterializerDatabaseRole(input.db), readinessTimeoutMs);
|
|
2247
|
+
};
|
|
2248
|
+
const run = () => {
|
|
2249
|
+
if (runPromise) return runPromise;
|
|
2250
|
+
if (state !== "starting") {
|
|
2251
|
+
return Promise.reject(new Error("Artifact materializer service cannot be restarted"));
|
|
2252
|
+
}
|
|
2253
|
+
runPromise = (async () => {
|
|
2254
|
+
try {
|
|
2255
|
+
await ready();
|
|
2256
|
+
if (abort.signal.aborted) {
|
|
2257
|
+
state = "stopped";
|
|
2258
|
+
return;
|
|
2259
|
+
}
|
|
2260
|
+
state = "ready";
|
|
2261
|
+
await input.materializer.run(abort.signal);
|
|
2262
|
+
const exitWasDraining = state === "draining";
|
|
2263
|
+
state = exitWasDraining ? "stopped" : "failed";
|
|
2264
|
+
if (!exitWasDraining) {
|
|
2265
|
+
throw new Error("Artifact materializer loop exited without a drain request");
|
|
2266
|
+
}
|
|
2267
|
+
} catch (error) {
|
|
2268
|
+
const caughtState = state;
|
|
2269
|
+
if (caughtState !== "draining" && caughtState !== "stopped") state = "failed";
|
|
2270
|
+
throw error;
|
|
2271
|
+
}
|
|
2272
|
+
})();
|
|
2273
|
+
return runPromise;
|
|
2274
|
+
};
|
|
2275
|
+
const drain = (_reason = "shutdown") => {
|
|
2276
|
+
if (state === "draining" || state === "stopped" || state === "failed") return false;
|
|
2277
|
+
state = "draining";
|
|
2278
|
+
abort.abort();
|
|
2279
|
+
return true;
|
|
2280
|
+
};
|
|
2281
|
+
const close = () => {
|
|
2282
|
+
if (closePromise) return closePromise;
|
|
2283
|
+
closePromise = (async () => {
|
|
2284
|
+
drain("close");
|
|
2285
|
+
if (runPromise) await runPromise.catch(() => void 0);
|
|
2286
|
+
await input.materializer.drain();
|
|
2287
|
+
if (!databaseClosed) {
|
|
2288
|
+
databaseClosed = true;
|
|
2289
|
+
await input.closeDatabase();
|
|
2290
|
+
}
|
|
2291
|
+
if (state !== "failed") state = "stopped";
|
|
2292
|
+
})();
|
|
2293
|
+
return closePromise;
|
|
2294
|
+
};
|
|
2295
|
+
const fetch = async (request) => {
|
|
2296
|
+
const url = new URL(request.url);
|
|
2297
|
+
if (request.method !== "GET") {
|
|
2298
|
+
return Response.json({ error: "method_not_allowed" }, { status: 405 });
|
|
2299
|
+
}
|
|
2300
|
+
if (url.pathname === "/healthz") {
|
|
2301
|
+
const ok = state !== "failed" && state !== "stopped";
|
|
2302
|
+
return Response.json(
|
|
2303
|
+
{
|
|
2304
|
+
service: input.serviceName,
|
|
2305
|
+
component: "artifact-materializer",
|
|
2306
|
+
environment: input.environment,
|
|
2307
|
+
...input.deploymentRevision ? { deploymentRevision: input.deploymentRevision } : {},
|
|
2308
|
+
runtime: input.runtimeCapability ?? {
|
|
2309
|
+
mode: "production",
|
|
2310
|
+
sandboxEnforced: true
|
|
2311
|
+
},
|
|
2312
|
+
state,
|
|
2313
|
+
ok
|
|
2314
|
+
},
|
|
2315
|
+
{ status: ok ? 200 : 503 }
|
|
2316
|
+
);
|
|
2317
|
+
}
|
|
2318
|
+
if (url.pathname === "/readyz") {
|
|
2319
|
+
if (state !== "ready") {
|
|
2320
|
+
return Response.json(
|
|
2321
|
+
{ ok: false, state, checks: { databaseRole: { ok: false } } },
|
|
2322
|
+
{ status: 503 }
|
|
2323
|
+
);
|
|
2324
|
+
}
|
|
2325
|
+
try {
|
|
2326
|
+
await ready();
|
|
2327
|
+
return Response.json({
|
|
2328
|
+
ok: true,
|
|
2329
|
+
state,
|
|
2330
|
+
runtime: input.runtimeCapability ?? {
|
|
2331
|
+
mode: "production",
|
|
2332
|
+
sandboxEnforced: true
|
|
2333
|
+
},
|
|
2334
|
+
checks: { databaseRole: { ok: true } }
|
|
2335
|
+
});
|
|
2336
|
+
} catch {
|
|
2337
|
+
return Response.json(
|
|
2338
|
+
{ ok: false, state, checks: { databaseRole: { ok: false } } },
|
|
2339
|
+
{ status: 503 }
|
|
2340
|
+
);
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
if (url.pathname === "/metrics") {
|
|
2344
|
+
return new Response(await input.observability.prometheusMetrics(), {
|
|
2345
|
+
status: 200,
|
|
2346
|
+
headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" }
|
|
2347
|
+
});
|
|
2348
|
+
}
|
|
2349
|
+
return Response.json({ error: "not_found" }, { status: 404 });
|
|
2350
|
+
};
|
|
2351
|
+
return Object.freeze({
|
|
2352
|
+
state: () => state,
|
|
2353
|
+
run,
|
|
2354
|
+
drain,
|
|
2355
|
+
close,
|
|
2356
|
+
dispatchOnce: async (signal) => await input.materializer.dispatchOnce(signal),
|
|
2357
|
+
fetch
|
|
2358
|
+
});
|
|
2359
|
+
}
|
|
2360
|
+
async function assertDedicatedMaterializerDatabaseRole(db) {
|
|
2361
|
+
const raw = await db.execute(dbSql`
|
|
2362
|
+
select
|
|
2363
|
+
current_user::text as current_role,
|
|
2364
|
+
session_user::text as session_role,
|
|
2365
|
+
has_table_privilege(
|
|
2366
|
+
current_user,
|
|
2367
|
+
format('%I.editable_artifact_materialization_jobs', current_schema()),
|
|
2368
|
+
'SELECT'
|
|
2369
|
+
) as can_select_jobs,
|
|
2370
|
+
has_table_privilege(
|
|
2371
|
+
current_user,
|
|
2372
|
+
format('%I.editable_artifact_materialization_jobs', current_schema()),
|
|
2373
|
+
'INSERT'
|
|
2374
|
+
) as can_insert_jobs,
|
|
2375
|
+
has_table_privilege(
|
|
2376
|
+
current_user,
|
|
2377
|
+
format('%I.editable_artifact_materialization_jobs', current_schema()),
|
|
2378
|
+
'UPDATE'
|
|
2379
|
+
) as can_update_jobs,
|
|
2380
|
+
has_table_privilege(
|
|
2381
|
+
current_user,
|
|
2382
|
+
format('%I.editable_artifact_materialization_jobs', current_schema()),
|
|
2383
|
+
'DELETE'
|
|
2384
|
+
) as can_delete_jobs
|
|
2385
|
+
`);
|
|
2386
|
+
const rows = Array.isArray(raw) ? raw : raw.rows ?? [];
|
|
2387
|
+
const row = rows[0];
|
|
2388
|
+
if (row?.current_role !== EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE || row.session_role !== EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE || row.can_select_jobs !== false || row.can_insert_jobs !== false || row.can_update_jobs !== false || row.can_delete_jobs !== false) {
|
|
2389
|
+
throw new Error("Artifact materializer database connection has an unsafe role posture");
|
|
2390
|
+
}
|
|
2391
|
+
}
|
|
2392
|
+
function readEditableArtifactMaterializerSidecarEnvironment(environment = process.env) {
|
|
2393
|
+
if (environment.OPENGENI_ARTIFACT_MATERIALIZER_ENABLED !== "true") return null;
|
|
2394
|
+
const databaseUrl = required(environment, "OPENGENI_ARTIFACT_MATERIALIZER_DATABASE_URL");
|
|
2395
|
+
const declaredDatabaseRole = required(
|
|
2396
|
+
environment,
|
|
2397
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_DATABASE_ROLE"
|
|
2398
|
+
);
|
|
2399
|
+
if (declaredDatabaseRole !== EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE) {
|
|
2400
|
+
throw new Error("OPENGENI_ARTIFACT_MATERIALIZER_DATABASE_ROLE is invalid");
|
|
2401
|
+
}
|
|
2402
|
+
const productionManifest = optional(environment, "OPENGENI_ARTIFACT_RUNTIME_MANIFEST");
|
|
2403
|
+
const developmentManifest = optional(
|
|
2404
|
+
environment,
|
|
2405
|
+
"OPENGENI_ARTIFACT_DEVELOPMENT_RUNTIME_MANIFEST"
|
|
2406
|
+
);
|
|
2407
|
+
if (productionManifest && developmentManifest) {
|
|
2408
|
+
throw new Error(
|
|
2409
|
+
"Production and development artifact runtime manifests cannot both be configured"
|
|
2410
|
+
);
|
|
2411
|
+
}
|
|
2412
|
+
if (!productionManifest && !developmentManifest) {
|
|
2413
|
+
throw new Error(
|
|
2414
|
+
"An artifact runtime manifest is required when artifact materialization is enabled"
|
|
2415
|
+
);
|
|
2416
|
+
}
|
|
2417
|
+
const development = developmentManifest !== null;
|
|
2418
|
+
if (development) {
|
|
2419
|
+
if (environment.NODE_ENV === "production") {
|
|
2420
|
+
throw new Error("Development artifact runtime is forbidden when NODE_ENV=production");
|
|
2421
|
+
}
|
|
2422
|
+
if (environment.OPENGENI_ARTIFACT_LOCAL_DEVELOPMENT !== "true" || environment.OPENGENI_ARTIFACT_MATERIALIZER_UNSANDBOXED_DEVELOPMENT !== "true") {
|
|
2423
|
+
throw new Error(
|
|
2424
|
+
"Unsandboxed artifact materialization requires explicit local-development gates"
|
|
2425
|
+
);
|
|
2426
|
+
}
|
|
2427
|
+
assertLoopbackDatabaseUrl(databaseUrl);
|
|
2428
|
+
assertLoopbackObjectStorageUrl(required(environment, "OPENGENI_OBJECT_STORAGE_ENDPOINT"));
|
|
2429
|
+
}
|
|
2430
|
+
const httpHostname = optional(environment, "OPENGENI_ARTIFACT_MATERIALIZER_HTTP_HOST") ?? "0.0.0.0";
|
|
2431
|
+
if (development && !isLoopbackHostname(httpHostname)) {
|
|
2432
|
+
throw new Error("Unsandboxed artifact materialization requires a loopback HTTP listener");
|
|
2433
|
+
}
|
|
2434
|
+
const runtimeAuthority = developmentManifest ? Object.freeze({
|
|
2435
|
+
mode: "development-current-host",
|
|
2436
|
+
environmentVariable: "OPENGENI_ARTIFACT_DEVELOPMENT_RUNTIME_MANIFEST",
|
|
2437
|
+
manifestPath: developmentManifest
|
|
2438
|
+
}) : Object.freeze({
|
|
2439
|
+
mode: "production",
|
|
2440
|
+
environmentVariable: "OPENGENI_ARTIFACT_RUNTIME_MANIFEST",
|
|
2441
|
+
manifestPath: productionManifest
|
|
2442
|
+
});
|
|
2443
|
+
return Object.freeze({
|
|
2444
|
+
databaseUrl,
|
|
2445
|
+
declaredDatabaseRole,
|
|
2446
|
+
executable: required(environment, "OPENGENI_ARTIFACT_MATERIALIZER_EXECUTABLE"),
|
|
2447
|
+
bubblewrapExecutable: development ? null : required(environment, "OPENGENI_ARTIFACT_MATERIALIZER_BWRAP"),
|
|
2448
|
+
prlimitExecutable: development ? null : required(environment, "OPENGENI_ARTIFACT_MATERIALIZER_PRLIMIT"),
|
|
2449
|
+
runtimeAuthority,
|
|
2450
|
+
allowUnsandboxedLocalDevelopment: development,
|
|
2451
|
+
skillFacadeEntrypoint: required(environment, "OPENGENI_ARTIFACT_TOOL_ENTRY"),
|
|
2452
|
+
owner: environment.OPENGENI_ARTIFACT_MATERIALIZER_OWNER?.trim() || `materializer-${process.pid}-${crypto.randomUUID()}`,
|
|
2453
|
+
httpHostname,
|
|
2454
|
+
httpPort: integer(environment, "OPENGENI_ARTIFACT_MATERIALIZER_HTTP_PORT", 9465, 1, 65535),
|
|
2455
|
+
batchSize: integer(environment, "OPENGENI_ARTIFACT_MATERIALIZER_BATCH_SIZE", 8, 1, 64),
|
|
2456
|
+
concurrency: integer(environment, "OPENGENI_ARTIFACT_MATERIALIZER_CONCURRENCY", 2, 1, 64),
|
|
2457
|
+
leaseDurationMs: integer(
|
|
2458
|
+
environment,
|
|
2459
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_LEASE_MS",
|
|
2460
|
+
12e4,
|
|
2461
|
+
1e4,
|
|
2462
|
+
864e5
|
|
2463
|
+
),
|
|
2464
|
+
leaseRenewIntervalMs: integer(
|
|
2465
|
+
environment,
|
|
2466
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_RENEW_MS",
|
|
2467
|
+
3e4,
|
|
2468
|
+
1e3,
|
|
2469
|
+
43199999
|
|
2470
|
+
),
|
|
2471
|
+
pollIntervalMs: integer(
|
|
2472
|
+
environment,
|
|
2473
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_POLL_MS",
|
|
2474
|
+
1e3,
|
|
2475
|
+
10,
|
|
2476
|
+
6e4
|
|
2477
|
+
),
|
|
2478
|
+
maxAttempts: integer(environment, "OPENGENI_ARTIFACT_MATERIALIZER_MAX_ATTEMPTS", 20, 1, 1e3),
|
|
2479
|
+
memoryLimitBytes: integer(
|
|
2480
|
+
environment,
|
|
2481
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_MEMORY_BYTES",
|
|
2482
|
+
512 * 1024 * 1024,
|
|
2483
|
+
64 * 1024 * 1024,
|
|
2484
|
+
8 * 1024 * 1024 * 1024
|
|
2485
|
+
),
|
|
2486
|
+
cpuTimeLimitMs: integer(
|
|
2487
|
+
environment,
|
|
2488
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_CPU_MS",
|
|
2489
|
+
6e4,
|
|
2490
|
+
1e3,
|
|
2491
|
+
36e5
|
|
2492
|
+
),
|
|
2493
|
+
fileDescriptorLimit: integer(
|
|
2494
|
+
environment,
|
|
2495
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_OPEN_FILES",
|
|
2496
|
+
64,
|
|
2497
|
+
16,
|
|
2498
|
+
4096
|
|
2499
|
+
),
|
|
2500
|
+
processLimit: integer(environment, "OPENGENI_ARTIFACT_MATERIALIZER_PROCESSES", 64, 1, 1024),
|
|
2501
|
+
fileSizeLimitBytes: integer(
|
|
2502
|
+
environment,
|
|
2503
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_FILE_BYTES",
|
|
2504
|
+
512 * 1024 * 1024,
|
|
2505
|
+
1024 * 1024,
|
|
2506
|
+
8 * 1024 * 1024 * 1024
|
|
2507
|
+
),
|
|
2508
|
+
wallTimeoutMs: integer(
|
|
2509
|
+
environment,
|
|
2510
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_WALL_MS",
|
|
2511
|
+
12e4,
|
|
2512
|
+
1e3,
|
|
2513
|
+
36e5
|
|
2514
|
+
),
|
|
2515
|
+
maxSourceBytes: integer(
|
|
2516
|
+
environment,
|
|
2517
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES",
|
|
2518
|
+
EDITABLE_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES,
|
|
2519
|
+
1,
|
|
2520
|
+
EDITABLE_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES
|
|
2521
|
+
),
|
|
2522
|
+
maxOutputBytes: integer(
|
|
2523
|
+
environment,
|
|
2524
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES",
|
|
2525
|
+
EDITABLE_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES,
|
|
2526
|
+
1,
|
|
2527
|
+
EDITABLE_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES
|
|
2528
|
+
),
|
|
2529
|
+
...environment.NODE_ENV === void 0 ? {} : { nodeEnvironment: environment.NODE_ENV }
|
|
2530
|
+
});
|
|
2531
|
+
}
|
|
2532
|
+
async function createMaterializerSidecarFromEnvironment(environment = process.env) {
|
|
2533
|
+
const sidecar = readEditableArtifactMaterializerSidecarEnvironment(environment);
|
|
2534
|
+
if (!sidecar) return null;
|
|
2535
|
+
const settings = getSettings();
|
|
2536
|
+
const service = await createProductionEditableArtifactMaterializerService({
|
|
2537
|
+
settings,
|
|
2538
|
+
databaseUrl: sidecar.databaseUrl,
|
|
2539
|
+
declaredDatabaseRole: sidecar.declaredDatabaseRole,
|
|
2540
|
+
executable: sidecar.executable,
|
|
2541
|
+
bubblewrapExecutable: sidecar.bubblewrapExecutable,
|
|
2542
|
+
prlimitExecutable: sidecar.prlimitExecutable,
|
|
2543
|
+
runtimeAuthority: sidecar.runtimeAuthority,
|
|
2544
|
+
...sidecar.allowUnsandboxedLocalDevelopment ? { allowUnsandboxedLocalDevelopment: true } : {},
|
|
2545
|
+
skillFacadeEntrypoint: sidecar.skillFacadeEntrypoint,
|
|
2546
|
+
owner: sidecar.owner,
|
|
2547
|
+
batchSize: sidecar.batchSize,
|
|
2548
|
+
concurrency: sidecar.concurrency,
|
|
2549
|
+
leaseDurationMs: sidecar.leaseDurationMs,
|
|
2550
|
+
leaseRenewIntervalMs: sidecar.leaseRenewIntervalMs,
|
|
2551
|
+
pollIntervalMs: sidecar.pollIntervalMs,
|
|
2552
|
+
maxAttempts: sidecar.maxAttempts,
|
|
2553
|
+
memoryLimitBytes: sidecar.memoryLimitBytes,
|
|
2554
|
+
cpuTimeLimitMs: sidecar.cpuTimeLimitMs,
|
|
2555
|
+
fileDescriptorLimit: sidecar.fileDescriptorLimit,
|
|
2556
|
+
processLimit: sidecar.processLimit,
|
|
2557
|
+
fileSizeLimitBytes: sidecar.fileSizeLimitBytes,
|
|
2558
|
+
wallTimeoutMs: sidecar.wallTimeoutMs,
|
|
2559
|
+
maxSourceBytes: sidecar.maxSourceBytes,
|
|
2560
|
+
maxOutputBytes: sidecar.maxOutputBytes,
|
|
2561
|
+
...sidecar.nodeEnvironment === void 0 ? {} : { nodeEnvironment: sidecar.nodeEnvironment }
|
|
2562
|
+
});
|
|
2563
|
+
return Object.freeze({
|
|
2564
|
+
service,
|
|
2565
|
+
httpHostname: sidecar.httpHostname,
|
|
2566
|
+
httpPort: sidecar.httpPort
|
|
2567
|
+
});
|
|
2568
|
+
}
|
|
2569
|
+
async function withTimeout(promise, milliseconds) {
|
|
2570
|
+
let timer;
|
|
2571
|
+
try {
|
|
2572
|
+
return await Promise.race([
|
|
2573
|
+
promise,
|
|
2574
|
+
new Promise((_resolve, reject) => {
|
|
2575
|
+
timer = setTimeout(() => reject(new Error("readiness check timed out")), milliseconds);
|
|
2576
|
+
})
|
|
2577
|
+
]);
|
|
2578
|
+
} finally {
|
|
2579
|
+
if (timer) clearTimeout(timer);
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2582
|
+
function required(environment, name) {
|
|
2583
|
+
const value = environment[name]?.trim();
|
|
2584
|
+
if (!value) throw new Error(`${name} is required when artifact materialization is enabled`);
|
|
2585
|
+
return value;
|
|
2586
|
+
}
|
|
2587
|
+
function optional(environment, name) {
|
|
2588
|
+
const value = environment[name]?.trim();
|
|
2589
|
+
return value ? value : null;
|
|
2590
|
+
}
|
|
2591
|
+
function requiredOption(value, name) {
|
|
2592
|
+
if (!value) throw new Error(`Artifact materializer ${name} is required in production`);
|
|
2593
|
+
return value;
|
|
2594
|
+
}
|
|
2595
|
+
function assertLoopbackDatabaseUrl(databaseUrl) {
|
|
2596
|
+
let parsed;
|
|
2597
|
+
try {
|
|
2598
|
+
parsed = new URL(databaseUrl);
|
|
2599
|
+
} catch {
|
|
2600
|
+
throw new Error("Local-development artifact materializer database DSN is invalid");
|
|
2601
|
+
}
|
|
2602
|
+
if (parsed.protocol !== "postgres:" && parsed.protocol !== "postgresql:") {
|
|
2603
|
+
throw new Error("Local-development artifact materializer database DSN is invalid");
|
|
2604
|
+
}
|
|
2605
|
+
if (!isLoopbackHostname(parsed.hostname)) {
|
|
2606
|
+
throw new Error("Unsandboxed artifact materialization requires a loopback database DSN");
|
|
2607
|
+
}
|
|
2608
|
+
}
|
|
2609
|
+
function assertLoopbackObjectStorageUrl(endpoint) {
|
|
2610
|
+
let parsed;
|
|
2611
|
+
try {
|
|
2612
|
+
parsed = new URL(endpoint);
|
|
2613
|
+
} catch {
|
|
2614
|
+
throw new Error("Local-development artifact object-storage endpoint is invalid");
|
|
2615
|
+
}
|
|
2616
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:" || !isLoopbackHostname(parsed.hostname)) {
|
|
2617
|
+
throw new Error("Unsandboxed artifact materialization requires loopback object storage");
|
|
2618
|
+
}
|
|
2619
|
+
}
|
|
2620
|
+
function isLoopbackHostname(hostname) {
|
|
2621
|
+
const normalized = hostname.toLowerCase();
|
|
2622
|
+
return normalized === "localhost" || normalized === "127.0.0.1" || normalized === "[::1]";
|
|
2623
|
+
}
|
|
2624
|
+
function integer(environment, name, fallback, minimum, maximum) {
|
|
2625
|
+
const raw = environment[name];
|
|
2626
|
+
const value = raw === void 0 ? fallback : Number(raw);
|
|
2627
|
+
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
|
2628
|
+
throw new Error(`${name} is outside its allowed range`);
|
|
2629
|
+
}
|
|
2630
|
+
return value;
|
|
2631
|
+
}
|
|
2632
|
+
|
|
2633
|
+
// src/artifact-materializer-entry.ts
|
|
2634
|
+
async function runEditableArtifactMaterializerSidecar(environment = process.env) {
|
|
2635
|
+
const configured = await createMaterializerSidecarFromEnvironment(environment);
|
|
2636
|
+
if (!configured) {
|
|
2637
|
+
throw new Error("Editable artifact materializer sidecar is disabled");
|
|
2638
|
+
}
|
|
2639
|
+
const server = Bun.serve({
|
|
2640
|
+
hostname: configured.httpHostname,
|
|
2641
|
+
port: configured.httpPort,
|
|
2642
|
+
fetch: configured.service.fetch
|
|
2643
|
+
});
|
|
2644
|
+
const stop = () => configured.service.drain("process signal");
|
|
2645
|
+
process.on("SIGTERM", stop);
|
|
2646
|
+
process.on("SIGINT", stop);
|
|
2647
|
+
try {
|
|
2648
|
+
await configured.service.run();
|
|
2649
|
+
} finally {
|
|
2650
|
+
process.off("SIGTERM", stop);
|
|
2651
|
+
process.off("SIGINT", stop);
|
|
2652
|
+
server.stop(true);
|
|
2653
|
+
await configured.service.close();
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
if (import.meta.main) {
|
|
2657
|
+
await runEditableArtifactMaterializerSidecar();
|
|
2658
|
+
}
|
|
2659
|
+
export {
|
|
2660
|
+
runEditableArtifactMaterializerSidecar
|
|
2661
|
+
};
|
|
2662
|
+
//# sourceMappingURL=artifact-materializer-entry.js.map
|