nebula-notebook 0.2.15 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{errorwidget-B2XZ3hoa.js → errorwidget-t1RLHQYw.js} +1 -1
- package/dist/assets/{index-DqHzn4vg.js → index-BMuaZuus.js} +1 -1
- package/dist/assets/index-Bvhz0ltE.css +32 -0
- package/dist/assets/{index-NfxQwPBq.js → index-Cq1BG1_T.js} +205 -200
- package/dist/assets/{index-a9gKLTlv.js → index-DTGJj27s.js} +1 -1
- package/dist/assets/{index-CV0ZPmTH.js → index-DzeeB-u3.js} +1 -1
- package/dist/assets/{services-shim-D_SeYcB9.js → services-shim-Dg9Zi6xv.js} +1 -1
- package/dist/index.html +2 -2
- package/node-server/dist/auth/auth-middleware.js +3 -1
- package/node-server/dist/cluster/kernel-proxy.d.ts +15 -0
- package/node-server/dist/cluster/kernel-proxy.js +75 -0
- package/node-server/dist/fs/fs-service.js +104 -8
- package/node-server/dist/fs/index.d.ts +1 -0
- package/node-server/dist/fs/index.js +1 -0
- package/node-server/dist/fs/sealed-path.d.ts +41 -0
- package/node-server/dist/fs/sealed-path.js +228 -0
- package/node-server/dist/fs/types.d.ts +4 -2
- package/node-server/dist/fs/types.js +2 -2
- package/node-server/dist/index.js +24 -13
- package/node-server/dist/kernel/kernel-service.d.ts +2 -2
- package/node-server/dist/kernel/kernel-service.js +15 -5
- package/node-server/dist/kernel/types.d.ts +16 -0
- package/node-server/dist/notebook/headless-handler.d.ts +10 -0
- package/node-server/dist/notebook/headless-handler.js +64 -10
- package/node-server/dist/notebook/operation-router.js +13 -2
- package/node-server/dist/provenance/canonical-json.d.ts +36 -0
- package/node-server/dist/provenance/canonical-json.js +218 -0
- package/node-server/dist/provenance/provenance-store.d.ts +63 -0
- package/node-server/dist/provenance/provenance-store.js +598 -0
- package/node-server/dist/provenance/replay-seal-service.d.ts +223 -0
- package/node-server/dist/provenance/replay-seal-service.js +1702 -0
- package/node-server/dist/provenance/types.d.ts +65 -0
- package/node-server/dist/provenance/types.js +2 -0
- package/node-server/dist/routes/fs.js +27 -0
- package/node-server/dist/routes/kernel.js +15 -0
- package/node-server/dist/routes/notebook.js +54 -0
- package/node-server/dist/routes/replay-seal.d.ts +9 -0
- package/node-server/dist/routes/replay-seal.js +66 -0
- package/node-server/dist/scheduler/allocation-service.d.ts +3 -2
- package/node-server/dist/scheduler/allocation-service.js +45 -2
- package/node-server/dist/scheduler/arch.d.ts +30 -0
- package/node-server/dist/scheduler/arch.js +113 -0
- package/node-server/dist/scheduler/job-template.d.ts +11 -0
- package/node-server/dist/scheduler/mock-scheduler.d.ts +1 -0
- package/node-server/dist/scheduler/mock-scheduler.js +3 -0
- package/node-server/dist/scheduler/slurm-scheduler.d.ts +1 -0
- package/node-server/dist/scheduler/slurm-scheduler.js +34 -0
- package/node-server/dist/scheduler/types.d.ts +9 -0
- package/node-server/dist/scheduler/util.d.ts +6 -0
- package/node-server/dist/scheduler/util.js +16 -0
- package/node-server/dist/server/bind-host.d.ts +12 -0
- package/node-server/dist/server/bind-host.js +56 -0
- package/node-server/dist/server/cors-origin.d.ts +1 -0
- package/node-server/dist/server/cors-origin.js +32 -0
- package/package.json +1 -1
- package/dist/assets/index-Czch8hB-.css +0 -32
|
@@ -0,0 +1,1702 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.ReplaySealService = exports.ReplaySealConflictError = exports.ReplaySealValidationError = void 0;
|
|
37
|
+
exports.normalizeNotebookOutputsForComparison = normalizeNotebookOutputsForComparison;
|
|
38
|
+
const crypto_1 = require("crypto");
|
|
39
|
+
const fs_1 = require("fs");
|
|
40
|
+
const fsp = __importStar(require("fs/promises"));
|
|
41
|
+
const path = __importStar(require("path"));
|
|
42
|
+
const canonical_json_1 = require("./canonical-json");
|
|
43
|
+
const provenance_store_1 = require("./provenance-store");
|
|
44
|
+
const SHA256_RE = /^[a-f0-9]{64}$/;
|
|
45
|
+
const CELL_ID_RE = /^[A-Za-z0-9_-]{1,64}$/;
|
|
46
|
+
const SEAL_ID_RE = /^seal-[a-f0-9]{32}$/;
|
|
47
|
+
const PROBE_BEGIN = '__NEBULA_RUNTIME_ENV_BEGIN__';
|
|
48
|
+
const PROBE_END = '__NEBULA_RUNTIME_ENV_END__';
|
|
49
|
+
const OUTPUT_LIMIT_MARKER = 'Output limit reached';
|
|
50
|
+
class ReplaySealValidationError extends Error {
|
|
51
|
+
code = 'REPLAY_SEAL_VALIDATION_ERROR';
|
|
52
|
+
constructor(message) {
|
|
53
|
+
super(message);
|
|
54
|
+
this.name = 'ReplaySealValidationError';
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
exports.ReplaySealValidationError = ReplaySealValidationError;
|
|
58
|
+
class ReplaySealConflictError extends Error {
|
|
59
|
+
code = 'IDEMPOTENCY_CONFLICT';
|
|
60
|
+
constructor() {
|
|
61
|
+
super('Idempotency-Key was already used with a different replay-seal request');
|
|
62
|
+
this.name = 'ReplaySealConflictError';
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
exports.ReplaySealConflictError = ReplaySealConflictError;
|
|
66
|
+
/**
|
|
67
|
+
* Normalize only nbformat's equivalent text encodings for replay comparison.
|
|
68
|
+
*
|
|
69
|
+
* - stream.text: an array of strings is joined, a string is unchanged;
|
|
70
|
+
* - data values for text/*, application/javascript, and image/svg+xml use the
|
|
71
|
+
* same array-to-string join;
|
|
72
|
+
* - every other JSON value, including binary MIME arrays and traceback, is
|
|
73
|
+
* preserved exactly.
|
|
74
|
+
*
|
|
75
|
+
* This is intentionally separate from outputs_sha256, which hashes the exact
|
|
76
|
+
* output JSON written to the immutable executed notebook.
|
|
77
|
+
*/
|
|
78
|
+
function normalizeNotebookOutputsForComparison(value) {
|
|
79
|
+
if (!Array.isArray(value)) {
|
|
80
|
+
throw new ReplaySealValidationError('Notebook outputs must be an array');
|
|
81
|
+
}
|
|
82
|
+
const cloned = JSON.parse((0, canonical_json_1.canonicalJson)(value));
|
|
83
|
+
for (const rawOutput of cloned) {
|
|
84
|
+
if (!rawOutput || typeof rawOutput !== 'object' || Array.isArray(rawOutput))
|
|
85
|
+
continue;
|
|
86
|
+
const output = rawOutput;
|
|
87
|
+
if (output.output_type === 'stream' && 'text' in output) {
|
|
88
|
+
output.text = joinTextArray(output.text, 'stream.text');
|
|
89
|
+
}
|
|
90
|
+
const data = output.data;
|
|
91
|
+
if (data && typeof data === 'object' && !Array.isArray(data)) {
|
|
92
|
+
for (const [mimeType, mimeValue] of Object.entries(data)) {
|
|
93
|
+
if (mimeType.startsWith('text/')
|
|
94
|
+
|| mimeType === 'application/javascript'
|
|
95
|
+
|| mimeType === 'image/svg+xml') {
|
|
96
|
+
data[mimeType] = joinTextArray(mimeValue, `data.${mimeType}`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return cloned;
|
|
102
|
+
}
|
|
103
|
+
function joinTextArray(value, field) {
|
|
104
|
+
if (typeof value === 'string')
|
|
105
|
+
return value;
|
|
106
|
+
if (Array.isArray(value) && value.every((item) => typeof item === 'string')) {
|
|
107
|
+
return value.join('');
|
|
108
|
+
}
|
|
109
|
+
throw new ReplaySealValidationError(`${field} must be a string or an array of strings`);
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Fresh-kernel, top-to-bottom replay and immutable evidence sealing.
|
|
113
|
+
*
|
|
114
|
+
* The service deliberately makes no network/filesystem-isolation claim. A
|
|
115
|
+
* dedicated kernel prevents state inheritance from an exploratory session;
|
|
116
|
+
* the immutable copies, hashes, and server-authored event prefix bind what ran.
|
|
117
|
+
*/
|
|
118
|
+
class ReplaySealService {
|
|
119
|
+
static queues = new Map();
|
|
120
|
+
rootDirectory;
|
|
121
|
+
registryDirectory;
|
|
122
|
+
kernelService;
|
|
123
|
+
provenanceStore;
|
|
124
|
+
durable;
|
|
125
|
+
idFactory;
|
|
126
|
+
clock;
|
|
127
|
+
processInstanceId = (0, crypto_1.randomUUID)();
|
|
128
|
+
activeJobs = new Map();
|
|
129
|
+
attemptedJobs = new Set();
|
|
130
|
+
constructor(options) {
|
|
131
|
+
if (!options || typeof options !== 'object') {
|
|
132
|
+
throw new TypeError('ReplaySealService options are required');
|
|
133
|
+
}
|
|
134
|
+
this.rootDirectory = path.resolve(options.rootDirectory);
|
|
135
|
+
this.registryDirectory = path.join(this.rootDirectory, '.nebula', 'replay-seal-jobs');
|
|
136
|
+
this.kernelService = options.kernelService;
|
|
137
|
+
this.provenanceStore = options.provenanceStore ?? new provenance_store_1.ProvenanceStore();
|
|
138
|
+
this.durable = options.durable !== false;
|
|
139
|
+
this.idFactory = options.idFactory ?? crypto_1.randomUUID;
|
|
140
|
+
this.clock = options.clock ?? (() => new Date());
|
|
141
|
+
}
|
|
142
|
+
async submit(rawRequest, idempotencyKey) {
|
|
143
|
+
const request = this.normalizeRequest(rawRequest);
|
|
144
|
+
this.validateIdempotencyKey(idempotencyKey);
|
|
145
|
+
await this.ensureRegistryDirectory();
|
|
146
|
+
const sealId = `seal-${(0, canonical_json_1.sha256Hex)(idempotencyKey).slice(0, 32)}`;
|
|
147
|
+
const requestFingerprint = (0, canonical_json_1.sha256Canonical)(request);
|
|
148
|
+
const registryPath = this.registryPath(sealId);
|
|
149
|
+
const job = await this.runSerialized(`seal:${registryPath}`, async () => {
|
|
150
|
+
const existing = await this.readJob(registryPath);
|
|
151
|
+
if (existing) {
|
|
152
|
+
if (existing.requestFingerprint !== requestFingerprint) {
|
|
153
|
+
throw new ReplaySealConflictError();
|
|
154
|
+
}
|
|
155
|
+
return existing;
|
|
156
|
+
}
|
|
157
|
+
const sourceBytes = await this.readNotebookSource(request.notebook_path);
|
|
158
|
+
const sourceSha256 = (0, canonical_json_1.sha256Hex)(sourceBytes);
|
|
159
|
+
if (sourceSha256 !== request.source_notebook_sha256) {
|
|
160
|
+
throw new ReplaySealValidationError(`source_notebook_sha256 mismatch: expected ${request.source_notebook_sha256}, got ${sourceSha256}`);
|
|
161
|
+
}
|
|
162
|
+
const frozen = this.parseFrozenNotebook(sourceBytes, request.kernel_name);
|
|
163
|
+
this.validateEnvironmentDeclaration(request.environment_manifest, path.dirname(request.notebook_path));
|
|
164
|
+
await this.preflightArtifacts(request);
|
|
165
|
+
const sealDirectory = await this.createSafeSealDirectory(path.dirname(request.notebook_path), sealId);
|
|
166
|
+
const frozenNotebookPath = path.join(sealDirectory, 'source.ipynb');
|
|
167
|
+
await this.writeImmutable(frozenNotebookPath, sourceBytes);
|
|
168
|
+
const now = this.clock().toISOString();
|
|
169
|
+
const created = {
|
|
170
|
+
schemaVersion: 1,
|
|
171
|
+
sealId,
|
|
172
|
+
status: 'pending',
|
|
173
|
+
requestFingerprint,
|
|
174
|
+
request,
|
|
175
|
+
sourceNotebookSha256: sourceSha256,
|
|
176
|
+
frozenNotebookPath,
|
|
177
|
+
sealDirectory,
|
|
178
|
+
processInstanceId: this.processInstanceId,
|
|
179
|
+
createdAt: now,
|
|
180
|
+
updatedAt: now,
|
|
181
|
+
};
|
|
182
|
+
await this.persistJob(created);
|
|
183
|
+
return created;
|
|
184
|
+
});
|
|
185
|
+
if (job.status === 'pending' || job.status === 'running') {
|
|
186
|
+
if (job.processInstanceId !== this.processInstanceId) {
|
|
187
|
+
await this.failJob(job, new Error('Nebula restarted during replay; automatic re-execution is refused because notebook code may have external side effects'));
|
|
188
|
+
const interrupted = await this.readJob(registryPath);
|
|
189
|
+
if (!interrupted)
|
|
190
|
+
throw new Error(`Interrupted replay job disappeared: ${sealId}`);
|
|
191
|
+
return this.verifiedResponseFor(interrupted);
|
|
192
|
+
}
|
|
193
|
+
this.ensureRunning(job);
|
|
194
|
+
}
|
|
195
|
+
return this.verifiedResponseFor(job);
|
|
196
|
+
}
|
|
197
|
+
async getStatus(sealId) {
|
|
198
|
+
if (!SEAL_ID_RE.test(sealId))
|
|
199
|
+
return null;
|
|
200
|
+
await this.ensureRegistryDirectory();
|
|
201
|
+
const job = await this.readJob(this.registryPath(sealId));
|
|
202
|
+
if (!job)
|
|
203
|
+
return null;
|
|
204
|
+
if (job.status === 'pending' || job.status === 'running') {
|
|
205
|
+
if (job.processInstanceId !== this.processInstanceId) {
|
|
206
|
+
await this.failJob(job, new Error('Nebula restarted during replay; automatic re-execution is refused because notebook code may have external side effects'));
|
|
207
|
+
const interrupted = await this.readJob(this.registryPath(sealId));
|
|
208
|
+
return interrupted ? this.responseFor(interrupted) : null;
|
|
209
|
+
}
|
|
210
|
+
this.ensureRunning(job);
|
|
211
|
+
}
|
|
212
|
+
return this.verifiedResponseFor(job);
|
|
213
|
+
}
|
|
214
|
+
ensureRunning(job) {
|
|
215
|
+
if (this.activeJobs.has(job.sealId) || this.attemptedJobs.has(job.sealId))
|
|
216
|
+
return;
|
|
217
|
+
this.attemptedJobs.add(job.sealId);
|
|
218
|
+
const run = this.runJob(job).catch(async (error) => {
|
|
219
|
+
await this.failJob(job, error).catch(() => undefined);
|
|
220
|
+
}).finally(() => {
|
|
221
|
+
if (this.activeJobs.get(job.sealId) === run)
|
|
222
|
+
this.activeJobs.delete(job.sealId);
|
|
223
|
+
});
|
|
224
|
+
this.activeJobs.set(job.sealId, run);
|
|
225
|
+
}
|
|
226
|
+
async runJob(initialJob) {
|
|
227
|
+
let job = await this.updateJob(initialJob, { status: 'running', error: undefined });
|
|
228
|
+
const request = job.request;
|
|
229
|
+
const actor = { kind: 'agent', id: request.source_id, name: 'kosmos' };
|
|
230
|
+
const inputManifestSha256 = (0, canonical_json_1.sha256Canonical)(request.input_manifest);
|
|
231
|
+
const environmentManifestSha256 = (0, canonical_json_1.sha256Canonical)(request.environment_manifest);
|
|
232
|
+
await this.provenanceStore.putBlob(request.notebook_path, (0, canonical_json_1.hashCanonicalJson)(request.input_manifest));
|
|
233
|
+
await this.provenanceStore.putBlob(request.notebook_path, (0, canonical_json_1.hashCanonicalJson)(request.environment_manifest));
|
|
234
|
+
const frozenBytes = await fsp.readFile(job.frozenNotebookPath);
|
|
235
|
+
if ((0, canonical_json_1.sha256Hex)(frozenBytes) !== job.sourceNotebookSha256) {
|
|
236
|
+
throw new Error('Frozen source notebook failed hash verification');
|
|
237
|
+
}
|
|
238
|
+
const frozen = this.parseFrozenNotebook(frozenBytes, request.kernel_name);
|
|
239
|
+
const commonEvent = {
|
|
240
|
+
actor,
|
|
241
|
+
runId: request.run_id,
|
|
242
|
+
taskId: request.task_id,
|
|
243
|
+
sourceId: request.source_id,
|
|
244
|
+
inputManifestSha256,
|
|
245
|
+
environmentManifestSha256,
|
|
246
|
+
};
|
|
247
|
+
await this.provenanceStore.append(request.notebook_path, {
|
|
248
|
+
...commonEvent,
|
|
249
|
+
type: 'seal.started',
|
|
250
|
+
idempotencyKey: this.sealRequestEventKey(job.sealId),
|
|
251
|
+
payload: {
|
|
252
|
+
schemaVersion: 1,
|
|
253
|
+
sealId: job.sealId,
|
|
254
|
+
sourceNotebook: {
|
|
255
|
+
path: request.notebook_path,
|
|
256
|
+
sha256: job.sourceNotebookSha256,
|
|
257
|
+
},
|
|
258
|
+
kernelName: frozen.kernelName,
|
|
259
|
+
artifacts: request.artifacts.map((artifact) => ({
|
|
260
|
+
path: artifact.path,
|
|
261
|
+
sha256: artifact.sha256,
|
|
262
|
+
sizeBytes: artifact.size_bytes,
|
|
263
|
+
})),
|
|
264
|
+
isolation: {
|
|
265
|
+
freshKernel: true,
|
|
266
|
+
networkIsolated: false,
|
|
267
|
+
filesystemIsolated: false,
|
|
268
|
+
},
|
|
269
|
+
},
|
|
270
|
+
});
|
|
271
|
+
let sessionId;
|
|
272
|
+
let executionError;
|
|
273
|
+
let stopError;
|
|
274
|
+
let runtimeEnvironment;
|
|
275
|
+
let bootstrapManifest;
|
|
276
|
+
const executedCells = [];
|
|
277
|
+
try {
|
|
278
|
+
sessionId = await this.kernelService.startKernel({
|
|
279
|
+
kernelName: frozen.kernelName,
|
|
280
|
+
cwd: path.dirname(request.notebook_path),
|
|
281
|
+
internalEnv: { PYTHONNOUSERSITE: '1' },
|
|
282
|
+
});
|
|
283
|
+
const baseProbe = await this.executeServerCode(sessionId, this.baseEnvironmentProbeSource());
|
|
284
|
+
const probed = this.parseBaseEnvironmentProbe(baseProbe.rawStdout);
|
|
285
|
+
this.compareBaseEnvironment(probed, request.environment_manifest);
|
|
286
|
+
await this.appendExecutionEvent(request.notebook_path, commonEvent, {
|
|
287
|
+
type: 'seal.environment.probed',
|
|
288
|
+
executionId: baseProbe.executionId,
|
|
289
|
+
payload: {
|
|
290
|
+
schemaVersion: 1,
|
|
291
|
+
sourceSha256: baseProbe.sourceSha256,
|
|
292
|
+
outputsSha256: baseProbe.outputsSha256,
|
|
293
|
+
executionCount: baseProbe.executionCount,
|
|
294
|
+
basePackagesSha256: probed.base_packages_sha256,
|
|
295
|
+
executableRealpath: probed.python.executable_realpath,
|
|
296
|
+
pythonNoUserSite: true,
|
|
297
|
+
status: 'completed',
|
|
298
|
+
},
|
|
299
|
+
});
|
|
300
|
+
const taskLocal = this.taskLocalDeclaration(request.environment_manifest, path.dirname(request.notebook_path));
|
|
301
|
+
let actualTaskLocal = null;
|
|
302
|
+
if (taskLocal) {
|
|
303
|
+
const bootstrap = await this.executeServerCode(sessionId, this.taskLocalBootstrapSource(taskLocal.packageDirectory, taskLocal.canonicalPackageDirectory));
|
|
304
|
+
const packages = this.parseTaskLocalProbe(bootstrap.rawStdout);
|
|
305
|
+
const packagesSha256 = (0, canonical_json_1.sha256Canonical)(packages);
|
|
306
|
+
this.compareTaskLocalEnvironment(packages, packagesSha256, taskLocal.declaration);
|
|
307
|
+
actualTaskLocal = {
|
|
308
|
+
package_directory: taskLocal.packageDirectory,
|
|
309
|
+
packages,
|
|
310
|
+
packages_sha256: packagesSha256,
|
|
311
|
+
};
|
|
312
|
+
bootstrapManifest = {
|
|
313
|
+
package_directory: taskLocal.packageDirectory,
|
|
314
|
+
source_sha256: bootstrap.sourceSha256,
|
|
315
|
+
outputs_sha256: bootstrap.outputsSha256,
|
|
316
|
+
execution_id: bootstrap.executionId,
|
|
317
|
+
execution_count: bootstrap.executionCount,
|
|
318
|
+
status: 'completed',
|
|
319
|
+
};
|
|
320
|
+
await this.appendExecutionEvent(request.notebook_path, commonEvent, {
|
|
321
|
+
type: 'seal.bootstrap.completed',
|
|
322
|
+
executionId: bootstrap.executionId,
|
|
323
|
+
payload: {
|
|
324
|
+
schemaVersion: 1,
|
|
325
|
+
packageDirectory: taskLocal.packageDirectory,
|
|
326
|
+
packagesSha256,
|
|
327
|
+
sourceSha256: bootstrap.sourceSha256,
|
|
328
|
+
outputsSha256: bootstrap.outputsSha256,
|
|
329
|
+
executionCount: bootstrap.executionCount,
|
|
330
|
+
status: 'completed',
|
|
331
|
+
},
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
runtimeEnvironment = { ...probed, task_local: actualTaskLocal };
|
|
335
|
+
for (const cell of frozen.codeCells) {
|
|
336
|
+
const execution = await this.executeNotebookCell(sessionId, cell);
|
|
337
|
+
const manifest = {
|
|
338
|
+
cell_id: cell.id,
|
|
339
|
+
source_sha256: execution.sourceSha256,
|
|
340
|
+
exploratory_outputs_sha256: cell.exploratoryOutputsSha256,
|
|
341
|
+
outputs_sha256: execution.outputsSha256,
|
|
342
|
+
replay_comparison_outputs_sha256: execution.comparisonOutputsSha256,
|
|
343
|
+
outputs_match_exploratory: cell.exploratoryOutputsSha256 === execution.comparisonOutputsSha256,
|
|
344
|
+
exploratory_execution_count: cell.exploratoryExecutionCount,
|
|
345
|
+
execution_id: execution.executionId,
|
|
346
|
+
execution_count: execution.executionCount,
|
|
347
|
+
status: 'completed',
|
|
348
|
+
};
|
|
349
|
+
executedCells.push({ index: cell.index, manifest, outputs: execution.outputs });
|
|
350
|
+
await this.appendExecutionEvent(request.notebook_path, commonEvent, {
|
|
351
|
+
type: 'seal.cell.completed',
|
|
352
|
+
executionId: execution.executionId,
|
|
353
|
+
payload: {
|
|
354
|
+
schemaVersion: 1,
|
|
355
|
+
sealId: job.sealId,
|
|
356
|
+
cellId: cell.id,
|
|
357
|
+
sourceSha256: execution.sourceSha256,
|
|
358
|
+
exploratoryOutputsSha256: cell.exploratoryOutputsSha256,
|
|
359
|
+
outputsSha256: execution.outputsSha256,
|
|
360
|
+
replayComparisonOutputsSha256: execution.comparisonOutputsSha256,
|
|
361
|
+
outputsMatchExploratory: cell.exploratoryOutputsSha256 === execution.comparisonOutputsSha256,
|
|
362
|
+
exploratoryExecutionCount: cell.exploratoryExecutionCount,
|
|
363
|
+
executionCount: execution.executionCount,
|
|
364
|
+
status: 'completed',
|
|
365
|
+
},
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
catch (error) {
|
|
370
|
+
executionError = error;
|
|
371
|
+
}
|
|
372
|
+
finally {
|
|
373
|
+
if (sessionId !== undefined) {
|
|
374
|
+
try {
|
|
375
|
+
const stopped = await this.kernelService.stopKernel(sessionId);
|
|
376
|
+
if (!stopped)
|
|
377
|
+
stopError = new Error('Fresh replay kernel could not be stopped');
|
|
378
|
+
}
|
|
379
|
+
catch (error) {
|
|
380
|
+
stopError = error;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
if (executionError)
|
|
385
|
+
throw executionError;
|
|
386
|
+
if (stopError)
|
|
387
|
+
throw stopError;
|
|
388
|
+
if (!runtimeEnvironment)
|
|
389
|
+
throw new Error('Fresh kernel produced no environment attestation');
|
|
390
|
+
const sealedNotebook = this.buildSealedNotebook(frozen.notebook, executedCells);
|
|
391
|
+
const sealedBytes = Buffer.from(`${(0, canonical_json_1.canonicalJson)(sealedNotebook)}\n`, 'utf8');
|
|
392
|
+
const sealedNotebookPath = path.join(job.sealDirectory, 'notebook.ipynb');
|
|
393
|
+
await this.writeImmutable(sealedNotebookPath, sealedBytes);
|
|
394
|
+
const notebookSha256 = (0, canonical_json_1.sha256Hex)(sealedBytes);
|
|
395
|
+
const artifacts = await this.sealArtifacts(request, job.sealDirectory, sealedNotebookPath, notebookSha256, sealedBytes.length);
|
|
396
|
+
const runtimeEnvironmentSha256 = (0, canonical_json_1.sha256Canonical)(runtimeEnvironment);
|
|
397
|
+
await this.provenanceStore.putBlob(request.notebook_path, (0, canonical_json_1.hashCanonicalJson)(runtimeEnvironment));
|
|
398
|
+
const completion = await this.provenanceStore.append(request.notebook_path, {
|
|
399
|
+
...commonEvent,
|
|
400
|
+
type: 'seal.completed',
|
|
401
|
+
payload: {
|
|
402
|
+
schemaVersion: 1,
|
|
403
|
+
hashProfile: canonical_json_1.CANONICAL_HASH_PROFILE,
|
|
404
|
+
sealId: job.sealId,
|
|
405
|
+
sourceNotebook: {
|
|
406
|
+
path: request.notebook_path,
|
|
407
|
+
sha256: job.sourceNotebookSha256,
|
|
408
|
+
},
|
|
409
|
+
notebook: { path: sealedNotebookPath, sha256: notebookSha256 },
|
|
410
|
+
inputManifestSha256,
|
|
411
|
+
environmentManifestSha256,
|
|
412
|
+
runtimeEnvironmentSha256,
|
|
413
|
+
cells: executedCells.map(({ manifest }) => ({
|
|
414
|
+
cellId: manifest.cell_id,
|
|
415
|
+
sourceSha256: manifest.source_sha256,
|
|
416
|
+
exploratoryOutputsSha256: manifest.exploratory_outputs_sha256,
|
|
417
|
+
outputsSha256: manifest.outputs_sha256,
|
|
418
|
+
replayComparisonOutputsSha256: manifest.replay_comparison_outputs_sha256,
|
|
419
|
+
outputsMatchExploratory: manifest.outputs_match_exploratory,
|
|
420
|
+
exploratoryExecutionCount: manifest.exploratory_execution_count,
|
|
421
|
+
executionId: manifest.execution_id,
|
|
422
|
+
executionCount: manifest.execution_count,
|
|
423
|
+
status: manifest.status,
|
|
424
|
+
})),
|
|
425
|
+
artifacts: artifacts.map((artifact) => ({
|
|
426
|
+
sourcePath: artifact.source_path,
|
|
427
|
+
path: artifact.path,
|
|
428
|
+
sha256: artifact.sha256,
|
|
429
|
+
sizeBytes: artifact.size_bytes,
|
|
430
|
+
})),
|
|
431
|
+
isolation: {
|
|
432
|
+
freshKernel: true,
|
|
433
|
+
networkIsolated: false,
|
|
434
|
+
filesystemIsolated: false,
|
|
435
|
+
},
|
|
436
|
+
},
|
|
437
|
+
});
|
|
438
|
+
const eventLedger = await this.captureLedgerPrefix(request.notebook_path, completion.event.eventHash, completion.event.seq);
|
|
439
|
+
const unsignedManifest = {
|
|
440
|
+
schema_version: 1,
|
|
441
|
+
hash_profile: canonical_json_1.CANONICAL_HASH_PROFILE,
|
|
442
|
+
seal_id: job.sealId,
|
|
443
|
+
source_notebook: {
|
|
444
|
+
path: request.notebook_path,
|
|
445
|
+
sha256: job.sourceNotebookSha256,
|
|
446
|
+
},
|
|
447
|
+
notebook: { path: sealedNotebookPath, sha256: notebookSha256 },
|
|
448
|
+
input_manifest_sha256: inputManifestSha256,
|
|
449
|
+
environment_manifest_sha256: environmentManifestSha256,
|
|
450
|
+
runtime_environment: runtimeEnvironment,
|
|
451
|
+
runtime_environment_sha256: runtimeEnvironmentSha256,
|
|
452
|
+
event_chain_head_sha256: completion.event.eventHash,
|
|
453
|
+
event_ledger: eventLedger,
|
|
454
|
+
cells: executedCells.map(({ manifest }) => manifest),
|
|
455
|
+
artifacts,
|
|
456
|
+
isolation: {
|
|
457
|
+
fresh_kernel: true,
|
|
458
|
+
network_isolated: false,
|
|
459
|
+
filesystem_isolated: false,
|
|
460
|
+
},
|
|
461
|
+
...(bootstrapManifest ? { bootstrap: bootstrapManifest } : {}),
|
|
462
|
+
};
|
|
463
|
+
const manifest = {
|
|
464
|
+
...unsignedManifest,
|
|
465
|
+
manifest_sha256: (0, canonical_json_1.sha256Canonical)(unsignedManifest),
|
|
466
|
+
};
|
|
467
|
+
await this.writeImmutable(path.join(job.sealDirectory, 'manifest.json'), Buffer.from(`${(0, canonical_json_1.canonicalJson)(manifest)}\n`, 'utf8'));
|
|
468
|
+
job = await this.updateJob(job, { status: 'completed', manifest, error: undefined });
|
|
469
|
+
await this.hardenSealDirectory(job.sealDirectory);
|
|
470
|
+
}
|
|
471
|
+
async failJob(job, error) {
|
|
472
|
+
const message = this.errorMessage(error);
|
|
473
|
+
const request = job.request;
|
|
474
|
+
const common = {
|
|
475
|
+
actor: { kind: 'agent', id: request.source_id, name: 'kosmos' },
|
|
476
|
+
runId: request.run_id,
|
|
477
|
+
taskId: request.task_id,
|
|
478
|
+
sourceId: request.source_id,
|
|
479
|
+
inputManifestSha256: (0, canonical_json_1.sha256Canonical)(request.input_manifest),
|
|
480
|
+
environmentManifestSha256: (0, canonical_json_1.sha256Canonical)(request.environment_manifest),
|
|
481
|
+
};
|
|
482
|
+
await this.provenanceStore.append(request.notebook_path, {
|
|
483
|
+
...common,
|
|
484
|
+
type: 'seal.failed',
|
|
485
|
+
payload: {
|
|
486
|
+
schemaVersion: 1,
|
|
487
|
+
hashProfile: canonical_json_1.CANONICAL_HASH_PROFILE,
|
|
488
|
+
sealId: job.sealId,
|
|
489
|
+
error: message,
|
|
490
|
+
},
|
|
491
|
+
}).catch(() => undefined);
|
|
492
|
+
await this.updateJob(job, { status: 'failed', error: message, manifest: undefined });
|
|
493
|
+
}
|
|
494
|
+
async executeNotebookCell(sessionId, cell) {
|
|
495
|
+
return this.executeAndCapture(sessionId, cell.source, cell.id);
|
|
496
|
+
}
|
|
497
|
+
async executeServerCode(sessionId, source) {
|
|
498
|
+
return this.executeAndCapture(sessionId, source, null, { storeHistory: false });
|
|
499
|
+
}
|
|
500
|
+
async executeAndCapture(sessionId, source, cellId, internalOptions) {
|
|
501
|
+
const executionId = this.idFactory();
|
|
502
|
+
const kernelOutputs = [];
|
|
503
|
+
const result = await this.kernelService.executeCode(sessionId, source, async (output) => { kernelOutputs.push(this.cloneJson(output)); }, undefined, cellId, internalOptions);
|
|
504
|
+
const errorOutput = kernelOutputs.find((output) => output.type === 'error');
|
|
505
|
+
if (result.status !== 'ok' || errorOutput) {
|
|
506
|
+
throw new Error(result.error || errorOutput?.content || `Execution failed for ${cellId ?? 'server bootstrap'}`);
|
|
507
|
+
}
|
|
508
|
+
if (!Number.isSafeInteger(result.executionCount) || (result.executionCount ?? -1) < 0) {
|
|
509
|
+
throw new Error(`Kernel returned no valid execution count for ${cellId ?? 'server bootstrap'}`);
|
|
510
|
+
}
|
|
511
|
+
if (kernelOutputs.some((output) => output.content.includes(OUTPUT_LIMIT_MARKER))) {
|
|
512
|
+
throw new Error(`Kernel output was truncated for ${cellId ?? 'server bootstrap'}`);
|
|
513
|
+
}
|
|
514
|
+
const outputs = kernelOutputs.map((output) => this.toJupyterOutput(output));
|
|
515
|
+
return {
|
|
516
|
+
executionId,
|
|
517
|
+
executionCount: result.executionCount,
|
|
518
|
+
sourceSha256: (0, canonical_json_1.sha256Hex)(source),
|
|
519
|
+
outputsSha256: (0, canonical_json_1.sha256Canonical)(outputs),
|
|
520
|
+
comparisonOutputsSha256: (0, canonical_json_1.sha256Canonical)(normalizeNotebookOutputsForComparison(outputs)),
|
|
521
|
+
outputs,
|
|
522
|
+
rawStdout: kernelOutputs
|
|
523
|
+
.filter((output) => output.type === 'stdout')
|
|
524
|
+
.map((output) => output.content)
|
|
525
|
+
.join(''),
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
async appendExecutionEvent(notebookPath, common, details) {
|
|
529
|
+
await this.provenanceStore.append(notebookPath, { ...common, ...details });
|
|
530
|
+
}
|
|
531
|
+
buildSealedNotebook(notebook, executed) {
|
|
532
|
+
const sealed = this.cloneJson(notebook);
|
|
533
|
+
for (const item of executed) {
|
|
534
|
+
const cell = sealed.cells[item.index];
|
|
535
|
+
cell.outputs = item.outputs;
|
|
536
|
+
cell.execution_count = item.manifest.execution_count;
|
|
537
|
+
}
|
|
538
|
+
return sealed;
|
|
539
|
+
}
|
|
540
|
+
async preflightArtifacts(request) {
|
|
541
|
+
const workspace = path.dirname(request.notebook_path);
|
|
542
|
+
for (const descriptor of request.artifacts) {
|
|
543
|
+
try {
|
|
544
|
+
await this.assertNoSymlinkComponents(workspace, descriptor.path);
|
|
545
|
+
const actual = await this.hashRegularFile(descriptor.path);
|
|
546
|
+
if (actual.size_bytes !== descriptor.size_bytes) {
|
|
547
|
+
throw new ReplaySealValidationError(`Requested artifact size mismatch: ${descriptor.path}; expected ${descriptor.size_bytes}, got ${actual.size_bytes}`);
|
|
548
|
+
}
|
|
549
|
+
if (actual.sha256 !== descriptor.sha256) {
|
|
550
|
+
throw new ReplaySealValidationError(`Requested artifact sha256 mismatch: ${descriptor.path}; expected ${descriptor.sha256}, got ${actual.sha256}`);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
catch (error) {
|
|
554
|
+
if (error instanceof ReplaySealValidationError)
|
|
555
|
+
throw error;
|
|
556
|
+
throw new ReplaySealValidationError(this.errorMessage(error));
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
async sealArtifacts(request, sealDirectory, sealedNotebookPath, notebookSha256, notebookSize) {
|
|
561
|
+
const artifacts = [{
|
|
562
|
+
source_path: request.notebook_path,
|
|
563
|
+
path: sealedNotebookPath,
|
|
564
|
+
sha256: notebookSha256,
|
|
565
|
+
size_bytes: notebookSize,
|
|
566
|
+
}];
|
|
567
|
+
const workspace = path.dirname(request.notebook_path);
|
|
568
|
+
for (const descriptor of request.artifacts) {
|
|
569
|
+
const sourcePath = descriptor.path;
|
|
570
|
+
const relative = this.relativeInside(workspace, sourcePath, 'artifact path');
|
|
571
|
+
if (relative.split(path.sep)[0] === '.nebula') {
|
|
572
|
+
throw new Error(`Requested artifact must not be inside .nebula: ${sourcePath}`);
|
|
573
|
+
}
|
|
574
|
+
await this.assertNoSymlinkComponents(workspace, sourcePath);
|
|
575
|
+
const components = relative.split(path.sep);
|
|
576
|
+
const filename = components.pop();
|
|
577
|
+
if (!filename)
|
|
578
|
+
throw new Error(`Requested artifact has no filename: ${sourcePath}`);
|
|
579
|
+
const destinationDirectory = await this.ensureSafeDirectoryChain(sealDirectory, ['artifacts', ...components]);
|
|
580
|
+
const destination = path.join(destinationDirectory, filename);
|
|
581
|
+
const copied = await this.copyRegularFileImmutable(sourcePath, destination, descriptor);
|
|
582
|
+
artifacts.push({ source_path: sourcePath, path: destination, ...copied });
|
|
583
|
+
}
|
|
584
|
+
return artifacts;
|
|
585
|
+
}
|
|
586
|
+
async copyRegularFileImmutable(sourcePath, destination, expected) {
|
|
587
|
+
let source;
|
|
588
|
+
try {
|
|
589
|
+
source = await fsp.open(sourcePath, fs_1.constants.O_RDONLY | fs_1.constants.O_NOFOLLOW);
|
|
590
|
+
}
|
|
591
|
+
catch (error) {
|
|
592
|
+
if (this.isNodeError(error, 'ENOENT'))
|
|
593
|
+
throw new Error(`Requested artifact is missing: ${sourcePath}`);
|
|
594
|
+
if (this.isNodeError(error, 'ELOOP'))
|
|
595
|
+
throw new Error(`Requested artifact is a symbolic link: ${sourcePath}`);
|
|
596
|
+
throw error;
|
|
597
|
+
}
|
|
598
|
+
const directory = path.dirname(destination);
|
|
599
|
+
const temporary = `${destination}.${process.pid}.${(0, crypto_1.randomUUID)()}.tmp`;
|
|
600
|
+
let destinationHandle;
|
|
601
|
+
try {
|
|
602
|
+
const before = await source.stat();
|
|
603
|
+
if (!before.isFile())
|
|
604
|
+
throw new Error(`Requested artifact is not a regular file: ${sourcePath}`);
|
|
605
|
+
destinationHandle = await fsp.open(temporary, 'wx', 0o400);
|
|
606
|
+
const hash = (0, crypto_1.createHash)('sha256');
|
|
607
|
+
const buffer = Buffer.allocUnsafe(1024 * 1024);
|
|
608
|
+
let sizeBytes = 0;
|
|
609
|
+
for (;;) {
|
|
610
|
+
const { bytesRead } = await source.read(buffer, 0, buffer.length, null);
|
|
611
|
+
if (bytesRead === 0)
|
|
612
|
+
break;
|
|
613
|
+
const chunk = buffer.subarray(0, bytesRead);
|
|
614
|
+
hash.update(chunk);
|
|
615
|
+
await destinationHandle.write(chunk);
|
|
616
|
+
sizeBytes += bytesRead;
|
|
617
|
+
}
|
|
618
|
+
if (this.durable)
|
|
619
|
+
await destinationHandle.sync();
|
|
620
|
+
const after = await source.stat();
|
|
621
|
+
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size
|
|
622
|
+
|| before.mtimeMs !== after.mtimeMs || after.size !== sizeBytes) {
|
|
623
|
+
throw new Error(`Requested artifact changed while it was sealed: ${sourcePath}`);
|
|
624
|
+
}
|
|
625
|
+
const sha256 = hash.digest('hex');
|
|
626
|
+
if (sizeBytes !== expected.size_bytes) {
|
|
627
|
+
throw new Error(`Requested artifact size mismatch after replay: ${sourcePath}; expected ${expected.size_bytes}, got ${sizeBytes}`);
|
|
628
|
+
}
|
|
629
|
+
if (sha256 !== expected.sha256) {
|
|
630
|
+
throw new Error(`Requested artifact sha256 mismatch after replay: ${sourcePath}; expected ${expected.sha256}, got ${sha256}`);
|
|
631
|
+
}
|
|
632
|
+
await destinationHandle.close();
|
|
633
|
+
destinationHandle = undefined;
|
|
634
|
+
try {
|
|
635
|
+
await fsp.link(temporary, destination);
|
|
636
|
+
}
|
|
637
|
+
catch (error) {
|
|
638
|
+
if (this.isNodeError(error, 'EEXIST')) {
|
|
639
|
+
throw new Error(`Immutable artifact destination already exists: ${destination}`);
|
|
640
|
+
}
|
|
641
|
+
throw error;
|
|
642
|
+
}
|
|
643
|
+
await fsp.chmod(destination, 0o400);
|
|
644
|
+
if (this.durable)
|
|
645
|
+
await this.syncDirectory(directory);
|
|
646
|
+
return { sha256, size_bytes: sizeBytes };
|
|
647
|
+
}
|
|
648
|
+
finally {
|
|
649
|
+
await source.close().catch(() => undefined);
|
|
650
|
+
await destinationHandle?.close().catch(() => undefined);
|
|
651
|
+
await fsp.unlink(temporary).catch(() => undefined);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
baseEnvironmentProbeSource() {
|
|
655
|
+
return [
|
|
656
|
+
'def __nebula_server_environment_probe__():',
|
|
657
|
+
' import importlib.metadata as metadata',
|
|
658
|
+
' import json',
|
|
659
|
+
' import os',
|
|
660
|
+
' import platform',
|
|
661
|
+
' import sys',
|
|
662
|
+
' packages = {}',
|
|
663
|
+
' for distribution in metadata.distributions():',
|
|
664
|
+
' name = distribution.metadata.get("Name")',
|
|
665
|
+
' if name:',
|
|
666
|
+
' packages[str(name).lower()] = str(distribution.version)',
|
|
667
|
+
' value = {',
|
|
668
|
+
' "executable": sys.executable,',
|
|
669
|
+
' "executable_realpath": os.path.realpath(sys.executable),',
|
|
670
|
+
' "implementation": platform.python_implementation(),',
|
|
671
|
+
' "version": platform.python_version(),',
|
|
672
|
+
' "platform": platform.platform(),',
|
|
673
|
+
' "base_packages": dict(sorted(packages.items())),',
|
|
674
|
+
' }',
|
|
675
|
+
` print(${JSON.stringify(PROBE_BEGIN)} + json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + ${JSON.stringify(PROBE_END)})`,
|
|
676
|
+
'__nebula_server_environment_probe__()',
|
|
677
|
+
'del __nebula_server_environment_probe__',
|
|
678
|
+
].join('\n');
|
|
679
|
+
}
|
|
680
|
+
taskLocalBootstrapSource(packageDirectory, canonicalPackageDirectory) {
|
|
681
|
+
return [
|
|
682
|
+
'def __nebula_server_task_local_bootstrap__():',
|
|
683
|
+
' import importlib.metadata as metadata',
|
|
684
|
+
' import json',
|
|
685
|
+
' import os',
|
|
686
|
+
' import sys',
|
|
687
|
+
` package_directory = ${JSON.stringify(packageDirectory)}`,
|
|
688
|
+
` expected_realpath = ${JSON.stringify(canonicalPackageDirectory)}`,
|
|
689
|
+
' if os.path.realpath(package_directory) != expected_realpath or not os.path.isdir(package_directory):',
|
|
690
|
+
' raise RuntimeError("task-local package directory changed after server validation")',
|
|
691
|
+
' packages = {}',
|
|
692
|
+
' for distribution in metadata.distributions(path=[package_directory]):',
|
|
693
|
+
' name = distribution.metadata.get("Name")',
|
|
694
|
+
' if name:',
|
|
695
|
+
' packages[str(name).lower()] = str(distribution.version)',
|
|
696
|
+
' if package_directory not in sys.path:',
|
|
697
|
+
' sys.path.insert(0, package_directory)',
|
|
698
|
+
` print(${JSON.stringify(PROBE_BEGIN)} + json.dumps(dict(sorted(packages.items())), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + ${JSON.stringify(PROBE_END)})`,
|
|
699
|
+
'__nebula_server_task_local_bootstrap__()',
|
|
700
|
+
'del __nebula_server_task_local_bootstrap__',
|
|
701
|
+
].join('\n');
|
|
702
|
+
}
|
|
703
|
+
parseBaseEnvironmentProbe(stdout) {
|
|
704
|
+
const raw = this.markerPayload(stdout, 'environment probe');
|
|
705
|
+
const value = this.parseObject(raw, 'environment probe');
|
|
706
|
+
const packages = this.stringMap(value.base_packages, 'environment probe base_packages');
|
|
707
|
+
const python = {
|
|
708
|
+
executable: this.nonempty(value.executable, 'environment probe executable'),
|
|
709
|
+
executable_realpath: this.nonempty(value.executable_realpath, 'environment probe executable_realpath'),
|
|
710
|
+
implementation: this.nonempty(value.implementation, 'environment probe implementation'),
|
|
711
|
+
version: this.nonempty(value.version, 'environment probe version'),
|
|
712
|
+
platform: this.nonempty(value.platform, 'environment probe platform'),
|
|
713
|
+
};
|
|
714
|
+
if (!path.isAbsolute(python.executable) || !path.isAbsolute(python.executable_realpath)) {
|
|
715
|
+
throw new Error('Fresh kernel environment probe returned a non-absolute executable');
|
|
716
|
+
}
|
|
717
|
+
return {
|
|
718
|
+
schema_version: 1,
|
|
719
|
+
python,
|
|
720
|
+
python_no_user_site: true,
|
|
721
|
+
base_packages: packages,
|
|
722
|
+
base_packages_sha256: (0, canonical_json_1.sha256Canonical)(packages),
|
|
723
|
+
};
|
|
724
|
+
}
|
|
725
|
+
parseTaskLocalProbe(stdout) {
|
|
726
|
+
return this.stringMap(this.parseObject(this.markerPayload(stdout, 'task-local bootstrap'), 'task-local bootstrap'), 'task-local packages');
|
|
727
|
+
}
|
|
728
|
+
compareBaseEnvironment(actual, declaration) {
|
|
729
|
+
const base = this.optionalRecord(declaration.base, 'environment_manifest.base');
|
|
730
|
+
if (!base || base.inspection === 'not_configured')
|
|
731
|
+
return;
|
|
732
|
+
const comparisons = [
|
|
733
|
+
['python_binary_target', base.python_binary_target, actual.python.executable_realpath],
|
|
734
|
+
['implementation', base.implementation, actual.python.implementation],
|
|
735
|
+
['python_version', base.python_version, actual.python.version],
|
|
736
|
+
['platform', base.platform, actual.python.platform],
|
|
737
|
+
['base_packages_sha256', base.base_packages_sha256, actual.base_packages_sha256],
|
|
738
|
+
];
|
|
739
|
+
for (const [name, expected, observed] of comparisons) {
|
|
740
|
+
if (expected !== undefined && expected !== null && expected !== observed) {
|
|
741
|
+
throw new Error(`Fresh kernel environment mismatch for ${name}: expected ${String(expected)}, got ${String(observed)}`);
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
if (base.base_packages !== undefined) {
|
|
745
|
+
const expectedPackages = this.stringMap(base.base_packages, 'environment_manifest.base.base_packages');
|
|
746
|
+
if ((0, canonical_json_1.sha256Canonical)(expectedPackages) !== actual.base_packages_sha256) {
|
|
747
|
+
throw new Error('Fresh kernel environment mismatch for base_packages');
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
compareTaskLocalEnvironment(packages, packagesSha256, declaration) {
|
|
752
|
+
if (declaration.packages_sha256 !== undefined && declaration.packages_sha256 !== packagesSha256) {
|
|
753
|
+
throw new Error(`Task-local package hash mismatch: expected ${String(declaration.packages_sha256)}, got ${packagesSha256}`);
|
|
754
|
+
}
|
|
755
|
+
if (declaration.packages !== undefined) {
|
|
756
|
+
const expected = this.stringMap(declaration.packages, 'environment_manifest.task_local.packages');
|
|
757
|
+
if ((0, canonical_json_1.sha256Canonical)(expected) !== (0, canonical_json_1.sha256Canonical)(packages)) {
|
|
758
|
+
throw new Error('Task-local package distribution map mismatch');
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
taskLocalDeclaration(environment, workspace) {
|
|
763
|
+
const taskLocal = this.optionalRecord(environment.task_local, 'environment_manifest.task_local');
|
|
764
|
+
if (!taskLocal || taskLocal.package_directory === null || taskLocal.package_directory === undefined)
|
|
765
|
+
return null;
|
|
766
|
+
const packageDirectory = this.nonempty(taskLocal.package_directory, 'environment_manifest.task_local.package_directory');
|
|
767
|
+
if (!path.isAbsolute(packageDirectory)) {
|
|
768
|
+
throw new ReplaySealValidationError('environment_manifest.task_local.package_directory must be absolute');
|
|
769
|
+
}
|
|
770
|
+
this.relativeInside(workspace, packageDirectory, 'task-local package_directory');
|
|
771
|
+
let stat;
|
|
772
|
+
try {
|
|
773
|
+
stat = (0, fs_1.lstatSync)(packageDirectory);
|
|
774
|
+
}
|
|
775
|
+
catch {
|
|
776
|
+
throw new ReplaySealValidationError('task-local package_directory must be an existing directory');
|
|
777
|
+
}
|
|
778
|
+
if (stat.isSymbolicLink() || !stat.isDirectory()) {
|
|
779
|
+
throw new ReplaySealValidationError('task-local package_directory must be a real directory, not a symbolic link');
|
|
780
|
+
}
|
|
781
|
+
const realWorkspace = (0, fs_1.realpathSync)(workspace);
|
|
782
|
+
const realPackageDirectory = (0, fs_1.realpathSync)(packageDirectory);
|
|
783
|
+
this.relativeInside(realWorkspace, realPackageDirectory, 'task-local package_directory');
|
|
784
|
+
this.assertNoSymlinkComponentsSync(path.resolve(workspace), path.resolve(packageDirectory));
|
|
785
|
+
return {
|
|
786
|
+
packageDirectory,
|
|
787
|
+
canonicalPackageDirectory: realPackageDirectory,
|
|
788
|
+
declaration: taskLocal,
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
validateEnvironmentDeclaration(environment, workspace) {
|
|
792
|
+
this.taskLocalDeclaration(environment, workspace);
|
|
793
|
+
}
|
|
794
|
+
parseFrozenNotebook(bytes, requestedKernelName) {
|
|
795
|
+
let parsed;
|
|
796
|
+
try {
|
|
797
|
+
parsed = JSON.parse(bytes.toString('utf8'));
|
|
798
|
+
}
|
|
799
|
+
catch (error) {
|
|
800
|
+
throw new ReplaySealValidationError(`Notebook is not valid JSON: ${this.errorMessage(error)}`);
|
|
801
|
+
}
|
|
802
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
803
|
+
throw new ReplaySealValidationError('Notebook must be a JSON object');
|
|
804
|
+
}
|
|
805
|
+
const notebook = parsed;
|
|
806
|
+
if (!Array.isArray(notebook.cells) || notebook.nbformat !== 4) {
|
|
807
|
+
throw new ReplaySealValidationError('Replay sealing requires a Jupyter nbformat 4 notebook');
|
|
808
|
+
}
|
|
809
|
+
if (!notebook.metadata || typeof notebook.metadata !== 'object' || Array.isArray(notebook.metadata)) {
|
|
810
|
+
throw new ReplaySealValidationError('Notebook metadata must be an object');
|
|
811
|
+
}
|
|
812
|
+
const seen = new Set();
|
|
813
|
+
const codeCells = [];
|
|
814
|
+
notebook.cells.forEach((cell, index) => {
|
|
815
|
+
if (!cell || typeof cell !== 'object' || Array.isArray(cell)) {
|
|
816
|
+
throw new ReplaySealValidationError(`Notebook cell ${index + 1} must be an object`);
|
|
817
|
+
}
|
|
818
|
+
if (typeof cell.id !== 'string' || !CELL_ID_RE.test(cell.id)) {
|
|
819
|
+
throw new ReplaySealValidationError(`Notebook cell ${index + 1} requires a canonical top-level Jupyter cell id`);
|
|
820
|
+
}
|
|
821
|
+
if (seen.has(cell.id)) {
|
|
822
|
+
throw new ReplaySealValidationError(`Notebook has duplicate cell id ${cell.id}`);
|
|
823
|
+
}
|
|
824
|
+
seen.add(cell.id);
|
|
825
|
+
if (cell.cell_type === 'code') {
|
|
826
|
+
const source = this.cellSource(cell.source, index);
|
|
827
|
+
const exploratoryOutputs = cell.outputs === undefined ? [] : cell.outputs;
|
|
828
|
+
if (!Array.isArray(exploratoryOutputs)) {
|
|
829
|
+
throw new ReplaySealValidationError(`Notebook code cell ${index + 1} has invalid outputs`);
|
|
830
|
+
}
|
|
831
|
+
(0, canonical_json_1.canonicalJson)(exploratoryOutputs);
|
|
832
|
+
const exploratoryExecutionCount = cell.execution_count === null
|
|
833
|
+
|| cell.execution_count === undefined
|
|
834
|
+
? null
|
|
835
|
+
: cell.execution_count;
|
|
836
|
+
if (exploratoryExecutionCount !== null
|
|
837
|
+
&& (!Number.isSafeInteger(exploratoryExecutionCount) || exploratoryExecutionCount < 0)) {
|
|
838
|
+
throw new ReplaySealValidationError(`Notebook code cell ${index + 1} has invalid execution_count`);
|
|
839
|
+
}
|
|
840
|
+
codeCells.push({
|
|
841
|
+
index,
|
|
842
|
+
id: cell.id,
|
|
843
|
+
source,
|
|
844
|
+
exploratoryOutputsSha256: (0, canonical_json_1.sha256Canonical)(normalizeNotebookOutputsForComparison(exploratoryOutputs)),
|
|
845
|
+
exploratoryExecutionCount: exploratoryExecutionCount,
|
|
846
|
+
});
|
|
847
|
+
}
|
|
848
|
+
});
|
|
849
|
+
if (codeCells.length === 0) {
|
|
850
|
+
throw new ReplaySealValidationError('Notebook must contain at least one code cell');
|
|
851
|
+
}
|
|
852
|
+
let kernelName = requestedKernelName;
|
|
853
|
+
if (!kernelName) {
|
|
854
|
+
const kernelspec = this.optionalRecord(notebook.metadata.kernelspec, 'notebook metadata.kernelspec');
|
|
855
|
+
kernelName = typeof kernelspec?.name === 'string' && kernelspec.name.trim()
|
|
856
|
+
? kernelspec.name
|
|
857
|
+
: 'python3';
|
|
858
|
+
}
|
|
859
|
+
return { notebook: this.cloneJson(notebook), codeCells, kernelName };
|
|
860
|
+
}
|
|
861
|
+
normalizeRequest(raw) {
|
|
862
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
|
863
|
+
throw new ReplaySealValidationError('Replay-seal request body must be an object');
|
|
864
|
+
}
|
|
865
|
+
const allowedKeys = new Set([
|
|
866
|
+
'notebook_path',
|
|
867
|
+
'run_id',
|
|
868
|
+
'task_id',
|
|
869
|
+
'source_id',
|
|
870
|
+
'source_notebook_sha256',
|
|
871
|
+
'artifacts',
|
|
872
|
+
'kernel_name',
|
|
873
|
+
'input_manifest',
|
|
874
|
+
'environment_manifest',
|
|
875
|
+
]);
|
|
876
|
+
const unknownKeys = Object.keys(raw).filter((key) => !allowedKeys.has(key));
|
|
877
|
+
if (unknownKeys.length > 0) {
|
|
878
|
+
throw new ReplaySealValidationError(`Replay-seal request has unsupported field: ${unknownKeys[0]}`);
|
|
879
|
+
}
|
|
880
|
+
const notebookPath = this.nonempty(raw.notebook_path, 'notebook_path');
|
|
881
|
+
if (!path.isAbsolute(notebookPath) || path.extname(notebookPath).toLowerCase() !== '.ipynb') {
|
|
882
|
+
throw new ReplaySealValidationError('notebook_path must be an absolute .ipynb path');
|
|
883
|
+
}
|
|
884
|
+
if (path.normalize(notebookPath) !== notebookPath) {
|
|
885
|
+
throw new ReplaySealValidationError('notebook_path must be normalized without dot segments');
|
|
886
|
+
}
|
|
887
|
+
const sourceSha256 = this.nonempty(raw.source_notebook_sha256, 'source_notebook_sha256');
|
|
888
|
+
if (!SHA256_RE.test(sourceSha256)) {
|
|
889
|
+
throw new ReplaySealValidationError('source_notebook_sha256 must be a lowercase SHA-256 digest');
|
|
890
|
+
}
|
|
891
|
+
const artifactDescriptors = raw.artifacts;
|
|
892
|
+
if (!Array.isArray(artifactDescriptors)) {
|
|
893
|
+
throw new ReplaySealValidationError('artifacts must be an array');
|
|
894
|
+
}
|
|
895
|
+
const workspace = path.dirname(notebookPath);
|
|
896
|
+
const normalizedArtifacts = [];
|
|
897
|
+
for (const [index, value] of artifactDescriptors.entries()) {
|
|
898
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
899
|
+
throw new ReplaySealValidationError(`artifacts item ${index + 1} must be an object`);
|
|
900
|
+
}
|
|
901
|
+
const descriptor = value;
|
|
902
|
+
const descriptorKeys = Object.keys(descriptor);
|
|
903
|
+
const unknownDescriptorKey = descriptorKeys.find((key) => key !== 'path' && key !== 'sha256' && key !== 'size_bytes');
|
|
904
|
+
if (unknownDescriptorKey) {
|
|
905
|
+
throw new ReplaySealValidationError(`artifacts item ${index + 1} has unsupported field: ${unknownDescriptorKey}`);
|
|
906
|
+
}
|
|
907
|
+
const artifactPath = this.nonempty(descriptor.path, `artifacts item ${index + 1} path`);
|
|
908
|
+
if (!path.isAbsolute(artifactPath)) {
|
|
909
|
+
throw new ReplaySealValidationError(`artifacts item ${index + 1} path must be absolute`);
|
|
910
|
+
}
|
|
911
|
+
if (path.normalize(artifactPath) !== artifactPath) {
|
|
912
|
+
throw new ReplaySealValidationError(`artifacts item ${index + 1} path must be normalized without dot segments`);
|
|
913
|
+
}
|
|
914
|
+
const relative = this.relativeInside(workspace, artifactPath, `artifacts item ${index + 1} path`);
|
|
915
|
+
if (relative.split(path.sep)[0] === '.nebula') {
|
|
916
|
+
throw new ReplaySealValidationError(`artifacts item ${index + 1} must not be inside .nebula`);
|
|
917
|
+
}
|
|
918
|
+
if (artifactPath === notebookPath) {
|
|
919
|
+
throw new ReplaySealValidationError('artifacts must not repeat notebook_path');
|
|
920
|
+
}
|
|
921
|
+
if (normalizedArtifacts.some((artifact) => artifact.path === artifactPath)) {
|
|
922
|
+
throw new ReplaySealValidationError(`artifacts repeats ${artifactPath}`);
|
|
923
|
+
}
|
|
924
|
+
const sha256 = this.nonempty(descriptor.sha256, `artifacts item ${index + 1} sha256`);
|
|
925
|
+
if (!SHA256_RE.test(sha256)) {
|
|
926
|
+
throw new ReplaySealValidationError(`artifacts item ${index + 1} sha256 must be a lowercase SHA-256 digest`);
|
|
927
|
+
}
|
|
928
|
+
const sizeBytes = descriptor.size_bytes;
|
|
929
|
+
if (!Number.isSafeInteger(sizeBytes) || sizeBytes < 0) {
|
|
930
|
+
throw new ReplaySealValidationError(`artifacts item ${index + 1} size_bytes must be a non-negative safe integer`);
|
|
931
|
+
}
|
|
932
|
+
normalizedArtifacts.push({
|
|
933
|
+
path: path.normalize(artifactPath),
|
|
934
|
+
sha256,
|
|
935
|
+
size_bytes: sizeBytes,
|
|
936
|
+
});
|
|
937
|
+
}
|
|
938
|
+
let inputManifest;
|
|
939
|
+
let environmentManifest;
|
|
940
|
+
try {
|
|
941
|
+
inputManifest = this.plainRecord(raw.input_manifest, 'input_manifest');
|
|
942
|
+
environmentManifest = this.plainRecord(raw.environment_manifest, 'environment_manifest');
|
|
943
|
+
}
|
|
944
|
+
catch (error) {
|
|
945
|
+
if (error instanceof ReplaySealValidationError)
|
|
946
|
+
throw error;
|
|
947
|
+
throw new ReplaySealValidationError(this.errorMessage(error));
|
|
948
|
+
}
|
|
949
|
+
let kernelName;
|
|
950
|
+
if (raw.kernel_name !== undefined) {
|
|
951
|
+
kernelName = this.nonempty(raw.kernel_name, 'kernel_name');
|
|
952
|
+
if (kernelName.startsWith('env:') && !path.isAbsolute(kernelName.slice(4))) {
|
|
953
|
+
throw new ReplaySealValidationError('env: kernel_name must contain an absolute Python interpreter path');
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
const request = {
|
|
957
|
+
notebook_path: path.normalize(notebookPath),
|
|
958
|
+
run_id: this.nonempty(raw.run_id, 'run_id'),
|
|
959
|
+
task_id: this.nonempty(raw.task_id, 'task_id'),
|
|
960
|
+
source_id: this.nonempty(raw.source_id, 'source_id'),
|
|
961
|
+
source_notebook_sha256: sourceSha256,
|
|
962
|
+
artifacts: normalizedArtifacts,
|
|
963
|
+
input_manifest: inputManifest,
|
|
964
|
+
environment_manifest: environmentManifest,
|
|
965
|
+
...(kernelName !== undefined ? { kernel_name: kernelName } : {}),
|
|
966
|
+
};
|
|
967
|
+
return this.cloneJson(request);
|
|
968
|
+
}
|
|
969
|
+
validateIdempotencyKey(value) {
|
|
970
|
+
if (typeof value !== 'string' || value.length === 0 || value.length > 512 || /[\u0000-\u001f\u007f]/.test(value)) {
|
|
971
|
+
throw new ReplaySealValidationError('Idempotency-Key must be 1-512 characters without control characters');
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
responseFor(job) {
|
|
975
|
+
if (job.status === 'completed' && job.manifest) {
|
|
976
|
+
return { seal_id: job.sealId, status: 'completed', manifest: job.manifest };
|
|
977
|
+
}
|
|
978
|
+
if (job.status === 'failed') {
|
|
979
|
+
return { seal_id: job.sealId, status: 'failed', error: job.error || 'Replay seal failed' };
|
|
980
|
+
}
|
|
981
|
+
if (job.status === 'completed') {
|
|
982
|
+
throw new Error(`Completed replay seal ${job.sealId} has no manifest`);
|
|
983
|
+
}
|
|
984
|
+
return { seal_id: job.sealId, status: job.status };
|
|
985
|
+
}
|
|
986
|
+
async verifiedResponseFor(job) {
|
|
987
|
+
if (job.status !== 'completed')
|
|
988
|
+
return this.responseFor(job);
|
|
989
|
+
try {
|
|
990
|
+
const manifest = await this.verifyCompletedSeal(job);
|
|
991
|
+
return { seal_id: job.sealId, status: 'completed', manifest };
|
|
992
|
+
}
|
|
993
|
+
catch (error) {
|
|
994
|
+
return {
|
|
995
|
+
seal_id: job.sealId,
|
|
996
|
+
status: 'failed',
|
|
997
|
+
error: `Seal integrity verification failed: ${this.errorMessage(error)}`,
|
|
998
|
+
};
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
async verifyCompletedSeal(job) {
|
|
1002
|
+
if (!job.manifest)
|
|
1003
|
+
throw new Error('completed job has no manifest');
|
|
1004
|
+
if (job.requestFingerprint !== (0, canonical_json_1.sha256Canonical)(job.request)) {
|
|
1005
|
+
throw new Error('persisted request fingerprint mismatch');
|
|
1006
|
+
}
|
|
1007
|
+
const sealDirectory = await fsp.realpath(job.sealDirectory);
|
|
1008
|
+
if (sealDirectory !== job.sealDirectory)
|
|
1009
|
+
throw new Error('seal directory path is not canonical');
|
|
1010
|
+
const expectedFrozenPath = path.join(sealDirectory, 'source.ipynb');
|
|
1011
|
+
if (job.frozenNotebookPath !== expectedFrozenPath)
|
|
1012
|
+
throw new Error('frozen notebook path mismatch');
|
|
1013
|
+
const manifestPath = path.join(sealDirectory, 'manifest.json');
|
|
1014
|
+
const manifestStat = await fsp.lstat(manifestPath);
|
|
1015
|
+
if (manifestStat.isSymbolicLink() || !manifestStat.isFile()) {
|
|
1016
|
+
throw new Error('manifest is not a regular immutable file');
|
|
1017
|
+
}
|
|
1018
|
+
const manifestBytes = await fsp.readFile(manifestPath);
|
|
1019
|
+
let parsed;
|
|
1020
|
+
try {
|
|
1021
|
+
parsed = JSON.parse(manifestBytes.toString('utf8'));
|
|
1022
|
+
}
|
|
1023
|
+
catch {
|
|
1024
|
+
throw new Error('manifest contains invalid JSON');
|
|
1025
|
+
}
|
|
1026
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
1027
|
+
throw new Error('manifest is not an object');
|
|
1028
|
+
}
|
|
1029
|
+
const manifest = parsed;
|
|
1030
|
+
if (manifestBytes.toString('utf8') !== `${(0, canonical_json_1.canonicalJson)(manifest)}\n`) {
|
|
1031
|
+
throw new Error('manifest is not canonical JSON');
|
|
1032
|
+
}
|
|
1033
|
+
const { manifest_sha256: claimedManifestSha256, ...unsignedManifest } = manifest;
|
|
1034
|
+
if (!SHA256_RE.test(claimedManifestSha256) || (0, canonical_json_1.sha256Canonical)(unsignedManifest) !== claimedManifestSha256) {
|
|
1035
|
+
throw new Error('manifest_sha256 mismatch');
|
|
1036
|
+
}
|
|
1037
|
+
if ((0, canonical_json_1.canonicalJson)(manifest) !== (0, canonical_json_1.canonicalJson)(job.manifest)) {
|
|
1038
|
+
throw new Error('job state and immutable manifest disagree');
|
|
1039
|
+
}
|
|
1040
|
+
if (manifest.schema_version !== 1
|
|
1041
|
+
|| manifest.hash_profile !== canonical_json_1.CANONICAL_HASH_PROFILE
|
|
1042
|
+
|| manifest.seal_id !== job.sealId) {
|
|
1043
|
+
throw new Error('manifest identity mismatch');
|
|
1044
|
+
}
|
|
1045
|
+
if (manifest.source_notebook.path !== job.request.notebook_path
|
|
1046
|
+
|| manifest.source_notebook.sha256 !== job.sourceNotebookSha256
|
|
1047
|
+
|| manifest.source_notebook.sha256 !== job.request.source_notebook_sha256) {
|
|
1048
|
+
throw new Error('source notebook attestation mismatch');
|
|
1049
|
+
}
|
|
1050
|
+
const frozen = await this.hashRegularFile(expectedFrozenPath);
|
|
1051
|
+
if (frozen.sha256 !== job.sourceNotebookSha256)
|
|
1052
|
+
throw new Error('frozen source notebook hash mismatch');
|
|
1053
|
+
const expectedNotebookPath = path.join(sealDirectory, 'notebook.ipynb');
|
|
1054
|
+
if (manifest.notebook.path !== expectedNotebookPath)
|
|
1055
|
+
throw new Error('sealed notebook path mismatch');
|
|
1056
|
+
const notebook = await this.hashRegularFile(expectedNotebookPath);
|
|
1057
|
+
if (notebook.sha256 !== manifest.notebook.sha256)
|
|
1058
|
+
throw new Error('sealed notebook hash mismatch');
|
|
1059
|
+
if (!Array.isArray(manifest.cells) || manifest.cells.length === 0) {
|
|
1060
|
+
throw new Error('manifest contains no executed cells');
|
|
1061
|
+
}
|
|
1062
|
+
await this.verifyManifestCells(expectedFrozenPath, expectedNotebookPath, manifest.cells);
|
|
1063
|
+
if ((0, canonical_json_1.sha256Canonical)(manifest.runtime_environment) !== manifest.runtime_environment_sha256) {
|
|
1064
|
+
throw new Error('runtime environment hash mismatch');
|
|
1065
|
+
}
|
|
1066
|
+
if ((0, canonical_json_1.sha256Canonical)(job.request.input_manifest) !== manifest.input_manifest_sha256) {
|
|
1067
|
+
throw new Error('input manifest hash mismatch');
|
|
1068
|
+
}
|
|
1069
|
+
if ((0, canonical_json_1.sha256Canonical)(job.request.environment_manifest) !== manifest.environment_manifest_sha256) {
|
|
1070
|
+
throw new Error('environment manifest hash mismatch');
|
|
1071
|
+
}
|
|
1072
|
+
if (!(await this.provenanceStore.verifyBlob(job.request.notebook_path, manifest.input_manifest_sha256)).valid) {
|
|
1073
|
+
throw new Error('input manifest blob verification failed');
|
|
1074
|
+
}
|
|
1075
|
+
if (!(await this.provenanceStore.verifyBlob(job.request.notebook_path, manifest.environment_manifest_sha256)).valid) {
|
|
1076
|
+
throw new Error('environment manifest blob verification failed');
|
|
1077
|
+
}
|
|
1078
|
+
if (!(await this.provenanceStore.verifyBlob(job.request.notebook_path, manifest.runtime_environment_sha256)).valid) {
|
|
1079
|
+
throw new Error('runtime environment blob verification failed');
|
|
1080
|
+
}
|
|
1081
|
+
const expectedArtifactSources = [
|
|
1082
|
+
job.request.notebook_path,
|
|
1083
|
+
...job.request.artifacts.map((artifact) => artifact.path),
|
|
1084
|
+
];
|
|
1085
|
+
if (!Array.isArray(manifest.artifacts) || manifest.artifacts.length !== expectedArtifactSources.length) {
|
|
1086
|
+
throw new Error('artifact attestation count mismatch');
|
|
1087
|
+
}
|
|
1088
|
+
for (let index = 0; index < manifest.artifacts.length; index += 1) {
|
|
1089
|
+
const artifact = manifest.artifacts[index];
|
|
1090
|
+
if (artifact.source_path !== expectedArtifactSources[index]) {
|
|
1091
|
+
throw new Error(`artifact ${index + 1} source path mismatch`);
|
|
1092
|
+
}
|
|
1093
|
+
const artifactRealPath = await fsp.realpath(artifact.path);
|
|
1094
|
+
this.relativeInside(sealDirectory, artifactRealPath, `artifact ${index + 1} sealed path`);
|
|
1095
|
+
const actual = await this.hashRegularFile(artifact.path);
|
|
1096
|
+
if (actual.sha256 !== artifact.sha256 || actual.size_bytes !== artifact.size_bytes) {
|
|
1097
|
+
throw new Error(`artifact ${index + 1} hash or size mismatch`);
|
|
1098
|
+
}
|
|
1099
|
+
const requested = job.request.artifacts[index - 1];
|
|
1100
|
+
if (requested && (artifact.sha256 !== requested.sha256
|
|
1101
|
+
|| artifact.size_bytes !== requested.size_bytes)) {
|
|
1102
|
+
throw new Error(`artifact ${index + 1} does not match its predeclared descriptor`);
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
if (manifest.artifacts[0].path !== manifest.notebook.path
|
|
1106
|
+
|| manifest.artifacts[0].sha256 !== manifest.notebook.sha256) {
|
|
1107
|
+
throw new Error('notebook artifact does not match the sealed notebook');
|
|
1108
|
+
}
|
|
1109
|
+
const ledger = manifest.event_ledger;
|
|
1110
|
+
if (ledger.path !== this.provenanceStore.getLedgerPath(job.request.notebook_path)
|
|
1111
|
+
|| ledger.head_sha256 !== manifest.event_chain_head_sha256
|
|
1112
|
+
|| ledger.prefix_size_bytes < 1
|
|
1113
|
+
|| !Number.isSafeInteger(ledger.head_seq)
|
|
1114
|
+
|| ledger.head_seq < 1) {
|
|
1115
|
+
throw new Error('event ledger metadata mismatch');
|
|
1116
|
+
}
|
|
1117
|
+
const ledgerBytes = await fsp.readFile(ledger.path);
|
|
1118
|
+
if (ledger.prefix_size_bytes > ledgerBytes.length)
|
|
1119
|
+
throw new Error('event ledger prefix exceeds file');
|
|
1120
|
+
const prefix = ledgerBytes.subarray(0, ledger.prefix_size_bytes);
|
|
1121
|
+
if (prefix.at(-1) !== 0x0a || (0, canonical_json_1.sha256Hex)(prefix) !== ledger.prefix_sha256) {
|
|
1122
|
+
throw new Error('event ledger prefix hash or boundary mismatch');
|
|
1123
|
+
}
|
|
1124
|
+
const head = this.verifyLedgerPrefix(prefix, job.request.notebook_path);
|
|
1125
|
+
if (head.eventHash !== ledger.head_sha256
|
|
1126
|
+
|| head.seq !== ledger.head_seq
|
|
1127
|
+
|| head.type !== 'seal.completed') {
|
|
1128
|
+
throw new Error('event ledger head mismatch');
|
|
1129
|
+
}
|
|
1130
|
+
const headPayload = this.plainRecord(head.payload, 'completed event payload');
|
|
1131
|
+
if ((0, canonical_json_1.canonicalJson)(head.record.actor) !== (0, canonical_json_1.canonicalJson)({
|
|
1132
|
+
kind: 'agent',
|
|
1133
|
+
id: job.request.source_id,
|
|
1134
|
+
name: 'kosmos',
|
|
1135
|
+
})
|
|
1136
|
+
|| head.record.runId !== job.request.run_id
|
|
1137
|
+
|| head.record.taskId !== job.request.task_id
|
|
1138
|
+
|| head.record.sourceId !== job.request.source_id
|
|
1139
|
+
|| head.record.inputManifestSha256 !== manifest.input_manifest_sha256
|
|
1140
|
+
|| head.record.environmentManifestSha256 !== manifest.environment_manifest_sha256) {
|
|
1141
|
+
throw new Error('completed event actor or request identity mismatch');
|
|
1142
|
+
}
|
|
1143
|
+
const expectedEventCells = manifest.cells.map((item) => ({
|
|
1144
|
+
cellId: item.cell_id,
|
|
1145
|
+
sourceSha256: item.source_sha256,
|
|
1146
|
+
exploratoryOutputsSha256: item.exploratory_outputs_sha256,
|
|
1147
|
+
outputsSha256: item.outputs_sha256,
|
|
1148
|
+
replayComparisonOutputsSha256: item.replay_comparison_outputs_sha256,
|
|
1149
|
+
outputsMatchExploratory: item.outputs_match_exploratory,
|
|
1150
|
+
exploratoryExecutionCount: item.exploratory_execution_count,
|
|
1151
|
+
executionId: item.execution_id,
|
|
1152
|
+
executionCount: item.execution_count,
|
|
1153
|
+
status: item.status,
|
|
1154
|
+
}));
|
|
1155
|
+
const expectedEventArtifacts = manifest.artifacts.map((item) => ({
|
|
1156
|
+
sourcePath: item.source_path,
|
|
1157
|
+
path: item.path,
|
|
1158
|
+
sha256: item.sha256,
|
|
1159
|
+
sizeBytes: item.size_bytes,
|
|
1160
|
+
}));
|
|
1161
|
+
if (headPayload.sealId !== job.sealId
|
|
1162
|
+
|| headPayload.hashProfile !== canonical_json_1.CANONICAL_HASH_PROFILE
|
|
1163
|
+
|| (0, canonical_json_1.canonicalJson)(headPayload.sourceNotebook) !== (0, canonical_json_1.canonicalJson)(manifest.source_notebook)
|
|
1164
|
+
|| (0, canonical_json_1.canonicalJson)(headPayload.notebook) !== (0, canonical_json_1.canonicalJson)(manifest.notebook)
|
|
1165
|
+
|| headPayload.inputManifestSha256 !== manifest.input_manifest_sha256
|
|
1166
|
+
|| headPayload.environmentManifestSha256 !== manifest.environment_manifest_sha256
|
|
1167
|
+
|| headPayload.runtimeEnvironmentSha256 !== manifest.runtime_environment_sha256
|
|
1168
|
+
|| (0, canonical_json_1.canonicalJson)(headPayload.cells) !== (0, canonical_json_1.canonicalJson)(expectedEventCells)
|
|
1169
|
+
|| (0, canonical_json_1.canonicalJson)(headPayload.artifacts) !== (0, canonical_json_1.canonicalJson)(expectedEventArtifacts)
|
|
1170
|
+
|| (0, canonical_json_1.canonicalJson)(headPayload.isolation) !== (0, canonical_json_1.canonicalJson)({
|
|
1171
|
+
freshKernel: true,
|
|
1172
|
+
networkIsolated: false,
|
|
1173
|
+
filesystemIsolated: false,
|
|
1174
|
+
})) {
|
|
1175
|
+
throw new Error('completed event does not attest this seal');
|
|
1176
|
+
}
|
|
1177
|
+
return manifest;
|
|
1178
|
+
}
|
|
1179
|
+
verifyLedgerPrefix(prefix, notebookPath) {
|
|
1180
|
+
const lines = prefix.toString('utf8').split('\n');
|
|
1181
|
+
if (lines.at(-1) !== '')
|
|
1182
|
+
throw new Error('ledger prefix lacks final newline');
|
|
1183
|
+
lines.pop();
|
|
1184
|
+
let previousHash = null;
|
|
1185
|
+
let head;
|
|
1186
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
1187
|
+
const record = this.parseObject(lines[index], `ledger event ${index + 1}`);
|
|
1188
|
+
const eventHash = record.eventHash;
|
|
1189
|
+
if (typeof eventHash !== 'string' || !SHA256_RE.test(eventHash)) {
|
|
1190
|
+
throw new Error(`ledger event ${index + 1} has an invalid hash`);
|
|
1191
|
+
}
|
|
1192
|
+
const { eventHash: _eventHash, ...unsigned } = record;
|
|
1193
|
+
if ((0, canonical_json_1.sha256Canonical)(unsigned) !== eventHash
|
|
1194
|
+
|| record.seq !== index + 1
|
|
1195
|
+
|| record.prevHash !== previousHash
|
|
1196
|
+
|| record.notebookPath !== notebookPath) {
|
|
1197
|
+
throw new Error(`ledger event ${index + 1} failed chain verification`);
|
|
1198
|
+
}
|
|
1199
|
+
if (typeof record.type !== 'string')
|
|
1200
|
+
throw new Error(`ledger event ${index + 1} has no type`);
|
|
1201
|
+
head = {
|
|
1202
|
+
eventHash,
|
|
1203
|
+
seq: index + 1,
|
|
1204
|
+
type: record.type,
|
|
1205
|
+
payload: record.payload,
|
|
1206
|
+
record,
|
|
1207
|
+
};
|
|
1208
|
+
previousHash = eventHash;
|
|
1209
|
+
}
|
|
1210
|
+
if (!head)
|
|
1211
|
+
throw new Error('ledger prefix contains no events');
|
|
1212
|
+
return head;
|
|
1213
|
+
}
|
|
1214
|
+
async verifyManifestCells(sourceNotebookPath, sealedNotebookPath, cells) {
|
|
1215
|
+
const readNotebook = async (filePath) => {
|
|
1216
|
+
const raw = JSON.parse(await fsp.readFile(filePath, 'utf8'));
|
|
1217
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw) || !Array.isArray(raw.cells)) {
|
|
1218
|
+
throw new Error(`Invalid notebook JSON during seal verification: ${filePath}`);
|
|
1219
|
+
}
|
|
1220
|
+
return raw;
|
|
1221
|
+
};
|
|
1222
|
+
const source = await readNotebook(sourceNotebookPath);
|
|
1223
|
+
const sealed = await readNotebook(sealedNotebookPath);
|
|
1224
|
+
const sourceCodeCells = source.cells.filter((cell) => cell.cell_type === 'code');
|
|
1225
|
+
const sealedCodeCells = sealed.cells.filter((cell) => cell.cell_type === 'code');
|
|
1226
|
+
if (sourceCodeCells.length !== cells.length || sealedCodeCells.length !== cells.length) {
|
|
1227
|
+
throw new Error('manifest executed-cell count does not match notebook code cells');
|
|
1228
|
+
}
|
|
1229
|
+
for (let index = 0; index < cells.length; index += 1) {
|
|
1230
|
+
const record = cells[index];
|
|
1231
|
+
const original = sourceCodeCells[index];
|
|
1232
|
+
const replayed = sealedCodeCells[index];
|
|
1233
|
+
const originalSource = this.cellSource(original.source, index);
|
|
1234
|
+
const replayedSource = this.cellSource(replayed.source, index);
|
|
1235
|
+
const originalOutputs = original.outputs ?? [];
|
|
1236
|
+
const replayedOutputs = replayed.outputs ?? [];
|
|
1237
|
+
if (!Array.isArray(originalOutputs) || !Array.isArray(replayedOutputs)) {
|
|
1238
|
+
throw new Error(`Invalid notebook outputs for cell ${record.cell_id}`);
|
|
1239
|
+
}
|
|
1240
|
+
const exploratoryDigest = (0, canonical_json_1.sha256Canonical)(normalizeNotebookOutputsForComparison(originalOutputs));
|
|
1241
|
+
const replayComparisonDigest = (0, canonical_json_1.sha256Canonical)(normalizeNotebookOutputsForComparison(replayedOutputs));
|
|
1242
|
+
if (original.id !== record.cell_id
|
|
1243
|
+
|| replayed.id !== record.cell_id
|
|
1244
|
+
|| originalSource !== replayedSource
|
|
1245
|
+
|| (0, canonical_json_1.sha256Hex)(replayedSource) !== record.source_sha256
|
|
1246
|
+
|| (0, canonical_json_1.sha256Canonical)(replayedOutputs) !== record.outputs_sha256
|
|
1247
|
+
|| exploratoryDigest !== record.exploratory_outputs_sha256
|
|
1248
|
+
|| replayComparisonDigest !== record.replay_comparison_outputs_sha256
|
|
1249
|
+
|| record.outputs_match_exploratory !== (exploratoryDigest === replayComparisonDigest)
|
|
1250
|
+
|| (original.execution_count ?? null) !== record.exploratory_execution_count
|
|
1251
|
+
|| replayed.execution_count !== record.execution_count
|
|
1252
|
+
|| record.status !== 'completed') {
|
|
1253
|
+
throw new Error(`manifest cell attestation mismatch for ${record.cell_id}`);
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
async hashRegularFile(filePath) {
|
|
1258
|
+
let handle;
|
|
1259
|
+
try {
|
|
1260
|
+
handle = await fsp.open(filePath, fs_1.constants.O_RDONLY | fs_1.constants.O_NOFOLLOW);
|
|
1261
|
+
}
|
|
1262
|
+
catch (error) {
|
|
1263
|
+
if (this.isNodeError(error, 'ELOOP'))
|
|
1264
|
+
throw new Error(`File is a symbolic link: ${filePath}`);
|
|
1265
|
+
throw error;
|
|
1266
|
+
}
|
|
1267
|
+
try {
|
|
1268
|
+
const before = await handle.stat();
|
|
1269
|
+
if (!before.isFile())
|
|
1270
|
+
throw new Error(`Not a regular file: ${filePath}`);
|
|
1271
|
+
const hash = (0, crypto_1.createHash)('sha256');
|
|
1272
|
+
const buffer = Buffer.allocUnsafe(1024 * 1024);
|
|
1273
|
+
let sizeBytes = 0;
|
|
1274
|
+
for (;;) {
|
|
1275
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
|
|
1276
|
+
if (bytesRead === 0)
|
|
1277
|
+
break;
|
|
1278
|
+
hash.update(buffer.subarray(0, bytesRead));
|
|
1279
|
+
sizeBytes += bytesRead;
|
|
1280
|
+
}
|
|
1281
|
+
const after = await handle.stat();
|
|
1282
|
+
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size
|
|
1283
|
+
|| before.mtimeMs !== after.mtimeMs || after.size !== sizeBytes) {
|
|
1284
|
+
throw new Error(`File changed during verification: ${filePath}`);
|
|
1285
|
+
}
|
|
1286
|
+
return { sha256: hash.digest('hex'), size_bytes: sizeBytes };
|
|
1287
|
+
}
|
|
1288
|
+
finally {
|
|
1289
|
+
await handle.close();
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
registryPath(sealId) {
|
|
1293
|
+
return path.join(this.registryDirectory, `${sealId}.json`);
|
|
1294
|
+
}
|
|
1295
|
+
async ensureRegistryDirectory() {
|
|
1296
|
+
await this.ensureSafeDirectoryChain(this.rootDirectory, ['.nebula', 'replay-seal-jobs']);
|
|
1297
|
+
}
|
|
1298
|
+
async createSafeSealDirectory(workspace, sealId) {
|
|
1299
|
+
const sealsDirectory = await this.ensureSafeDirectoryChain(workspace, ['.nebula', 'seals']);
|
|
1300
|
+
const sealDirectory = path.join(sealsDirectory, sealId);
|
|
1301
|
+
try {
|
|
1302
|
+
await fsp.mkdir(sealDirectory, { mode: 0o700 });
|
|
1303
|
+
if (this.durable)
|
|
1304
|
+
await this.syncDirectory(sealsDirectory);
|
|
1305
|
+
}
|
|
1306
|
+
catch (error) {
|
|
1307
|
+
if (!this.isNodeError(error, 'EEXIST'))
|
|
1308
|
+
throw error;
|
|
1309
|
+
const stat = await fsp.lstat(sealDirectory);
|
|
1310
|
+
if (stat.isSymbolicLink() || !stat.isDirectory()) {
|
|
1311
|
+
throw new ReplaySealValidationError('Seal path is not a real directory');
|
|
1312
|
+
}
|
|
1313
|
+
// An indexed job is handled before this method. An unindexed directory
|
|
1314
|
+
// is ambiguous crash/attacker state and must never be adopted silently.
|
|
1315
|
+
throw new ReplaySealValidationError(`Seal directory already exists without authoritative job state: ${sealDirectory}`);
|
|
1316
|
+
}
|
|
1317
|
+
return await fsp.realpath(sealDirectory);
|
|
1318
|
+
}
|
|
1319
|
+
async ensureSafeDirectoryChain(base, components) {
|
|
1320
|
+
let current = await fsp.realpath(base);
|
|
1321
|
+
const baseStat = await fsp.lstat(current);
|
|
1322
|
+
if (!baseStat.isDirectory())
|
|
1323
|
+
throw new Error(`Directory root is not a directory: ${base}`);
|
|
1324
|
+
for (const component of components) {
|
|
1325
|
+
if (!component || component === '.' || component === '..' || component.includes(path.sep)) {
|
|
1326
|
+
throw new Error(`Unsafe directory component: ${component}`);
|
|
1327
|
+
}
|
|
1328
|
+
const candidate = path.join(current, component);
|
|
1329
|
+
try {
|
|
1330
|
+
await fsp.mkdir(candidate, { mode: 0o700 });
|
|
1331
|
+
if (this.durable)
|
|
1332
|
+
await this.syncDirectory(current);
|
|
1333
|
+
}
|
|
1334
|
+
catch (error) {
|
|
1335
|
+
if (!this.isNodeError(error, 'EEXIST'))
|
|
1336
|
+
throw error;
|
|
1337
|
+
}
|
|
1338
|
+
const stat = await fsp.lstat(candidate);
|
|
1339
|
+
if (stat.isSymbolicLink() || !stat.isDirectory()) {
|
|
1340
|
+
throw new ReplaySealValidationError(`Refusing unsafe directory in seal path: ${candidate}`);
|
|
1341
|
+
}
|
|
1342
|
+
const realCandidate = await fsp.realpath(candidate);
|
|
1343
|
+
if (path.dirname(realCandidate) !== current) {
|
|
1344
|
+
throw new ReplaySealValidationError(`Seal directory escapes its parent: ${candidate}`);
|
|
1345
|
+
}
|
|
1346
|
+
current = realCandidate;
|
|
1347
|
+
}
|
|
1348
|
+
return current;
|
|
1349
|
+
}
|
|
1350
|
+
async hardenSealDirectory(directory) {
|
|
1351
|
+
const visit = async (current) => {
|
|
1352
|
+
const stat = await fsp.lstat(current);
|
|
1353
|
+
if (stat.isSymbolicLink())
|
|
1354
|
+
throw new Error(`Seal contains a symbolic link: ${current}`);
|
|
1355
|
+
if (!stat.isDirectory()) {
|
|
1356
|
+
if (!stat.isFile())
|
|
1357
|
+
throw new Error(`Seal contains a non-regular entry: ${current}`);
|
|
1358
|
+
await fsp.chmod(current, 0o400);
|
|
1359
|
+
return;
|
|
1360
|
+
}
|
|
1361
|
+
const entries = await fsp.readdir(current);
|
|
1362
|
+
for (const entry of entries)
|
|
1363
|
+
await visit(path.join(current, entry));
|
|
1364
|
+
await fsp.chmod(current, 0o500);
|
|
1365
|
+
};
|
|
1366
|
+
await visit(directory);
|
|
1367
|
+
}
|
|
1368
|
+
sealRequestEventKey(sealId) {
|
|
1369
|
+
return `replay-seal:${sealId}`;
|
|
1370
|
+
}
|
|
1371
|
+
async persistJob(job) {
|
|
1372
|
+
await Promise.all([
|
|
1373
|
+
this.writeJsonAtomic(this.registryPath(job.sealId), job),
|
|
1374
|
+
this.writeJsonAtomic(path.join(job.sealDirectory, 'job.json'), job),
|
|
1375
|
+
]);
|
|
1376
|
+
}
|
|
1377
|
+
async updateJob(job, patch) {
|
|
1378
|
+
const updated = {
|
|
1379
|
+
...job,
|
|
1380
|
+
...patch,
|
|
1381
|
+
updatedAt: this.clock().toISOString(),
|
|
1382
|
+
};
|
|
1383
|
+
if (patch.manifest === undefined)
|
|
1384
|
+
delete updated.manifest;
|
|
1385
|
+
if (patch.error === undefined)
|
|
1386
|
+
delete updated.error;
|
|
1387
|
+
await this.persistJob(updated);
|
|
1388
|
+
return updated;
|
|
1389
|
+
}
|
|
1390
|
+
async readJob(filePath) {
|
|
1391
|
+
let raw;
|
|
1392
|
+
try {
|
|
1393
|
+
const stat = await fsp.lstat(filePath);
|
|
1394
|
+
if (stat.isSymbolicLink() || !stat.isFile()) {
|
|
1395
|
+
throw new Error(`Replay-seal job state must be a regular file: ${filePath}`);
|
|
1396
|
+
}
|
|
1397
|
+
raw = await fsp.readFile(filePath, 'utf8');
|
|
1398
|
+
}
|
|
1399
|
+
catch (error) {
|
|
1400
|
+
if (this.isNodeError(error, 'ENOENT'))
|
|
1401
|
+
return null;
|
|
1402
|
+
throw error;
|
|
1403
|
+
}
|
|
1404
|
+
const parsed = JSON.parse(raw);
|
|
1405
|
+
if (!parsed || parsed.schemaVersion !== 1 || !SEAL_ID_RE.test(parsed.sealId)) {
|
|
1406
|
+
throw new Error(`Invalid replay-seal job state: ${filePath}`);
|
|
1407
|
+
}
|
|
1408
|
+
return parsed;
|
|
1409
|
+
}
|
|
1410
|
+
async writeJsonAtomic(filePath, value) {
|
|
1411
|
+
const directory = path.dirname(filePath);
|
|
1412
|
+
const temporary = `${filePath}.${process.pid}.${(0, crypto_1.randomUUID)()}.tmp`;
|
|
1413
|
+
const handle = await fsp.open(temporary, 'wx', 0o600);
|
|
1414
|
+
try {
|
|
1415
|
+
await handle.writeFile(`${(0, canonical_json_1.canonicalJson)(value)}\n`, 'utf8');
|
|
1416
|
+
if (this.durable)
|
|
1417
|
+
await handle.sync();
|
|
1418
|
+
}
|
|
1419
|
+
finally {
|
|
1420
|
+
await handle.close();
|
|
1421
|
+
}
|
|
1422
|
+
await fsp.rename(temporary, filePath);
|
|
1423
|
+
}
|
|
1424
|
+
async writeImmutable(filePath, bytes) {
|
|
1425
|
+
const directory = path.dirname(filePath);
|
|
1426
|
+
const temporary = path.join(directory, `.${path.basename(filePath)}.${process.pid}.${(0, crypto_1.randomUUID)()}.tmp`);
|
|
1427
|
+
try {
|
|
1428
|
+
const handle = await fsp.open(temporary, fs_1.constants.O_WRONLY | fs_1.constants.O_CREAT | fs_1.constants.O_EXCL | fs_1.constants.O_NOFOLLOW, 0o400);
|
|
1429
|
+
try {
|
|
1430
|
+
await handle.writeFile(bytes);
|
|
1431
|
+
if (this.durable)
|
|
1432
|
+
await handle.sync();
|
|
1433
|
+
}
|
|
1434
|
+
finally {
|
|
1435
|
+
await handle.close();
|
|
1436
|
+
}
|
|
1437
|
+
try {
|
|
1438
|
+
await fsp.link(temporary, filePath);
|
|
1439
|
+
}
|
|
1440
|
+
catch (error) {
|
|
1441
|
+
if (!this.isNodeError(error, 'EEXIST'))
|
|
1442
|
+
throw error;
|
|
1443
|
+
const stat = await fsp.lstat(filePath);
|
|
1444
|
+
if (stat.isSymbolicLink() || !stat.isFile()) {
|
|
1445
|
+
throw new Error(`Immutable seal target is not a regular file: ${filePath}`);
|
|
1446
|
+
}
|
|
1447
|
+
const existing = await fsp.readFile(filePath);
|
|
1448
|
+
if (!existing.equals(bytes)) {
|
|
1449
|
+
throw new Error(`Immutable seal file already exists with different content: ${filePath}`);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
await fsp.chmod(filePath, 0o400);
|
|
1453
|
+
if (this.durable)
|
|
1454
|
+
await this.syncDirectory(directory);
|
|
1455
|
+
}
|
|
1456
|
+
finally {
|
|
1457
|
+
await fsp.unlink(temporary).catch(() => undefined);
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
async readNotebookSource(notebookPath) {
|
|
1461
|
+
let lexicalStat;
|
|
1462
|
+
try {
|
|
1463
|
+
lexicalStat = await fsp.lstat(notebookPath);
|
|
1464
|
+
}
|
|
1465
|
+
catch (error) {
|
|
1466
|
+
if (this.isNodeError(error, 'ENOENT'))
|
|
1467
|
+
throw new ReplaySealValidationError(`Notebook not found: ${notebookPath}`);
|
|
1468
|
+
throw error;
|
|
1469
|
+
}
|
|
1470
|
+
if (lexicalStat.isSymbolicLink() || !lexicalStat.isFile()) {
|
|
1471
|
+
throw new ReplaySealValidationError('notebook_path must be a regular file, not a symbolic link');
|
|
1472
|
+
}
|
|
1473
|
+
const rootRealPath = await fsp.realpath(this.rootDirectory);
|
|
1474
|
+
const notebookRealPath = await fsp.realpath(notebookPath);
|
|
1475
|
+
this.relativeInside(rootRealPath, notebookRealPath, 'notebook_path');
|
|
1476
|
+
await this.assertNoSymlinkComponents(path.resolve(this.rootDirectory), path.resolve(notebookPath));
|
|
1477
|
+
let handle;
|
|
1478
|
+
try {
|
|
1479
|
+
handle = await fsp.open(notebookPath, fs_1.constants.O_RDONLY | fs_1.constants.O_NOFOLLOW);
|
|
1480
|
+
}
|
|
1481
|
+
catch (error) {
|
|
1482
|
+
if (this.isNodeError(error, 'ELOOP')) {
|
|
1483
|
+
throw new ReplaySealValidationError('notebook_path became a symbolic link during preflight');
|
|
1484
|
+
}
|
|
1485
|
+
throw error;
|
|
1486
|
+
}
|
|
1487
|
+
try {
|
|
1488
|
+
const before = await handle.stat();
|
|
1489
|
+
if (!before.isFile()
|
|
1490
|
+
|| before.dev !== lexicalStat.dev
|
|
1491
|
+
|| before.ino !== lexicalStat.ino) {
|
|
1492
|
+
throw new ReplaySealValidationError('notebook_path changed during preflight');
|
|
1493
|
+
}
|
|
1494
|
+
const bytes = await handle.readFile();
|
|
1495
|
+
const after = await handle.stat();
|
|
1496
|
+
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size
|
|
1497
|
+
|| before.mtimeMs !== after.mtimeMs || bytes.length !== after.size) {
|
|
1498
|
+
throw new ReplaySealValidationError('notebook_path changed while it was frozen');
|
|
1499
|
+
}
|
|
1500
|
+
return bytes;
|
|
1501
|
+
}
|
|
1502
|
+
finally {
|
|
1503
|
+
await handle.close();
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
async captureLedgerPrefix(notebookPath, headHash, headSeq) {
|
|
1507
|
+
const ledgerPath = this.provenanceStore.getLedgerPath(notebookPath);
|
|
1508
|
+
const bytes = await fsp.readFile(ledgerPath);
|
|
1509
|
+
let offset = 0;
|
|
1510
|
+
let foundEnd = -1;
|
|
1511
|
+
while (offset < bytes.length) {
|
|
1512
|
+
const newline = bytes.indexOf(0x0a, offset);
|
|
1513
|
+
if (newline < 0)
|
|
1514
|
+
throw new Error('Provenance ledger does not end on an event boundary');
|
|
1515
|
+
const line = bytes.subarray(offset, newline).toString('utf8');
|
|
1516
|
+
const event = JSON.parse(line);
|
|
1517
|
+
if (event.eventHash === headHash && event.seq === headSeq) {
|
|
1518
|
+
foundEnd = newline + 1;
|
|
1519
|
+
break;
|
|
1520
|
+
}
|
|
1521
|
+
offset = newline + 1;
|
|
1522
|
+
}
|
|
1523
|
+
if (foundEnd < 0)
|
|
1524
|
+
throw new Error('Completed seal event is missing from provenance ledger');
|
|
1525
|
+
const prefix = bytes.subarray(0, foundEnd);
|
|
1526
|
+
return {
|
|
1527
|
+
path: ledgerPath,
|
|
1528
|
+
prefix_size_bytes: prefix.length,
|
|
1529
|
+
prefix_sha256: (0, canonical_json_1.sha256Hex)(prefix),
|
|
1530
|
+
head_sha256: headHash,
|
|
1531
|
+
head_seq: headSeq,
|
|
1532
|
+
};
|
|
1533
|
+
}
|
|
1534
|
+
toJupyterOutput(output) {
|
|
1535
|
+
switch (output.type) {
|
|
1536
|
+
case 'stdout':
|
|
1537
|
+
case 'stderr':
|
|
1538
|
+
return { output_type: 'stream', name: output.type, text: output.content };
|
|
1539
|
+
case 'error':
|
|
1540
|
+
return {
|
|
1541
|
+
output_type: 'error',
|
|
1542
|
+
ename: 'Error',
|
|
1543
|
+
evalue: output.content,
|
|
1544
|
+
traceback: output.content.split('\n'),
|
|
1545
|
+
};
|
|
1546
|
+
case 'image':
|
|
1547
|
+
return { output_type: 'display_data', data: { 'image/png': output.content }, metadata: output.metadata ?? {} };
|
|
1548
|
+
case 'html':
|
|
1549
|
+
return { output_type: 'display_data', data: { 'text/html': output.content }, metadata: output.metadata ?? {} };
|
|
1550
|
+
case 'display_data':
|
|
1551
|
+
if (output.jupyterOutputType === 'execute_result') {
|
|
1552
|
+
return {
|
|
1553
|
+
output_type: 'execute_result',
|
|
1554
|
+
data: output.mimeBundle ?? { 'text/plain': output.content },
|
|
1555
|
+
metadata: output.metadata ?? {},
|
|
1556
|
+
execution_count: output.jupyterExecutionCount ?? null,
|
|
1557
|
+
};
|
|
1558
|
+
}
|
|
1559
|
+
return {
|
|
1560
|
+
output_type: 'display_data',
|
|
1561
|
+
data: output.mimeBundle ?? { 'text/plain': output.content },
|
|
1562
|
+
metadata: output.metadata ?? {},
|
|
1563
|
+
};
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
markerPayload(stdout, context) {
|
|
1567
|
+
const begin = stdout.lastIndexOf(PROBE_BEGIN);
|
|
1568
|
+
const end = stdout.indexOf(PROBE_END, begin + PROBE_BEGIN.length);
|
|
1569
|
+
if (begin < 0 || end < 0)
|
|
1570
|
+
throw new Error(`Fresh kernel ${context} returned no attestation payload`);
|
|
1571
|
+
return stdout.slice(begin + PROBE_BEGIN.length, end);
|
|
1572
|
+
}
|
|
1573
|
+
parseObject(raw, context) {
|
|
1574
|
+
let parsed;
|
|
1575
|
+
try {
|
|
1576
|
+
parsed = JSON.parse(raw);
|
|
1577
|
+
}
|
|
1578
|
+
catch {
|
|
1579
|
+
throw new Error(`Fresh kernel ${context} returned invalid JSON`);
|
|
1580
|
+
}
|
|
1581
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
1582
|
+
throw new Error(`Fresh kernel ${context} returned a non-object payload`);
|
|
1583
|
+
}
|
|
1584
|
+
return parsed;
|
|
1585
|
+
}
|
|
1586
|
+
stringMap(value, context) {
|
|
1587
|
+
const record = this.plainRecord(value, context);
|
|
1588
|
+
const result = {};
|
|
1589
|
+
for (const key of Object.keys(record).sort()) {
|
|
1590
|
+
if (typeof record[key] !== 'string')
|
|
1591
|
+
throw new Error(`${context} values must be strings`);
|
|
1592
|
+
result[key] = record[key];
|
|
1593
|
+
}
|
|
1594
|
+
return result;
|
|
1595
|
+
}
|
|
1596
|
+
plainRecord(value, name) {
|
|
1597
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
1598
|
+
throw new ReplaySealValidationError(`${name} must be an object`);
|
|
1599
|
+
}
|
|
1600
|
+
(0, canonical_json_1.canonicalJson)(value);
|
|
1601
|
+
return this.cloneJson(value);
|
|
1602
|
+
}
|
|
1603
|
+
optionalRecord(value, name) {
|
|
1604
|
+
if (value === undefined || value === null)
|
|
1605
|
+
return null;
|
|
1606
|
+
return this.plainRecord(value, name);
|
|
1607
|
+
}
|
|
1608
|
+
nonempty(value, name) {
|
|
1609
|
+
if (typeof value !== 'string' || value.trim().length === 0 || /\u0000/.test(value)) {
|
|
1610
|
+
throw new ReplaySealValidationError(`${name} must be non-empty text`);
|
|
1611
|
+
}
|
|
1612
|
+
return value;
|
|
1613
|
+
}
|
|
1614
|
+
cellSource(value, index) {
|
|
1615
|
+
if (typeof value === 'string')
|
|
1616
|
+
return value;
|
|
1617
|
+
if (Array.isArray(value) && value.every((item) => typeof item === 'string'))
|
|
1618
|
+
return value.join('');
|
|
1619
|
+
throw new ReplaySealValidationError(`Notebook code cell ${index + 1} has invalid source`);
|
|
1620
|
+
}
|
|
1621
|
+
relativeInside(root, target, name) {
|
|
1622
|
+
const relative = path.relative(path.resolve(root), path.resolve(target));
|
|
1623
|
+
if (!relative || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
|
1624
|
+
throw new ReplaySealValidationError(`${name} must be a distinct path inside the notebook workspace`);
|
|
1625
|
+
}
|
|
1626
|
+
return relative;
|
|
1627
|
+
}
|
|
1628
|
+
async assertNoSymlinkComponents(root, target) {
|
|
1629
|
+
const relative = this.relativeInside(root, target, 'artifact path');
|
|
1630
|
+
let current = path.resolve(root);
|
|
1631
|
+
for (const component of relative.split(path.sep)) {
|
|
1632
|
+
current = path.join(current, component);
|
|
1633
|
+
let stat;
|
|
1634
|
+
try {
|
|
1635
|
+
stat = await fsp.lstat(current);
|
|
1636
|
+
}
|
|
1637
|
+
catch (error) {
|
|
1638
|
+
if (this.isNodeError(error, 'ENOENT'))
|
|
1639
|
+
throw new Error(`Requested artifact is missing: ${target}`);
|
|
1640
|
+
throw error;
|
|
1641
|
+
}
|
|
1642
|
+
if (stat.isSymbolicLink())
|
|
1643
|
+
throw new Error(`Requested artifact path contains a symbolic link: ${target}`);
|
|
1644
|
+
}
|
|
1645
|
+
const real = await fsp.realpath(target);
|
|
1646
|
+
const realRoot = await fsp.realpath(root);
|
|
1647
|
+
this.relativeInside(realRoot, real, 'resolved artifact path');
|
|
1648
|
+
}
|
|
1649
|
+
assertNoSymlinkComponentsSync(root, target) {
|
|
1650
|
+
const relative = this.relativeInside(root, target, 'path');
|
|
1651
|
+
let current = path.resolve(root);
|
|
1652
|
+
for (const component of relative.split(path.sep)) {
|
|
1653
|
+
current = path.join(current, component);
|
|
1654
|
+
let stat;
|
|
1655
|
+
try {
|
|
1656
|
+
stat = (0, fs_1.lstatSync)(current);
|
|
1657
|
+
}
|
|
1658
|
+
catch {
|
|
1659
|
+
throw new ReplaySealValidationError(`Path component does not exist: ${current}`);
|
|
1660
|
+
}
|
|
1661
|
+
if (stat.isSymbolicLink()) {
|
|
1662
|
+
throw new ReplaySealValidationError(`Path contains a symbolic link: ${current}`);
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
}
|
|
1666
|
+
async syncDirectory(directory) {
|
|
1667
|
+
try {
|
|
1668
|
+
const handle = await fsp.open(directory, 'r');
|
|
1669
|
+
try {
|
|
1670
|
+
await handle.sync();
|
|
1671
|
+
}
|
|
1672
|
+
finally {
|
|
1673
|
+
await handle.close();
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
catch {
|
|
1677
|
+
// Some filesystems do not support directory fsync. File data remains
|
|
1678
|
+
// fsynced and create/rename operations still retain atomicity.
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
cloneJson(value) {
|
|
1682
|
+
return JSON.parse((0, canonical_json_1.canonicalJson)(value));
|
|
1683
|
+
}
|
|
1684
|
+
errorMessage(error) {
|
|
1685
|
+
return error instanceof Error ? error.message : String(error);
|
|
1686
|
+
}
|
|
1687
|
+
isNodeError(error, code) {
|
|
1688
|
+
return !!error && typeof error === 'object' && error.code === code;
|
|
1689
|
+
}
|
|
1690
|
+
runSerialized(key, task) {
|
|
1691
|
+
const previous = ReplaySealService.queues.get(key) ?? Promise.resolve();
|
|
1692
|
+
const result = previous.catch(() => undefined).then(task);
|
|
1693
|
+
const settled = result.then(() => undefined, () => undefined);
|
|
1694
|
+
ReplaySealService.queues.set(key, settled);
|
|
1695
|
+
settled.finally(() => {
|
|
1696
|
+
if (ReplaySealService.queues.get(key) === settled)
|
|
1697
|
+
ReplaySealService.queues.delete(key);
|
|
1698
|
+
});
|
|
1699
|
+
return result;
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
exports.ReplaySealService = ReplaySealService;
|