nebula-notebook 0.2.15 → 0.2.17

package/node-server/dist/provenance/types.d.ts

@@ -0,0 +1,65 @@
+import type { CanonicalJsonValue } from './canonical-json';
+export interface ProvenanceActor {
+    /** Authenticated principal category, for example user, agent, or system. */
+    kind: string;
+    /** Stable principal identifier within the actor category. */
+    id?: string;
+    /** Optional human-readable client/principal label. */
+    name?: string;
+    attributes?: Record<string, CanonicalJsonValue>;
+}
+export interface ProvenanceEventInput {
+    type: string;
+    actor: ProvenanceActor;
+    payload?: CanonicalJsonValue;
+    runId?: string;
+    taskId?: string;
+    sourceId?: string;
+    executionId?: string;
+    inputManifestSha256?: string;
+    environmentManifestSha256?: string;
+    idempotencyKey?: string;
+}
+export interface ProvenanceEvent {
+    schemaVersion: 1;
+    eventId: string;
+    seq: number;
+    timestamp: string;
+    notebookPath: string;
+    type: string;
+    actor: ProvenanceActor;
+    payload: CanonicalJsonValue;
+    runId?: string;
+    taskId?: string;
+    sourceId?: string;
+    executionId?: string;
+    inputManifestSha256?: string;
+    environmentManifestSha256?: string;
+    idempotencyKey?: string;
+    idempotencyRequestHash?: string;
+    prevHash: string | null;
+    eventHash: string;
+}
+export interface ProvenanceAppendResult {
+    event: ProvenanceEvent;
+    appended: boolean;
+}
+export interface ProvenanceVerification {
+    valid: boolean;
+    eventCount: number;
+    lastSeq: number;
+    headHash: string | null;
+    error?: string;
+    failedLine?: number;
+}
+export interface ProvenanceBlobReference {
+    sha256: string;
+    sizeBytes: number;
+}
+export interface ProvenanceBlobVerification {
+    valid: boolean;
+    expectedSha256: string;
+    actualSha256?: string;
+    sizeBytes?: number;
+    error?: string;
+}

package/node-server/dist/provenance/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/node-server/dist/routes/fs.js

@@ -43,6 +43,12 @@ Object.defineProperty(exports, "fsService", { enumerable: true, get: function ()
 const path = __importStar(require("path"));
 const private_tmp_1 = require("../private-tmp");
 const nodeFs = __importStar(require("fs"));
+const sealed_path_1 = require("../fs/sealed-path");
+function sendSealedError(reply, error) {
+    if (!(error instanceof sealed_path_1.SealedPathLockedError))
+        return null;
+    return reply.code(error.statusCode).send((0, sealed_path_1.sealedErrorBody)(error));
+}
 async function fsRoutes(fastify) {
     /**
      * List directory contents
@@ -176,6 +182,9 @@ async function fsRoutes(fastify) {
             return reply.send({ status: 'ok', path: filePath });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             const message = err instanceof Error ? err.message : 'Unknown error';
             return reply.code(500).send({ detail: message });
         }
@@ -193,6 +202,9 @@ async function fsRoutes(fastify) {
             return reply.send({ status: 'ok', file: info });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             if (err instanceof Error && err.message.includes('exists')) {
                 return reply.code(409).send({ detail: err.message });
             }
@@ -215,6 +227,9 @@ async function fsRoutes(fastify) {
             return reply.send({ status: 'ok' });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             if (err instanceof Error && err.message.includes('not found')) {
                 return reply.code(404).send({ detail: err.message });
             }
@@ -237,6 +252,9 @@ async function fsRoutes(fastify) {
             return reply.send({ status: 'ok', file: info });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             if (err instanceof Error) {
                 if (err.message.includes('not found')) {
                     return reply.code(404).send({ detail: err.message });
@@ -266,6 +284,9 @@ async function fsRoutes(fastify) {
             return reply.send({ status: 'ok', file: info });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             if (err instanceof Error) {
                 if (err.message.includes('not found')) {
                     return reply.code(404).send({ detail: err.message });
@@ -373,6 +394,9 @@ async function fsRoutes(fastify) {
             if (!destPath) {
                 return reply.code(400).send({ detail: 'path is required' });
             }
+            // Reject sealed destinations before buffering the multipart payload to a
+            // temp file. FilesystemService repeats this at the actual write boundary.
+            (0, sealed_path_1.assertPathMutable)(fs_service_1.fsService.normalizePath(destPath), { operation: 'upload into' });
             // Save the uploaded file to a temp location first
             const tmpPath = path.join((0, private_tmp_1.privateTmpDir)('uploads'), `upload-${Date.now()}-${path.basename(data.filename || 'file')}`);
             const writeStream = nodeFs.createWriteStream(tmpPath);
@@ -395,6 +419,9 @@ async function fsRoutes(fastify) {
             return reply.send({ status: 'ok', file: info });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             if (err instanceof Error) {
                 if (err.message.includes('already exists')) {
                     return reply.code(409).send({ detail: err.message });

package/node-server/dist/routes/kernel.js

@@ -52,6 +52,7 @@ const operation_router_1 = require("../notebook/operation-router");
 const kernel_proxy_1 = require("../cluster/kernel-proxy");
 const server_registry_1 = require("../cluster/server-registry");
 const allocation_service_1 = require("../scheduler/allocation-service");
+const sealed_path_1 = require("../fs/sealed-path");
 // Track all WebSocket connections per kernel session for broadcasting
 const sessionWebSockets = new Map();
 // Only send streaming outputs to sockets after they've performed an initial output sync.
@@ -138,6 +139,12 @@ function sendProvisionError(reply, err) {
     reply.code(status).send({ detail: err.message, code: err.code, install_hint: err.installHint });
     return true;
 }
+function sendSealedError(reply, err) {
+    if (!(err instanceof sealed_path_1.SealedPathLockedError))
+        return false;
+    reply.code(err.statusCode).send((0, sealed_path_1.sealedErrorBody)(err));
+    return true;
+}
 async function persistNotebookKernelMetadata(filePath, kernelName) {
     // Best-effort: persisting the kernel choice into the notebook's metadata is a
     // secondary side-effect. If it fails (e.g. the notebook file doesn't exist yet,
@@ -484,6 +491,9 @@ async function kernelRoutes(fastify) {
     fastify.post('/kernels/start', async (request, reply) => {
         try {
             const { kernel_name = 'python3', cwd, file_path, server_id, client_origin } = request.body;
+            if (file_path) {
+                (0, sealed_path_1.assertPathMutable)(kernelService.normalizeNotebookPath(file_path), { operation: 'start a kernel for' });
+            }
             const localServerId = server_registry_1.serverRegistry.getLocalServerId();
             // Check if we should start on a remote server
             if (server_id && server_id !== localServerId && server_id !== 'local') {
@@ -518,6 +528,8 @@ async function kernelRoutes(fastify) {
             return reply.send({ session_id: sessionId, kernel_name, server_id: localServerId, mtime: notebookMtime });
         }
         catch (err) {
+            if (sendSealedError(reply, err))
+                return reply;
             if (sendProvisionError(reply, err))
                 return reply;
             const message = err instanceof Error ? err.message : 'Unknown error';
@@ -553,6 +565,7 @@ async function kernelRoutes(fastify) {
                 return reply.code(400).send({ detail: 'file_path is required' });
             }
             const normalizedFilePath = kernelService.normalizeNotebookPath(file_path);
+            (0, sealed_path_1.assertPathMutable)(normalizedFilePath, { operation: 'start a kernel for' });
             let effectiveKernelName = kernel_name;
             let effectiveServerId = server_id;
             const localServerId = server_registry_1.serverRegistry.getLocalServerId();
@@ -635,6 +648,8 @@ async function kernelRoutes(fastify) {
             });
         }
         catch (err) {
+            if (sendSealedError(reply, err))
+                return reply;
             if (sendProvisionError(reply, err))
                 return reply;
             const message = err instanceof Error ? err.message : 'Unknown error';

package/node-server/dist/routes/notebook.js

@@ -2,6 +2,9 @@
 /**
  * Notebook API Routes
  */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.headlessHandler = exports.operationRouter = exports.fsService = void 0;
 exports.default = notebookRoutes;
@@ -11,11 +14,19 @@ const operation_router_1 = require("../notebook/operation-router");
 Object.defineProperty(exports, "operationRouter", { enumerable: true, get: function () { return operation_router_1.operationRouter; } });
 const headless_handler_1 = require("../notebook/headless-handler");
 const kernel_1 = require("./kernel");
+const sealed_path_1 = require("../fs/sealed-path");
+const replay_seal_1 = __importDefault(require("./replay-seal"));
 // Initialize headless handler with kernel service for cell execution
 const headlessHandler = new headless_handler_1.HeadlessOperationHandler(fs_service_1.fsService, operation_router_1.operationRouter, kernel_1.kernelService);
 exports.headlessHandler = headlessHandler;
 operation_router_1.operationRouter.setHeadlessHandler(headlessHandler);
+function sendSealedError(reply, error) {
+    if (!(error instanceof sealed_path_1.SealedPathLockedError))
+        return null;
+    return reply.code(error.statusCode).send((0, sealed_path_1.sealedErrorBody)(error));
+}
 async function notebookRoutes(fastify) {
+    await fastify.register(replay_seal_1.default);
     /**
      * Get cell metadata schema
      */
@@ -55,6 +66,16 @@ async function notebookRoutes(fastify) {
                 return reply.code(400).send({ detail: 'path query parameter is required' });
             }
             const normalizedPath = fs_service_1.fsService.normalizePath(filePath);
+            const sealInfo = (0, sealed_path_1.classifySealedPath)(normalizedPath);
+            const sealHint = request.query.seal;
+            if (sealHint && (!sealInfo.sealed || sealInfo.sealId !== sealHint)) {
+                return reply.code(409).send({
+                    code: 'seal_mismatch',
+                    detail: sealInfo.sealed
+                        ? `Seal hint ${sealHint} does not match canonical path seal ${sealInfo.sealId}`
+                        : `Seal hint ${sealHint} does not identify the requested notebook path`,
+                });
+            }
             const result = await fs_service_1.fsService.getNotebookCellsWithKernel(filePath);
             return reply.send({
                 path: normalizedPath,
@@ -62,6 +83,9 @@ async function notebookRoutes(fastify) {
                 metadata: result.metadata,
                 kernelspec: result.kernelspec,
                 mtime: result.mtime,
+                access: sealInfo.sealed
+                    ? { read_only: true, reason: 'sealed', seal_id: sealInfo.sealId }
+                    : { read_only: false },
             });
         }
         catch (err) {
@@ -102,6 +126,9 @@ async function notebookRoutes(fastify) {
             return reply.send({ status: 'ok', path: filePath, mtime: result.mtime });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             const message = err instanceof Error ? err.message : 'Unknown error';
             return reply.code(500).send({ detail: message });
         }
@@ -119,6 +146,9 @@ async function notebookRoutes(fastify) {
             return reply.send({ notebook_path: notebookPath, history });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             const message = err instanceof Error ? err.message : 'Unknown error';
             return reply.code(500).send({ detail: message });
         }
@@ -136,6 +166,9 @@ async function notebookRoutes(fastify) {
             return reply.send({ status: 'ok', notebook_path });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             const message = err instanceof Error ? err.message : 'Unknown error';
             return reply.code(500).send({ detail: message });
         }
@@ -170,6 +203,9 @@ async function notebookRoutes(fastify) {
             return reply.send({ status: 'ok', notebook_path });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             const message = err instanceof Error ? err.message : 'Unknown error';
             return reply.code(500).send({ detail: message });
         }
@@ -195,6 +231,9 @@ async function notebookRoutes(fastify) {
             });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             const message = err instanceof Error ? err.message : 'Unknown error';
             return reply.code(500).send({ detail: message });
         }
@@ -272,9 +311,15 @@ async function notebookRoutes(fastify) {
                 return reply.code(400).send({ detail: 'Operation with type is required' });
             }
             const result = await operation_router_1.operationRouter.applyOperation(operation);
+            if (result.code === 'sealed_read_only') {
+                return reply.code(403).send(result);
+            }
             return reply.send(result);
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             const message = err instanceof Error ? err.message : 'Unknown error';
             return reply.send({ success: false, error: message });
         }
@@ -297,6 +342,9 @@ async function notebookRoutes(fastify) {
             });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             const message = err instanceof Error ? err.message : 'Unknown error';
             return reply.code(500).send({ detail: message });
         }
@@ -341,6 +389,9 @@ async function notebookRoutes(fastify) {
             });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             const message = err instanceof Error ? err.message : 'Unknown error';
             return reply.code(500).send({ detail: message });
         }
@@ -368,6 +419,9 @@ async function notebookRoutes(fastify) {
             return reply.send({ status: 'ok', notebook_path: filePath, mtime: updateResult.mtime });
         }
         catch (err) {
+            const sealed = sendSealedError(reply, err);
+            if (sealed)
+                return sealed;
             const message = err instanceof Error ? err.message : 'Unknown error';
             return reply.code(500).send({ detail: message });
         }

package/node-server/dist/routes/replay-seal.d.ts

@@ -0,0 +1,9 @@
+import type { FastifyInstance } from 'fastify';
+import { type ReplaySealRequest, type ReplaySealStatusResponse } from '../provenance/replay-seal-service';
+export interface ReplaySealRouteService {
+    submit(request: ReplaySealRequest, idempotencyKey: string): Promise<ReplaySealStatusResponse>;
+    getStatus(sealId: string): Promise<ReplaySealStatusResponse | null>;
+}
+/** Build the protected KOSMOS replay/seal routes with an injectable service. */
+export declare function createReplaySealRoutes(service: ReplaySealRouteService): (fastify: FastifyInstance) => Promise<void>;
+export default function replaySealRoutes(fastify: FastifyInstance): Promise<void>;

package/node-server/dist/routes/replay-seal.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createReplaySealRoutes = createReplaySealRoutes;
+exports.default = replaySealRoutes;
+const fs_service_1 = require("../fs/fs-service");
+const replay_seal_service_1 = require("../provenance/replay-seal-service");
+const provenance_store_1 = require("../provenance/provenance-store");
+const kernel_1 = require("./kernel");
+let productionService;
+function getProductionService() {
+    productionService ??= new replay_seal_service_1.ReplaySealService({
+        rootDirectory: fs_service_1.fsService.getRootDirectory(),
+        kernelService: kernel_1.kernelService,
+        provenanceStore: new provenance_store_1.ProvenanceStore(),
+    });
+    return productionService;
+}
+function errorResponse(reply, error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const code = error && typeof error === 'object' ? error.code : undefined;
+    if (error instanceof replay_seal_service_1.ReplaySealConflictError || code === 'IDEMPOTENCY_CONFLICT') {
+        return reply.code(409).send({ detail: message, code: 'idempotency_conflict' });
+    }
+    if (error instanceof replay_seal_service_1.ReplaySealValidationError || code === 'REPLAY_SEAL_VALIDATION_ERROR') {
+        return reply.code(400).send({ detail: message, code: 'invalid_replay_seal_request' });
+    }
+    return reply.code(500).send({ detail: message });
+}
+/** Build the protected KOSMOS replay/seal routes with an injectable service. */
+function createReplaySealRoutes(service) {
+    return async function replaySealRoutes(fastify) {
+        fastify.post('/notebook/replay-seal', async (request, reply) => {
+            const header = request.headers['idempotency-key'];
+            const idempotencyKey = Array.isArray(header) ? header[0] : header;
+            if (typeof idempotencyKey !== 'string' || idempotencyKey.length === 0) {
+                return reply.code(400).send({ detail: 'Idempotency-Key header is required' });
+            }
+            try {
+                const status = await service.submit(request.body, idempotencyKey);
+                return reply.code(status.status === 'pending' || status.status === 'running' ? 202 : 200)
+                    .send(status);
+            }
+            catch (error) {
+                return errorResponse(reply, error);
+            }
+        });
+        fastify.get('/notebook/replay-seal/:sealId', async (request, reply) => {
+            try {
+                const sealId = request.params.sealId;
+                if (typeof sealId !== 'string' || sealId.length === 0) {
+                    return reply.code(400).send({ detail: 'sealId is required' });
+                }
+                const status = await service.getStatus(sealId);
+                if (!status)
+                    return reply.code(404).send({ detail: `Replay seal not found: ${sealId}` });
+                return reply.send(status);
+            }
+            catch (error) {
+                return errorResponse(reply, error);
+            }
+        });
+    };
+}
+async function replaySealRoutes(fastify) {
+    await createReplaySealRoutes(getProductionService())(fastify);
+}

package/node-server/dist/scheduler/allocation-service.d.ts

@@ -23,7 +23,7 @@ export interface Allocation {
     createdAt: number;
     walltimeEndsAt?: number;
 }
-declare class AllocationService {
+export declare class AllocationService {
     private scheduler;
     private ctx;
     private allocations;
@@ -57,7 +57,8 @@ declare class AllocationService {
     create(spec: JobSpec): Promise<Allocation>;
     cancel(id: string): Promise<boolean>;
     private poll;
+    /** Last few lines of an allocation's job log (bounded read), or null. */
+    private readLogTail;
     shutdown(): void;
 }
 export declare const allocationService: AllocationService;
-export {};

package/node-server/dist/scheduler/allocation-service.js

@@ -43,11 +43,13 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.allocationService = void 0;
+exports.allocationService = exports.AllocationService = void 0;
 const crypto_1 = require("crypto");
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const job_template_1 = require("./job-template");
+const arch_1 = require("./arch");
+const util_1 = require("./util");
 const server_registry_1 = require("../cluster/server-registry");
 // Adaptive polling: fast only while a transition is imminent (job climbing
 // the queue / server booting), slow once allocations are correlated and
@@ -192,7 +194,18 @@ class AllocationService {
         const id = (0, crypto_1.randomUUID)().slice(0, 8);
         const token = (0, crypto_1.randomUUID)();
         const alloc = { id, token, spec, state: 'pending', createdAt: Date.now() };
-        const script = (0, job_template_1.renderJobScript)(spec, this.ctx, id, token);
+        // Cross-arch partitions (aarch64 queues under an x86_64 server) launch a
+        // per-arch runtime — or refuse HERE, actionably, instead of letting the
+        // job die on the node with "Exec format error" in an unread log.
+        let targetArch = null;
+        try {
+            targetArch = await this.scheduler.partitionArch(spec.partition);
+        }
+        catch {
+            targetArch = null;
+        }
+        const launchCtx = (0, arch_1.pickLaunchContext)(this.ctx, targetArch, this.ctx.serverArch);
+        const script = (0, job_template_1.renderJobScript)(spec, launchCtx, id, token);
         const scriptPath = path.join(this.ctx.stateDir, `${id}.sh`);
         fs.writeFileSync(scriptPath, script, { mode: 0o700 });
         const { jobId } = await this.scheduler.submit(scriptPath);
@@ -266,6 +279,13 @@ class AllocationService {
             else if (['completed', 'cancelled', 'failed'].includes(status.state)) {
                 alloc.state = status.state === 'failed' ? 'failed' : status.state === 'cancelled' ? 'cancelled' : 'ended';
                 alloc.reason = status.reason;
+                // A failure (or an exit that never registered) explains itself in the
+                // job log; the scheduler's reason alone ("NonZeroExitCode") does not.
+                if (alloc.state === 'failed' || (alloc.state === 'ended' && !alloc.serverId)) {
+                    const tail = this.readLogTail(alloc.id);
+                    if (tail)
+                        alloc.reason = [status.reason, tail].filter(Boolean).join(' — ');
+                }
                 dirty = true;
                 if (alloc.serverId)
                     server_registry_1.serverRegistry.unregister(alloc.serverId);
@@ -275,6 +295,28 @@ class AllocationService {
         if (dirty)
             this.persist();
     }
+    /** Last few lines of an allocation's job log (bounded read), or null. */
+    readLogTail(allocId) {
+        if (!this.ctx)
+            return null;
+        const file = path.join(this.ctx.stateDir, `${allocId}.log`);
+        try {
+            const size = fs.statSync(file).size;
+            const fd = fs.openSync(file, 'r');
+            try {
+                const want = Math.min(size, 4096);
+                const buf = Buffer.alloc(want);
+                fs.readSync(fd, buf, 0, want, size - want);
+                return (0, util_1.summarizeLogTail)(buf.toString('utf-8'));
+            }
+            finally {
+                fs.closeSync(fd);
+            }
+        }
+        catch {
+            return null;
+        }
+    }
     shutdown() {
         if (this.pollTimer) {
             clearTimeout(this.pollTimer);
@@ -282,4 +324,5 @@ class AllocationService {
         }
     }
 }
+exports.AllocationService = AllocationService;
 exports.allocationService = new AllocationService();

package/node-server/dist/scheduler/arch.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Multi-arch launch support for compute allocations.
+ *
+ * An allocation re-launches THIS Nebula install on the compute node, so the
+ * node binary and node_modules must match the compute node's CPU arch. On
+ * heterogeneous clusters (CRI: x86_64 login nodes, aarch64 ghq/pearsonq) a
+ * per-arch runtime is configured via env:
+ *
+ *   NEBULA_ARM64_NODE_BIN=/shared/node22-arm64/bin/node
+ *   NEBULA_ARM64_DIR=/shared/nebula-notebook-arm64   # checkout with arm64 node_modules
+ *
+ * Both must live on storage the compute nodes share with the login node.
+ */
+import type { LaunchContext } from './job-template';
+/** SLURM spellings (uname -m) → node's process.arch identifiers. */
+export declare function normalizeArch(raw: string): string;
+/**
+ * Read per-arch runtime overrides from the environment. An arch is configured
+ * only when BOTH its vars are present — a node binary without its matching
+ * node_modules tree (or vice versa) would just fail later and worse.
+ */
+export declare function archOverridesFromEnv(env?: Record<string, string | undefined>): NonNullable<LaunchContext['archOverrides']>;
+/**
+ * Pick the launch context for a partition's arch: the server's own install
+ * when arches match (or the arch is unknown — the status quo), the configured
+ * override when they differ, and a loud, actionable refusal otherwise. The
+ * refusal is the point: without it the job dies on the compute node with
+ * "Exec format error" in a log nobody reads.
+ */
+export declare function pickLaunchContext(ctx: LaunchContext, targetArchRaw: string | null, serverArch?: string): LaunchContext;