nebula-notebook 0.1.1 → 0.2.0

package/node-server/dist/scheduler/allocation-service.js

@@ -0,0 +1,169 @@
+"use strict";
+/**
+ * Allocation service — owns the lifecycle a compute allocation has *before* and
+ * *around* the cluster registry: submit a job, follow it through the queue, and
+ * correlate the client-server's registration (by one-time token) back to the
+ * allocation. Once correlated, the allocation is a normal registered server and
+ * kernels run on it through the existing cluster path.
+ *
+ * Phase-1 MVP: in-memory allocations, direct transport (no SSH tunnel — not
+ * needed where compute↔login is directly reachable).
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.allocationService = void 0;
+const crypto_1 = require("crypto");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const job_template_1 = require("./job-template");
+const server_registry_1 = require("../cluster/server-registry");
+const POLL_INTERVAL_MS = 5_000;
+const TERMINAL = ['ended', 'failed', 'cancelled'];
+class AllocationService {
+    scheduler = null;
+    ctx = null;
+    allocations = new Map();
+    pollTimer = null;
+    enabled = false;
+    init(scheduler, ctx) {
+        this.scheduler = scheduler;
+        this.ctx = ctx;
+        this.enabled = true;
+        fs.mkdirSync(ctx.stateDir, { recursive: true });
+        if (this.pollTimer)
+            clearInterval(this.pollTimer);
+        this.pollTimer = setInterval(() => {
+            this.poll().catch((err) => console.error('[Scheduler] poll error:', err));
+        }, POLL_INTERVAL_MS);
+    }
+    isEnabled() {
+        return this.enabled;
+    }
+    getScheduler() {
+        return this.scheduler;
+    }
+    list() {
+        return [...this.allocations.values()].sort((a, b) => b.createdAt - a.createdAt);
+    }
+    get(id) {
+        return this.allocations.get(id);
+    }
+    async create(spec) {
+        if (!this.scheduler || !this.ctx)
+            throw new Error('scheduler not initialized');
+        const id = (0, crypto_1.randomUUID)().slice(0, 8);
+        const token = (0, crypto_1.randomUUID)();
+        const alloc = { id, token, spec, state: 'pending', createdAt: Date.now() };
+        const script = (0, job_template_1.renderJobScript)(spec, this.ctx, id, token);
+        const scriptPath = path.join(this.ctx.stateDir, `${id}.sh`);
+        fs.writeFileSync(scriptPath, script, { mode: 0o700 });
+        const { jobId } = await this.scheduler.submit(scriptPath);
+        alloc.jobId = jobId;
+        this.allocations.set(id, alloc);
+        console.log(`[Scheduler] Allocation ${id} submitted as job ${jobId} (${spec.partition}${spec.qos ? '/' + spec.qos : ''})`);
+        // Kick an immediate poll so a fast-starting job doesn't sit at "pending".
+        void this.poll().catch(() => { });
+        return alloc;
+    }
+    async cancel(id) {
+        const alloc = this.allocations.get(id);
+        if (!alloc)
+            return false;
+        if (alloc.jobId && this.scheduler) {
+            try {
+                await this.scheduler.cancel(alloc.jobId);
+            }
+            catch (err) {
+                console.error(`[Scheduler] scancel failed for ${alloc.jobId}:`, err);
+            }
+        }
+        if (alloc.serverId)
+            server_registry_1.serverRegistry.unregister(alloc.serverId);
+        alloc.state = 'cancelled';
+        return true;
+    }
+    async poll() {
+        if (!this.scheduler)
+            return;
+        for (const alloc of this.allocations.values()) {
+            if (TERMINAL.includes(alloc.state))
+                continue;
+            // Correlate: has the client-server for this allocation registered yet?
+            if (!alloc.serverId) {
+                const server = server_registry_1.serverRegistry.getServerByAllocationToken(alloc.token);
+                if (server) {
+                    alloc.serverId = server.id;
+                    alloc.state = 'active';
+                    alloc.nodes = [server.host];
+                    if (!alloc.walltimeEndsAt) {
+                        alloc.walltimeEndsAt = Date.now() + alloc.spec.walltimeMinutes * 60_000;
+                    }
+                    console.log(`[Scheduler] Allocation ${alloc.id} active — registered as ${server.id}`);
+                }
+            }
+            // Follow the job through the scheduler.
+            if (!alloc.jobId)
+                continue;
+            let status;
+            try {
+                status = await this.scheduler.query(alloc.jobId);
+            }
+            catch {
+                continue;
+            }
+            if (status.state === 'running' && alloc.state === 'pending') {
+                alloc.state = 'running';
+                alloc.nodes = status.nodes.length ? status.nodes : alloc.nodes;
+                if (!alloc.walltimeEndsAt) {
+                    alloc.walltimeEndsAt = Date.now() + alloc.spec.walltimeMinutes * 60_000;
+                }
+            }
+            else if (['completed', 'cancelled', 'failed'].includes(status.state)) {
+                alloc.state = status.state === 'failed' ? 'failed' : status.state === 'cancelled' ? 'cancelled' : 'ended';
+                alloc.reason = status.reason;
+                if (alloc.serverId)
+                    server_registry_1.serverRegistry.unregister(alloc.serverId);
+                console.log(`[Scheduler] Allocation ${alloc.id} ${alloc.state} (job ${alloc.jobId})`);
+            }
+        }
+    }
+    shutdown() {
+        if (this.pollTimer) {
+            clearInterval(this.pollTimer);
+            this.pollTimer = null;
+        }
+    }
+}
+exports.allocationService = new AllocationService();

package/node-server/dist/scheduler/job-template.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Renders the sbatch script that launches a Nebula client-server inside a job.
+ *
+ * The rendered job:
+ *  - re-invokes this very Nebula install (same node binary + entry) with --client
+ *    from the same working directory (all visible on the shared filesystem),
+ *  - points it at the login-node main server,
+ *  - carries a one-time allocation token so the main server can correlate the
+ *    resulting registration back to this allocation.
+ *
+ * The client spawns its kernels locally (ZeroMQ on 127.0.0.1 on the compute
+ * node); nothing here touches the kernel transport.
+ */
+import type { JobSpec } from './types';
+export interface LaunchContext {
+    /** URL the compute node uses to reach the main server, e.g. http://login:3000 */
+    mainUrl: string;
+    secret?: string;
+    /** Absolute node binary path (shared FS), e.g. process.execPath */
+    nodeBin: string;
+    /** Node exec args, e.g. ['--import', 'tsx'] */
+    execArgv: string[];
+    /** Server entry script, e.g. process.argv[1] */
+    scriptPath: string;
+    /** Working directory to launch from (the nebula checkout) */
+    cwd: string;
+    /** Directory (on shared storage) for job scripts + logs */
+    stateDir: string;
+}
+export declare function renderJobScript(spec: JobSpec, ctx: LaunchContext, allocId: string, token: string): string;

package/node-server/dist/scheduler/job-template.js

@@ -0,0 +1,85 @@
+"use strict";
+/**
+ * Renders the sbatch script that launches a Nebula client-server inside a job.
+ *
+ * The rendered job:
+ *  - re-invokes this very Nebula install (same node binary + entry) with --client
+ *    from the same working directory (all visible on the shared filesystem),
+ *  - points it at the login-node main server,
+ *  - carries a one-time allocation token so the main server can correlate the
+ *    resulting registration back to this allocation.
+ *
+ * The client spawns its kernels locally (ZeroMQ on 127.0.0.1 on the compute
+ * node); nothing here touches the kernel transport.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderJobScript = renderJobScript;
+const util_1 = require("./util");
+const path = __importStar(require("path"));
+function renderJobScript(spec, ctx, allocId, token) {
+    const walltime = (0, util_1.formatWalltime)(spec.walltimeMinutes);
+    const launch = [ctx.nodeBin, ...ctx.execArgv, ctx.scriptPath, '--client']
+        .map(util_1.shellQuote)
+        .join(' ');
+    const directives = [
+        `#SBATCH --job-name=${spec.jobName}`,
+        `#SBATCH --partition=${spec.partition}`,
+        spec.qos ? `#SBATCH --qos=${spec.qos}` : null,
+        spec.account ? `#SBATCH --account=${spec.account}` : null,
+        `#SBATCH --cpus-per-task=${spec.cpus}`,
+        `#SBATCH --mem=${spec.memGb}G`,
+        spec.gpus ? `#SBATCH --gres=gpu:${spec.gpuType ? `${spec.gpuType}:` : ''}${spec.gpus}` : null,
+        `#SBATCH --time=${walltime}`,
+        `#SBATCH --output=${path.join(ctx.stateDir, `${allocId}.log`)}`,
+    ].filter(Boolean).join('\n');
+    const secretLine = ctx.secret
+        ? `export NEBULA_CLUSTER_SECRET=${(0, util_1.shellQuote)(ctx.secret)}\n`
+        : '';
+    // PORT=0 → the client binds an ephemeral port and registers the *actual* bound
+    // port with the main server (see index.ts client-registration wiring). Binding
+    // 0.0.0.0 makes it reachable from the login node for the kernel proxy.
+    return `#!/bin/bash
+${directives}
+
+cd ${(0, util_1.shellQuote)(ctx.cwd)}
+export PATH=${(0, util_1.shellQuote)(path.dirname(ctx.nodeBin))}:"$PATH"
+export NEBULA_MAIN_SERVER=${(0, util_1.shellQuote)(ctx.mainUrl)}
+${secretLine}export NEBULA_ALLOCATION_TOKEN=${(0, util_1.shellQuote)(token)}
+export NEBULA_SERVER_NAME=${(0, util_1.shellQuote)(spec.jobName)}
+${spec.idleTimeoutMinutes ? `export NEBULA_IDLE_EXIT_MINUTES=${Math.floor(spec.idleTimeoutMinutes)}\n` : ''}export PORT=0
+exec ${launch}
+`;
+}

package/node-server/dist/scheduler/mock-scheduler.d.ts

@@ -0,0 +1,30 @@
+/**
+ * MockScheduler — a fake batch scheduler for demos and tests, with no real
+ * cluster behind it. Enabled with `NEBULA_SCHEDULER=mock`.
+ *
+ * It implements the full `Scheduler` interface with fabricated (but plausibly
+ * shaped) partition / QoS / load data, so the real compute UI — allocation modal,
+ * cluster-load panel, server list — renders exactly as it would against a live
+ * cluster. The names here are generic on purpose (no real site's queues, hosts,
+ * accounts, or GPU SKUs) so nothing site-specific leaks into screenshots or the repo.
+ *
+ * `submit()` runs the *real* rendered client-launch script locally (via `bash`)
+ * instead of `sbatch`, so the allocation still registers, flips to an online
+ * server, and runs real kernels — the only thing faked is the scheduler itself.
+ */
+import type { Scheduler, Associations, QueueLoad, JobStatus, StartEstimate } from './types';
+export declare class MockScheduler implements Scheduler {
+    readonly name = "mock";
+    private jobs;
+    private nextJobId;
+    detect(): Promise<boolean>;
+    associations(_user: string): Promise<Associations>;
+    load(): Promise<QueueLoad>;
+    allowedQos(partition: string): Promise<string[] | null>;
+    estimateStart(): Promise<StartEstimate>;
+    submit(scriptPath: string): Promise<{
+        jobId: string;
+    }>;
+    query(jobId: string): Promise<JobStatus>;
+    cancel(jobId: string): Promise<void>;
+}

package/node-server/dist/scheduler/mock-scheduler.js

@@ -0,0 +1,121 @@
+"use strict";
+/**
+ * MockScheduler — a fake batch scheduler for demos and tests, with no real
+ * cluster behind it. Enabled with `NEBULA_SCHEDULER=mock`.
+ *
+ * It implements the full `Scheduler` interface with fabricated (but plausibly
+ * shaped) partition / QoS / load data, so the real compute UI — allocation modal,
+ * cluster-load panel, server list — renders exactly as it would against a live
+ * cluster. The names here are generic on purpose (no real site's queues, hosts,
+ * accounts, or GPU SKUs) so nothing site-specific leaks into screenshots or the repo.
+ *
+ * `submit()` runs the *real* rendered client-launch script locally (via `bash`)
+ * instead of `sbatch`, so the allocation still registers, flips to an online
+ * server, and runs real kernels — the only thing faked is the scheduler itself.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MockScheduler = void 0;
+const child_process_1 = require("child_process");
+// A visible queue wait before the job "starts", so the UI shows "Queued · waiting…"
+// before flipping to online. Tunable for capture.
+const QUEUE_DELAY_MS = Number(process.env.NEBULA_MOCK_QUEUE_MS ?? 5_000);
+// Generic, non-identifying cluster shape. Two idle CPU queues, a couple of GPU
+// queues with different cards, a big-memory queue, and a preemptible lab queue.
+function partitions() {
+    const p = (name, total, idle, timeLimit, jobs, gpus, nodes) => ({
+        name, up: true, timeLimit,
+        cpus: { idle, alloc: total - idle - 0, other: 0, total },
+        gpus,
+        nodes: nodes ?? { idle: Math.round(idle / 32), mixed: 4, alloc: 3, down: 0, total: Math.round(total / 32) },
+        jobs,
+    });
+    return [
+        p('cpu', 512, 236, '1-00:00:00', { pending: 5, running: 44 }),
+        p('cpu-long', 256, 14, '7-00:00:00', { pending: 71, running: 58 }),
+        p('gpu', 128, 46, '1-00:00:00', { pending: 3, running: 12 }, { type: 'nvidia_l40s', total: 16, idle: 7 }),
+        p('gpu-a100', 96, 22, '1-00:00:00', { pending: 6, running: 9 }, { type: 'nvidia_a100_80gb', total: 8, idle: 3 }),
+        p('bigmem', 192, 104, '2-00:00:00', { pending: 1, running: 6 }),
+        p('lab', 64, 28, '30-00:00:00', { pending: 0, running: 3 }, { type: 'nvidia_h100_80gb', total: 8, idle: 5 }),
+    ];
+}
+function qoses() {
+    return [
+        { name: 'normal', priority: 100, preemptible: false, preempts: [], jobs: { running: 58, pending: 40 } },
+        { name: 'priority', priority: 1000, preemptible: false, preempts: ['opportunistic'], maxWall: '12:00:00', jobs: { running: 9, pending: 2 } },
+        { name: 'opportunistic', priority: 1, preemptible: true, preempts: [], jobs: { running: 27, pending: 14 } },
+    ];
+}
+class MockScheduler {
+    name = 'mock';
+    jobs = new Map();
+    nextJobId = 480217;
+    async detect() {
+        return true;
+    }
+    async associations(_user) {
+        return {
+            account: 'demo-lab',
+            partitions: ['cpu', 'cpu-long', 'gpu', 'gpu-a100', 'bigmem', 'lab'],
+            qoses: ['normal', 'priority', 'opportunistic'],
+            defaultQos: 'normal',
+        };
+    }
+    async load() {
+        return { partitions: partitions(), qoses: qoses(), fetchedAt: Date.now() };
+    }
+    async allowedQos(partition) {
+        // The lab-owned queues require an explicit QoS; the open queues accept any.
+        if (partition === 'lab' || partition === 'gpu-a100')
+            return ['priority', 'opportunistic'];
+        return null;
+    }
+    async estimateStart() {
+        // The launcher uses capacity-based availability, not this dry-run estimate.
+        return {};
+    }
+    async submit(scriptPath) {
+        const jobId = String(this.nextJobId++);
+        const now = Date.now();
+        const job = { child: null, startedAt: now, spawnAt: now + QUEUE_DELAY_MS, cancelled: false, exited: false };
+        this.jobs.set(jobId, job);
+        // After a short "queue wait", run the real client-launch script locally so the
+        // allocation registers and becomes a usable online server. Advertise a generic
+        // NEBULA_HOST so the (real, local) machine's hostname never surfaces in the UI —
+        // the whole allocation reads as a fabricated compute node.
+        const nodeName = `node-${String(this.nextJobId % 90 + 10)}`;
+        setTimeout(() => {
+            if (job.cancelled)
+                return;
+            const child = (0, child_process_1.spawn)('bash', [scriptPath], {
+                stdio: 'ignore',
+                env: { ...process.env, NEBULA_HOST: nodeName },
+            });
+            job.child = child;
+            job.node = nodeName;
+            child.on('exit', () => { job.exited = true; });
+            child.on('error', () => { job.exited = true; });
+        }, QUEUE_DELAY_MS);
+        return { jobId };
+    }
+    async query(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return { state: 'unknown', nodes: [] };
+        if (job.cancelled)
+            return { state: 'cancelled', nodes: [] };
+        if (job.exited)
+            return { state: 'completed', nodes: [] };
+        if (Date.now() < job.spawnAt)
+            return { state: 'pending', nodes: [], reason: 'Resources' };
+        return { state: 'running', nodes: job.node ? [job.node] : [] };
+    }
+    async cancel(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return;
+        job.cancelled = true;
+        if (job.child && job.child.exitCode === null)
+            job.child.kill('SIGTERM');
+    }
+}
+exports.MockScheduler = MockScheduler;

package/node-server/dist/scheduler/slurm-scheduler.d.ts

@@ -0,0 +1,31 @@
+/**
+ * SLURM implementation of the Scheduler interface.
+ *
+ * Everything shells out to the standard SLURM client CLIs (sbatch, squeue,
+ * scancel, sinfo, sacctmgr, sacct). The Nebula main server runs on the login
+ * node where these are available; detection is capability-based.
+ */
+import type { Scheduler, JobSpec, JobStatus, StartEstimate, Associations, QueueLoad } from './types';
+export declare class SlurmScheduler implements Scheduler {
+    readonly name = "slurm";
+    /** Cache of per-partition required-QoS (site job-submit filters change rarely). */
+    private qosCache;
+    private static readonly LOAD_TTL_MS;
+    private static readonly ASSOC_TTL_MS;
+    private loadCache;
+    private loadInflight;
+    private assocCache;
+    private assocInflight;
+    detect(): Promise<boolean>;
+    associations(user: string): Promise<Associations>;
+    private associationsFresh;
+    load(): Promise<QueueLoad>;
+    private loadFresh;
+    allowedQos(partition: string): Promise<string[] | null>;
+    estimateStart(spec: JobSpec): Promise<StartEstimate>;
+    submit(scriptPath: string): Promise<{
+        jobId: string;
+    }>;
+    query(jobId: string): Promise<JobStatus>;
+    cancel(jobId: string): Promise<void>;
+}