@nodetool-ai/websocket 0.7.0-rc.23 → 0.8.2

package/dist/interrupted-jobs.js

@@ -0,0 +1,28 @@
+import { createLogger } from "@nodetool-ai/config";
+import { Job, sweepInterruptedAppRuns } from "@nodetool-ai/models";
+import { getInstanceId } from "./lib/instance-id.js";
+const log = createLogger("nodetool.websocket.interrupted-jobs");
+/**
+ * Fail the job rows a previous process of this server left in flight.
+ *
+ * Runs once at startup. Every in-flight run lives in the process that started
+ * it, so its row would otherwise stay `running` forever, and every editor open
+ * would reattach to it. The sweep is scoped to rows this server owned. With an
+ * instance id that is the rows stamped with it. Without one it is every row,
+ * which is only sound when no peer shares the database, so a shared database
+ * without an instance id is left alone.
+ */
+export async function sweepInterruptedJobs(processStartIso, options) {
+    const instanceId = getInstanceId();
+    if (!instanceId && options.sharedDatabase) {
+        log.info("Skipping the interrupted-job sweep: the database may be shared and no instance id is set");
+        return 0;
+    }
+    const swept = await Job.sweepInterrupted(processStartIso, instanceId);
+    await sweepInterruptedAppRuns(processStartIso, instanceId);
+    if (swept.length > 0) {
+        log.info("Failed jobs interrupted by a restart", { count: swept.length });
+    }
+    return swept.length;
+}
+//# sourceMappingURL=interrupted-jobs.js.map

package/dist/interrupted-jobs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interrupted-jobs.js","sourceRoot":"","sources":["../src/interrupted-jobs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,GAAG,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,MAAM,GAAG,GAAG,YAAY,CAAC,qCAAqC,CAAC,CAAC;AAEhE;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,eAAuB,EACvB,OAAoC;IAEpC,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,IAAI,CAAC,UAAU,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC1C,GAAG,CAAC,IAAI,CACN,0FAA0F,CAC3F,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,gBAAgB,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;IACtE,MAAM,uBAAuB,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;IAC3D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,GAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAC;AACtB,CAAC"}

package/dist/job-control.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Cancelling a run that is executing on another instance.
+ *
+ * {@link jobRunRegistry} is process-wide, so a `cancel_job` that lands on a
+ * machine which does not hold the run has nothing local to stop. The job row is
+ * what crosses the gap: the cancel is written there, and every instance
+ * re-reads its *own* running runs on a slow timer
+ * ({@link startJobCancelPoller}), cancelling any whose row now reads
+ * `cancelled`. One indexed query per tick, bounded by that instance's
+ * concurrency.
+ *
+ * So a cross-instance cancel is not instant — worst case it takes a poll
+ * interval to land. That is the deliberate trade for having exactly one
+ * transport: the durable one. A `NOTIFY` bus would shave those seconds off but
+ * would need a direct (non-transaction-pooled) connection to be anything but a
+ * silent no-op, and it could never be the signal of record anyway — the row
+ * already is.
+ *
+ * A cancel on the machine that *does* hold the run never comes through here:
+ * `cancelJob` reaches the session's hooks directly and is immediate.
+ */
+interface RemoteCancelResult {
+    cancelled: boolean;
+    workflowId: string | null;
+}
+/**
+ * Cancel a run this process does not hold, when another instance does.
+ *
+ * Scoped deliberately narrowly. Plenty of runs have no registry session
+ * anywhere — an HTTP `POST /run`, a trigger firing, an MCP call — and their
+ * rows carry no `runner_instance`. Flipping one of those to `cancelled` would
+ * report a cancellation to the client while the run went on to complete, and
+ * would leave a completed run recorded as cancelled. So a row is only treated
+ * as remotely cancellable when it names an instance, and a different one than
+ * this process; anything else keeps the caller's "not found" answer.
+ *
+ * The write is conditional on the row still being active, so losing the race
+ * against the owner's terminal write leaves the owner's outcome standing — and
+ * its result is the verdict returned here. The owner's poller does the rest.
+ */
+export declare function requestRemoteJobCancel(userId: string, jobId: string): Promise<RemoteCancelResult>;
+/**
+ * Re-read this instance's own running runs and cancel the ones whose row says
+ * they were cancelled elsewhere.
+ *
+ * Nothing running locally means no query at all, so an idle instance costs one
+ * timer wakeup. Errors are logged and the tick is skipped: a database blip must
+ * not stop the poller, and the next tick sees the same rows.
+ */
+export declare function startJobCancelPoller(): () => void;
+/**
+ * One poll pass: cancel every local run whose row now reads cancelled. Called
+ * on the timer, and directly by tests — the interval is far too slow to await.
+ */
+export declare function pollCancelledJobsOnce(): Promise<void>;
+export {};
+//# sourceMappingURL=job-control.d.ts.map

package/dist/job-control.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-control.d.ts","sourceRoot":"","sources":["../src/job-control.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAkBH,UAAU,kBAAkB;IAC1B,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,kBAAkB,CAAC,CAsB7B;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,IAAI,CA8BjD;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC,CAa3D"}

package/dist/job-control.js

@@ -0,0 +1,134 @@
+/**
+ * Cancelling a run that is executing on another instance.
+ *
+ * {@link jobRunRegistry} is process-wide, so a `cancel_job` that lands on a
+ * machine which does not hold the run has nothing local to stop. The job row is
+ * what crosses the gap: the cancel is written there, and every instance
+ * re-reads its *own* running runs on a slow timer
+ * ({@link startJobCancelPoller}), cancelling any whose row now reads
+ * `cancelled`. One indexed query per tick, bounded by that instance's
+ * concurrency.
+ *
+ * So a cross-instance cancel is not instant — worst case it takes a poll
+ * interval to land. That is the deliberate trade for having exactly one
+ * transport: the durable one. A `NOTIFY` bus would shave those seconds off but
+ * would need a direct (non-transaction-pooled) connection to be anything but a
+ * silent no-op, and it could never be the signal of record anyway — the row
+ * already is.
+ *
+ * A cancel on the machine that *does* hold the run never comes through here:
+ * `cancelJob` reaches the session's hooks directly and is immediate.
+ */
+import { createLogger } from "@nodetool-ai/config";
+import { Job } from "@nodetool-ai/models";
+import { jobRunRegistry } from "./job-run-registry.js";
+import { getInstanceId } from "./lib/instance-id.js";
+const log = createLogger("nodetool.websocket.job-control");
+/** How often an instance re-reads its own running runs. 0 disables the poll. */
+function pollIntervalMs() {
+    const raw = process.env["NODETOOL_JOB_CANCEL_POLL_MS"];
+    if (raw === undefined)
+        return 15_000;
+    const parsed = Number.parseInt(raw, 10);
+    return Number.isFinite(parsed) && parsed >= 0 ? parsed : 15_000;
+}
+/**
+ * Cancel a run this process does not hold, when another instance does.
+ *
+ * Scoped deliberately narrowly. Plenty of runs have no registry session
+ * anywhere — an HTTP `POST /run`, a trigger firing, an MCP call — and their
+ * rows carry no `runner_instance`. Flipping one of those to `cancelled` would
+ * report a cancellation to the client while the run went on to complete, and
+ * would leave a completed run recorded as cancelled. So a row is only treated
+ * as remotely cancellable when it names an instance, and a different one than
+ * this process; anything else keeps the caller's "not found" answer.
+ *
+ * The write is conditional on the row still being active, so losing the race
+ * against the owner's terminal write leaves the owner's outcome standing — and
+ * its result is the verdict returned here. The owner's poller does the rest.
+ */
+export async function requestRemoteJobCancel(userId, jobId) {
+    const miss = { cancelled: false, workflowId: null };
+    const instanceId = getInstanceId();
+    if (!instanceId)
+        return miss;
+    try {
+        const job = await Job.find(userId, jobId);
+        if (!job || job.isComplete())
+            return miss;
+        if (!job.runner_instance || job.runner_instance === instanceId)
+            return miss;
+        if (!(await Job.markCancelledIfActive(jobId, userId)))
+            return miss;
+        log.info("Cancelled a run owned by another instance", {
+            jobId,
+            owner: job.runner_instance
+        });
+        return { cancelled: true, workflowId: job.workflow_id || null };
+    }
+    catch (err) {
+        log.warn("Remote cancel could not reach the job row", {
+            jobId,
+            error: err instanceof Error ? err.message : String(err)
+        });
+        return miss;
+    }
+}
+/**
+ * Re-read this instance's own running runs and cancel the ones whose row says
+ * they were cancelled elsewhere.
+ *
+ * Nothing running locally means no query at all, so an idle instance costs one
+ * timer wakeup. Errors are logged and the tick is skipped: a database blip must
+ * not stop the poller, and the next tick sees the same rows.
+ */
+export function startJobCancelPoller() {
+    const intervalMs = pollIntervalMs();
+    if (intervalMs === 0)
+        return () => { };
+    let stopped = false;
+    let inFlight = false;
+    const tick = async () => {
+        // Never overlap: a slow query would otherwise stack ticks behind it.
+        if (inFlight || stopped)
+            return;
+        inFlight = true;
+        try {
+            await pollCancelledJobsOnce();
+        }
+        catch (err) {
+            log.warn("Job cancel poll failed", {
+                error: err instanceof Error ? err.message : String(err)
+            });
+        }
+        finally {
+            inFlight = false;
+        }
+    };
+    const timer = setInterval(() => void tick(), intervalMs);
+    // The poller must not be why the process stays alive.
+    timer.unref?.();
+    return () => {
+        stopped = true;
+        clearInterval(timer);
+    };
+}
+/**
+ * One poll pass: cancel every local run whose row now reads cancelled. Called
+ * on the timer, and directly by tests — the interval is far too slow to await.
+ */
+export async function pollCancelledJobsOnce() {
+    const sessions = jobRunRegistry.runningSessions();
+    if (sessions.length === 0)
+        return;
+    const cancelled = new Set(await Job.cancelledAmong(sessions.map((s) => s.jobId)));
+    for (const session of sessions) {
+        if (!cancelled.has(session.jobId) || session.status !== "running")
+            continue;
+        log.info("Cancelling a local run its row reports cancelled", {
+            jobId: session.jobId
+        });
+        session.cancel();
+    }
+}
+//# sourceMappingURL=job-control.js.map

package/dist/job-control.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-control.js","sourceRoot":"","sources":["../src/job-control.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,MAAM,GAAG,GAAG,YAAY,CAAC,gCAAgC,CAAC,CAAC;AAE3D,gFAAgF;AAChF,SAAS,cAAc;IACrB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;IACvD,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACxC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AAClE,CAAC;AAOD;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAc,EACd,KAAa;IAEb,MAAM,IAAI,GAAuB,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACxE,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,IAAI,CAAC,UAAU;QAAE,OAAO,IAAI,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,EAAE;YAAE,OAAO,IAAI,CAAC;QAC1C,IAAI,CAAC,GAAG,CAAC,eAAe,IAAI,GAAG,CAAC,eAAe,KAAK,UAAU;YAAE,OAAO,IAAI,CAAC;QAC5E,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QACnE,GAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE;YACpD,KAAK;YACL,KAAK,EAAE,GAAG,CAAC,eAAe;SAC3B,CAAC,CAAC;QACH,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;IAClE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE;YACpD,KAAK;YACL,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB;IAClC,MAAM,UAAU,GAAG,cAAc,EAAE,CAAC;IACpC,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IAEtC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACrC,qEAAqE;QACrE,IAAI,QAAQ,IAAI,OAAO;YAAE,OAAO;QAChC,QAAQ,GAAG,IAAI,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,qBAAqB,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,wBAAwB,EAAE;gBACjC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,QAAQ,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC;IACzD,sDAAsD;IACtD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAEhB,OAAO,GAAG,EAAE;QACV,OAAO,GAAG,IAAI,CAAC;QACf,aAAa,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,MAAM,QAAQ,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;IAClD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAClC,MAAM,SAAS,GAAG,IAAI,GAAG,CACvB,MAAM,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CACvD,CAAC;IACF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YAAE,SAAS;QAC5E,GAAG,CAAC,IAAI,CAAC,kDAAkD,EAAE;YAC3D,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,EAAE,CAAC;IACnB,CAAC;AACH,CAAC"}

package/dist/job-queue.d.ts

@@ -0,0 +1,47 @@
+/**
+ * FIFO queue for run-job requests that arrive while a client already has the
+ * maximum number of concurrent runs in flight.
+ *
+ * The {@link WebSocketClientSession} is per-connection, so this queue is
+ * per-client: it caps how many workflow runs one client can execute at once
+ * (configurable via the `MAX_CONCURRENT_JOBS` setting) and holds the rest
+ * until a running job finishes. This prevents a single click on "Run
+ * Workflow" — or a burst of clicks across tabs — from firing dozens of jobs
+ * at a provider/API simultaneously.
+ *
+ * It is generic over the request type (only `job_id`/`workflow_id` are needed)
+ * so it stays decoupled from any particular `RunJobRequest` definition, and so
+ * it can be unit-tested in isolation.
+ */
+/** Minimal shape the queue needs from a queued run request. */
+export interface QueueableJob {
+    job_id?: string | null;
+    workflow_id?: string | null;
+    /** When true, the run may start even if its workflow already has a run in flight. */
+    concurrent?: boolean;
+}
+export interface QueuedPosition {
+    jobId: string;
+    workflowId: string | null;
+    /** 1-based position in the pending queue (1 = starts next). */
+    position: number;
+    concurrent: boolean;
+}
+export declare class JobConcurrencyQueue<T extends QueueableJob = QueueableJob> {
+    private pending;
+    /** Number of runs waiting to start. */
+    get size(): number;
+    /** Append a request and return its 1-based position in the queue. */
+    enqueue(req: T): number;
+    /** Remove and return the next request to run (FIFO), or undefined if empty. */
+    dequeue(): T | undefined;
+    /**
+     * Remove a still-queued request by job id (e.g. the user cancelled it before
+     * it started). Returns the removed request, or undefined if it was not
+     * queued (already running or unknown).
+     */
+    remove(jobId: string): T | undefined;
+    /** Current queued jobs with their (recomputed) 1-based positions. */
+    positions(): QueuedPosition[];
+}
+//# sourceMappingURL=job-queue.d.ts.map

package/dist/job-queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-queue.d.ts","sourceRoot":"","sources":["../src/job-queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,+DAA+D;AAC/D,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,qFAAqF;IACrF,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,qBAAa,mBAAmB,CAAC,CAAC,SAAS,YAAY,GAAG,YAAY;IACpE,OAAO,CAAC,OAAO,CAAW;IAE1B,uCAAuC;IACvC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,qEAAqE;IACrE,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,CAGtB;IAED,+EAA+E;IAC/E,OAAO,IAAI,CAAC,GAAG,SAAS,CAEvB;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAInC;IAED,qEAAqE;IACrE,SAAS,IAAI,cAAc,EAAE,CAO5B;CACF"}

package/dist/job-queue.js

@@ -0,0 +1,52 @@
+/**
+ * FIFO queue for run-job requests that arrive while a client already has the
+ * maximum number of concurrent runs in flight.
+ *
+ * The {@link WebSocketClientSession} is per-connection, so this queue is
+ * per-client: it caps how many workflow runs one client can execute at once
+ * (configurable via the `MAX_CONCURRENT_JOBS` setting) and holds the rest
+ * until a running job finishes. This prevents a single click on "Run
+ * Workflow" — or a burst of clicks across tabs — from firing dozens of jobs
+ * at a provider/API simultaneously.
+ *
+ * It is generic over the request type (only `job_id`/`workflow_id` are needed)
+ * so it stays decoupled from any particular `RunJobRequest` definition, and so
+ * it can be unit-tested in isolation.
+ */
+export class JobConcurrencyQueue {
+    pending = [];
+    /** Number of runs waiting to start. */
+    get size() {
+        return this.pending.length;
+    }
+    /** Append a request and return its 1-based position in the queue. */
+    enqueue(req) {
+        this.pending.push(req);
+        return this.pending.length;
+    }
+    /** Remove and return the next request to run (FIFO), or undefined if empty. */
+    dequeue() {
+        return this.pending.shift();
+    }
+    /**
+     * Remove a still-queued request by job id (e.g. the user cancelled it before
+     * it started). Returns the removed request, or undefined if it was not
+     * queued (already running or unknown).
+     */
+    remove(jobId) {
+        const index = this.pending.findIndex((r) => r.job_id === jobId);
+        if (index === -1)
+            return undefined;
+        return this.pending.splice(index, 1)[0];
+    }
+    /** Current queued jobs with their (recomputed) 1-based positions. */
+    positions() {
+        return this.pending.map((req, index) => ({
+            jobId: req.job_id ?? "",
+            workflowId: req.workflow_id ?? null,
+            position: index + 1,
+            concurrent: req.concurrent ?? false
+        }));
+    }
+}
+//# sourceMappingURL=job-queue.js.map

package/dist/job-queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-queue.js","sourceRoot":"","sources":["../src/job-queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAkBH,MAAM,OAAO,mBAAmB;IACtB,OAAO,GAAQ,EAAE,CAAC;IAE1B,uCAAuC;IACvC,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED,qEAAqE;IACrE,OAAO,CAAC,GAAM;QACZ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED,+EAA+E;IAC/E,OAAO;QACL,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IAC9B,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAa;QAClB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC;QAChE,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,OAAO,SAAS,CAAC;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,qEAAqE;IACrE,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACvC,KAAK,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;YACvB,UAAU,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI;YACnC,QAAQ,EAAE,KAAK,GAAG,CAAC;YACnB,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,KAAK;SACpC,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}

package/dist/job-run-registry.d.ts

@@ -0,0 +1,176 @@
+/**
+ * Detachable workflow runs.
+ *
+ * The WebSocket runner is per-connection, so before this module existed a
+ * dropped socket killed every in-flight run (`disconnect()` →
+ * `job.session.cancel()` for each entry of `activeJobs`): a laptop lid-close
+ * threw away minutes of paid GPU work, and `reconnect_job` from a fresh
+ * connection — a fresh `UnifiedWebSocketRunner`, with an empty `activeJobs` —
+ * could only read the persisted row, reporting even a completed run as
+ * `job_update failed` with "replay is unavailable". A {@link JobRunSession}
+ * decouples the run from the socket, exactly as {@link ChatTurnSession} does
+ * for a chat turn: every frame the run emits is stamped with a monotonically
+ * increasing `job_seq` and appended to a bounded buffer, and delivery goes to
+ * whichever connection is currently attached — or nowhere, while the client
+ * is away. On reconnect the client sends
+ * `{command: "reconnect_job", data: {job_id, last_seq}}` and gets the missed
+ * tail replayed, followed by live frames if the run is still going.
+ *
+ * Sessions are keyed by user+job in a process-wide registry. Job ids are
+ * unique per run, so — unlike a chat thread — a new run never supersedes an
+ * existing session; {@link JobRunRegistry.open} replacing an entry means the
+ * same job id was started twice, which is a caller bug, not a turn taking
+ * over.
+ *
+ * Two timers bound a session's life:
+ *   - detach grace: a running job nobody is attached to is cancelled after
+ *     `NODETOOL_JOB_DETACH_GRACE_MS` (default 10 min) so an abandoned client
+ *     cannot leave a workflow burning provider spend forever;
+ *   - retention: a finished session is kept for
+ *     `NODETOOL_JOB_REPLAY_RETENTION_MS` (default 5 min) so a client that
+ *     reconnects just after the run ended still gets the tail, then dropped.
+ *
+ * Terminal state is persisted to the `jobs` table independently of this
+ * buffer, so an expired or truncated replay degrades to `reconnect_job`'s
+ * persisted-row fallback — the run's real status, just without its events.
+ */
+/**
+ * A connection that can deliver frames to its client.
+ *
+ * One at a time: {@link JobRunSession.attach} replaces the target rather than
+ * fanning out, so a second tab resubscribing to a run takes the stream from
+ * the first. Same as a chat turn, and the same trade — a client that wants
+ * two views of one run opens them against one socket.
+ */
+interface JobRunDeliveryTarget {
+    deliver(message: Record<string, unknown>): Promise<void>;
+}
+/**
+ * The executing connection's per-run hooks. Kept on the session so a
+ * different connection (post-reconnect) can route a client's `cancel_job` /
+ * `stop` / `stream_input` / `update_node_properties` back to the runner that
+ * actually owns the `ExecutionSession`.
+ */
+export interface JobRunExecutionHooks {
+    cancel(): void;
+    pushInput(input: string, value: unknown, handle?: string): Promise<void>;
+    finishInputStream(input: string, handle?: string): void;
+    updateNodeProperties(nodeId: string, properties: Record<string, unknown>): boolean;
+}
+interface JobRunAttachResult {
+    /** Frames with `job_seq` greater than the requested `last_seq`. */
+    replay: Array<Record<string, unknown>>;
+    /** True when `last_seq` predates what the bounded buffer still holds. */
+    incomplete: boolean;
+}
+export declare class JobRunSession {
+    readonly hooks: JobRunExecutionHooks;
+    private readonly onDrop;
+    /** Told when the run settles, so the registry can answer `drained`. */
+    private readonly onFinish;
+    readonly userId: string;
+    readonly jobId: string;
+    readonly workflowId: string | null;
+    status: "running" | "finished";
+    /** The run's outcome, once known. Reported in the `job_resumed` header. */
+    terminalStatus: string | null;
+    private seq;
+    private buffer;
+    /** Highest seq evicted from the bounded buffer (0 = nothing evicted). */
+    private evictedThroughSeq;
+    private target;
+    /**
+     * Serializes delivery: replayed frames enqueue before any live frame that
+     * arrives after attach, so the client always sees seq order. Unbounded —
+     * a slow socket grows the chain rather than applying backpressure to the
+     * run; the buffer above is what bounds memory. Mirrored from the chat
+     * turn's chain.
+     */
+    private deliveryChain;
+    private detachTimer;
+    private retentionTimer;
+    constructor(userId: string, jobId: string, workflowId: string | null, hooks: JobRunExecutionHooks, onDrop: (session: JobRunSession) => void, 
+    /** Told when the run settles, so the registry can answer `drained`. */
+    onFinish?: () => void);
+    get lastSeq(): number;
+    /**
+     * Stamp, buffer, and (when a connection is attached) deliver one frame.
+     * Returns the stamped copy.
+     */
+    emit(message: Record<string, unknown>): {
+        job_seq: number;
+    };
+    /**
+     * Attach a connection and hand back the frames it missed. Live frames
+     * emitted after this call are delivered to the new target, strictly after
+     * the returned replay is delivered (both ride {@link deliveryChain} when
+     * sent via {@link deliverReplay}).
+     */
+    attach(target: JobRunDeliveryTarget, lastSeq: number): JobRunAttachResult;
+    /** Deliver frames on the session's ordered delivery chain. */
+    deliverReplay(target: JobRunDeliveryTarget, frames: Array<Record<string, unknown>>): Promise<void>;
+    /**
+     * The attached connection went away. A running job keeps executing and
+     * buffering; if nobody reattaches within the grace window the run is
+     * cancelled so it cannot burn spend unattended forever.
+     */
+    detach(target?: JobRunDeliveryTarget): void;
+    /** Cancel the run (client stop, or detach grace elapsed). */
+    cancel(): void;
+    /**
+     * The run reached a terminal state. The session sticks around (still
+     * replayable) for the retention window, then drops out of the registry.
+     */
+    finish(terminalStatus?: string): void;
+    /** Release timers when the registry drops the session. */
+    dispose(): void;
+    private clearDetachTimer;
+    private enqueueDelivery;
+}
+export declare class JobRunRegistry {
+    private sessions;
+    /** Resolvers waiting for {@link runningCount} to reach zero. */
+    private drainWaiters;
+    private key;
+    /**
+     * Open a session for a starting run. Job ids are unique, so an entry that
+     * already exists can only mean the same id was started twice — cancel the
+     * stale run before dropping it. Dropping alone would leave it executing
+     * with nothing that can reach it: no grace timer, no cancel path, no
+     * replay.
+     */
+    open(userId: string, jobId: string, workflowId: string | null, hooks: JobRunExecutionHooks): JobRunSession;
+    get(userId: string, jobId: string): JobRunSession | null;
+    /** Every run this process is still executing. The cancel poller's input. */
+    runningSessions(): JobRunSession[];
+    /**
+     * Runs still executing for a user, across every connection. The
+     * concurrency caps count runs, not sockets: a detached run still occupies
+     * a slot, or a client could reconnect its way past the cap simply by
+     * abandoning sockets.
+     */
+    countRunning(userId: string): number;
+    /** Same, narrowed to one workflow — the per-workflow cap's denominator. */
+    countRunningForWorkflow(userId: string, workflowId: string): number;
+    /** Runs this process is still executing. What a drain waits on. */
+    runningCount(): number;
+    /**
+     * Cancel every run still executing, through the same hooks a client's
+     * `cancel_job` reaches. Returns how many were cancelled.
+     */
+    cancelAll(): number;
+    /** Cancel this process's runs that belong to a project being deleted. */
+    cancelJobs(userId: string, jobIds: ReadonlySet<string>): number;
+    /**
+     * Resolve true once no run is executing, false when `timeoutMs` elapses
+     * first. A run is executing until its terminal `job_update` has been
+     * persisted, which is what `finish` marks.
+     */
+    drained(timeoutMs: number): Promise<boolean>;
+    private notifyDrainWaiters;
+    drop(session: JobRunSession): void;
+}
+/** Process-wide registry: sessions survive their originating connection. */
+export declare const jobRunRegistry: JobRunRegistry;
+export {};
+//# sourceMappingURL=job-run-registry.d.ts.map

package/dist/job-run-registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-run-registry.d.ts","sourceRoot":"","sources":["../src/job-run-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAoBH;;;;;;;GAOG;AACH,UAAU,oBAAoB;IAC5B,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACnC,MAAM,IAAI,IAAI,CAAC;IACf,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,oBAAoB,CAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAClC,OAAO,CAAC;CACZ;AAED,UAAU,kBAAkB;IAC1B,mEAAmE;IACnE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACvC,yEAAyE;IACzE,UAAU,EAAE,OAAO,CAAC;CACrB;AAOD,qBAAa,aAAa;IA4BtB,QAAQ,CAAC,KAAK,EAAE,oBAAoB;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,uEAAuE;IACvE,OAAO,CAAC,QAAQ,CAAC,QAAQ;IA9B3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,MAAM,EAAE,SAAS,GAAG,UAAU,CAAa;IAC3C,2EAA2E;IAC3E,cAAc,EAAE,MAAM,GAAG,IAAI,CAAQ;IAErC,OAAO,CAAC,GAAG,CAAK;IAChB,OAAO,CAAC,MAAM,CAAuB;IACrC,yEAAyE;IACzE,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,MAAM,CAAqC;IACnD;;;;;;OAMG;IACH,OAAO,CAAC,aAAa,CAAoC;IACzD,OAAO,CAAC,WAAW,CAA+B;IAClD,OAAO,CAAC,cAAc,CAA+B;IAErD,YACE,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,GAAG,IAAI,EAChB,KAAK,EAAE,oBAAoB,EACnB,MAAM,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI;IACzD,uEAAuE;IACtD,QAAQ,GAAE,MAAM,IAAe,EAKjD;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED;;;OAGG;IACH,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;MAcpC;IAED;;;;;OAKG;IACH,MAAM,CAAC,MAAM,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,GAAG,kBAAkB,CAOxE;IAED,8DAA8D;IAC9D,aAAa,CACX,MAAM,EAAE,oBAAoB,EAC5B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GACrC,OAAO,CAAC,IAAI,CAAC,CAKf;IAED;;;;OAIG;IACH,MAAM,CAAC,MAAM,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAU1C;IAED,6DAA6D;IAC7D,MAAM,IAAI,IAAI,CAEb;IAED;;;OAGG;IACH,MAAM,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAQpC;IAED,0DAA0D;IAC1D,OAAO,IAAI,IAAI,CAMd;IAED,OAAO,CAAC,gBAAgB;IAOxB,OAAO,CAAC,eAAe;CAQxB;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAoC;IACpD,gEAAgE;IAChE,OAAO,CAAC,YAAY,CAAyB;IAE7C,OAAO,CAAC,GAAG;IAIX;;;;;;OAMG;IACH,IAAI,CACF,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,GAAG,IAAI,EACzB,KAAK,EAAE,oBAAoB,GAC1B,aAAa,CAkBf;IAED,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAEvD;IAED,4EAA4E;IAC5E,eAAe,IAAI,aAAa,EAAE,CAEjC;IAED;;;;;OAKG;IACH,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMnC;IAED,2EAA2E;IAC3E,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAYlE;IAED,mEAAmE;IACnE,YAAY,IAAI,MAAM,CAErB;IAED;;;OAGG;IACH,SAAS,IAAI,MAAM,CAIlB;IAED,yEAAyE;IACzE,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,MAAM,CAQ9D;IAED;;;;OAIG;IACH,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAgB3C;IAED,OAAO,CAAC,kBAAkB;IAI1B,IAAI,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,CAOjC;CACF;AAED,4EAA4E;AAC5E,eAAO,MAAM,cAAc,gBAAuB,CAAC"}