onadit 0.0.1 → 1.0.0

package/dist/src/jobs/worker-process.js

@@ -0,0 +1,227 @@
+/**
+ * Worker process infrastructure — Slice 19.
+ *
+ * Moves heavy job execution into forked child processes.
+ * The parent (main daemon) keeps queue ownership, state transitions,
+ * and lifecycle. The child performs handler work and returns
+ * structured success/failure data via IPC.
+ *
+ * Child crash → parent marks job failed (no hung jobs).
+ * SIGTERM → parent kills active child.
+ */
+import { spawn } from "node:child_process";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { EXCLUSIVE_JOBS } from "./types.js";
+import { refreshTables } from "../storage/tables.js";
+/**
+ * Resolve the absolute path to the worker-entry module from the main module's
+ * `import.meta.url`.
+ *
+ * MUST go through fileURLToPath, never `new URL(url).pathname`: a file:// URL's
+ * pathname stays percent-encoded (a space is `%20`), so under an install path
+ * with spaces — the DEFAULT daemon path `~/Library/Application Support/Rift/` —
+ * `.pathname` yields `…/Application%20Support/…`, the spawned worker dies with
+ * MODULE_NOT_FOUND, and every save/ingest/triage/recover job fails.
+ */
+export function resolveWorkerEntryPoint(mainModuleUrl) {
+    return path.resolve(path.dirname(fileURLToPath(mainModuleUrl)), "jobs", "worker-entry.js");
+}
+// --- WorkerPool ---
+export class WorkerPool {
+    entryPoint;
+    configPath;
+    activeChild = null;
+    shuttingDown = false;
+    constructor(entryPoint, configPath) {
+        this.entryPoint = entryPoint;
+        this.configPath = configPath;
+    }
+    /**
+     * Execute a job in a forked child process.
+     * Resolves on success, rejects on failure or crash.
+     */
+    execute(msg, onProgress) {
+        if (this.shuttingDown) {
+            return Promise.reject(new Error("Worker pool is shutting down"));
+        }
+        return new Promise((resolve, reject) => {
+            let settled = false;
+            let started = false;
+            const child = spawn(process.execPath, [this.entryPoint], {
+                cwd: process.cwd(),
+                env: process.env,
+                // Pipe (not ignore) stderr so a child that crashes WITHOUT sending an
+                // IPC result — uncaught throw at module load, OOM — leaves a
+                // diagnosable trail. The handler's own failures still come back via
+                // IPC; this is the backstop for the opaque "exited with code 1" case.
+                stdio: ["ignore", "ignore", "pipe", "ipc"],
+            });
+            this.activeChild = child;
+            // Keep only a bounded tail — the exit cause is at the end, and we never
+            // want an unbounded buffer for a runaway child.
+            let stderrTail = "";
+            child.stderr?.setEncoding("utf-8");
+            child.stderr?.on("data", (chunk) => {
+                stderrTail = `${stderrTail}${chunk}`.slice(-2000);
+            });
+            child.on("spawn", () => {
+                started = true;
+            });
+            const cleanup = () => {
+                this.activeChild = null;
+                child.removeAllListeners();
+            };
+            child.on("message", (result) => {
+                if (settled)
+                    return;
+                if (result.jobId !== msg.jobId)
+                    return;
+                if (result.type === "progress") {
+                    onProgress?.(result.progress);
+                    return;
+                }
+                settled = true;
+                cleanup();
+                if (result.success) {
+                    resolve(result);
+                }
+                else {
+                    reject(new Error(result.error ?? "Worker failed"));
+                }
+            });
+            // Settle on `close`, not `exit`: `close` is emitted only after the
+            // child's stdio streams have ended, so `stderrTail` is guaranteed
+            // complete when we build the diagnostic. (On `exit` the final stderr
+            // chunk may not have been delivered yet, truncating the crash cause.)
+            child.on("close", (code, signal) => {
+                if (settled)
+                    return;
+                settled = true;
+                cleanup();
+                const stderrDetail = stderrTail.trim()
+                    ? ` — stderr: ${stderrTail.trim()}`
+                    : "";
+                if (signal) {
+                    reject(new Error(`Worker killed by signal ${signal}${stderrDetail}`));
+                }
+                else if (code !== 0 && code !== null) {
+                    reject(new Error(`Worker exited with code ${code}${stderrDetail}`));
+                }
+                else {
+                    // code === 0 (or null) without prior IPC result.
+                    // Shouldn't happen — worker always sends result before exit.
+                    // Resolve to avoid hanging the queue forever.
+                    resolve(undefined);
+                }
+            });
+            child.on("error", (err) => {
+                if (settled)
+                    return;
+                settled = true;
+                cleanup();
+                reject(new Error(started ? err.message : `Worker process failed to start: ${err.message}`));
+            });
+            child.send(msg);
+        });
+    }
+    /**
+     * Gracefully shut down: kill active child and wait for it to exit.
+     * Returns a Promise that resolves once the child is confirmed dead
+     * (or immediately if no child is active).
+     */
+    shutdown() {
+        this.shuttingDown = true;
+        if (!this.activeChild)
+            return Promise.resolve();
+        const child = this.activeChild;
+        this.activeChild = null;
+        return new Promise((resolve) => {
+            child.kill("SIGTERM");
+            // Give child 3 seconds to exit before SIGKILL.
+            const killTimer = setTimeout(() => {
+                try {
+                    child.kill("SIGKILL");
+                }
+                catch {
+                    // Already exited.
+                }
+            }, 3_000);
+            child.on("exit", () => {
+                clearTimeout(killTimer);
+                resolve();
+            });
+        });
+    }
+    /** Whether a child process is currently running. */
+    get isActive() {
+        return this.activeChild !== null;
+    }
+}
+// --- createWorkerHandler ---
+/**
+ * Walk the `retry_of` chain to the root job.
+ * Duplicated from save handler — needed in parent for pre-resolution.
+ */
+function resolveChainRoot(job, queue) {
+    let current = job;
+    let depth = 0;
+    while (current.retry_of && depth < 50) {
+        const parent = queue.get(current.retry_of);
+        if (!parent)
+            break;
+        current = parent;
+        depth++;
+    }
+    return current;
+}
+/**
+ * Create a JobHandler that executes work in a child process.
+ *
+ * - Pre-resolves retry chain root (save/ingest need it, child has no queue)
+ * - Pauses watchers/scanner for exclusive jobs (compact/reindex/reconcile)
+ * - Resumes watchers after child completes
+ */
+export function createWorkerHandler(pool, jobType, configPath, watchers, scanner, queue) {
+    return async (job) => {
+        const isExclusive = EXCLUSIVE_JOBS.has(jobType);
+        // Pause live indexers for exclusive maintenance jobs.
+        if (isExclusive) {
+            for (const w of watchers)
+                w.pause();
+            if (scanner)
+                scanner.pause();
+        }
+        try {
+            // Pre-resolve chain root so child doesn't need queue access.
+            const rootJob = resolveChainRoot(job, queue);
+            const hasChain = rootJob.id !== job.id;
+            const msg = {
+                type: "job",
+                jobType,
+                jobId: job.id,
+                jobPayload: job.payload,
+                configPath,
+                ...(hasChain
+                    ? { rootJobId: rootJob.id, rootPayload: rootJob.payload }
+                    : {}),
+            };
+            const result = await pool.execute(msg, (progress) => { job.progress = progress; });
+            if (result?.import_summary)
+                job.import_summary = result.import_summary;
+            // Worker wrote to LanceDB via its own connection — refresh the
+            // parent's cached table handles so searches see the new data.
+            await refreshTables();
+        }
+        finally {
+            // Resume live indexers (drain buffered events).
+            if (isExclusive) {
+                for (const w of watchers)
+                    await w.resume();
+                if (scanner)
+                    scanner.resume();
+            }
+        }
+    };
+}
+//# sourceMappingURL=worker-process.js.map

package/dist/src/jobs/worker-process.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-process.js","sourceRoot":"","sources":["../../../src/jobs/worker-process.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,KAAK,EAAqB,MAAM,oBAAoB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAI5C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD;;;;;;;;;GASG;AACH,MAAM,UAAU,uBAAuB,CAAC,aAAqB;IAC3D,OAAO,IAAI,CAAC,OAAO,CACjB,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,EAC1C,MAAM,EACN,iBAAiB,CAClB,CAAC;AACJ,CAAC;AA8BD,qBAAqB;AAErB,MAAM,OAAO,UAAU;IAKF;IACA;IALX,WAAW,GAAwB,IAAI,CAAC;IACxC,YAAY,GAAG,KAAK,CAAC;IAE7B,YACmB,UAAkB,EAClB,UAAkB;QADlB,eAAU,GAAV,UAAU,CAAQ;QAClB,eAAU,GAAV,UAAU,CAAQ;IAClC,CAAC;IAEJ;;;OAGG;IACH,OAAO,CAAC,GAAkB,EAAE,UAA6D;QACvF,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,IAAI,OAAO,CAA2B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC/D,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,OAAO,GAAG,KAAK,CAAC;YAEpB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;gBACvD,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;gBAClB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,sEAAsE;gBACtE,6DAA6D;gBAC7D,oEAAoE;gBACpE,sEAAsE;gBACtE,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC;aAC3C,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YAEzB,wEAAwE;YACxE,gDAAgD;YAChD,IAAI,UAAU,GAAG,EAAE,CAAC;YACpB,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;YACnC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBACzC,UAAU,GAAG,GAAG,UAAU,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACrB,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,GAAG,EAAE;gBACnB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,KAAK,CAAC,kBAAkB,EAAE,CAAC;YAC7B,CAAC,CAAC;YAEF,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,MAAqC,EAAE,EAAE;gBAC5D,IAAI,OAAO;oBAAE,OAAO;gBACpB,IAAI,MAAM,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK;oBAAE,OAAO;gBACvC,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;oBAC/B,UAAU,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAC9B,OAAO;gBACT,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,OAAO,CAAC,MAAM,CAAC,CAAC;gBAClB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC,CAAC,CAAC;gBACrD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,mEAAmE;YACnE,kEAAkE;YAClE,qEAAqE;YACrE,sEAAsE;YACtE,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;gBACjC,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,MAAM,YAAY,GAAG,UAAU,CAAC,IAAI,EAAE;oBACpC,CAAC,CAAC,cAAc,UAAU,CAAC,IAAI,EAAE,EAAE;oBACnC,CAAC,CAAC,EAAE,CAAC;gBACP,IAAI,MAAM,EAAE,CAAC;oBACX,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,MAAM,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC;gBACxE,CAAC;qBAAM,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBACvC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,IAAI,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC;gBACtE,CAAC;qBAAM,CAAC;oBACN,iDAAiD;oBACjD,6DAA6D;oBAC7D,8CAA8C;oBAC9C,OAAO,CAAC,SAAS,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAC/B,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,MAAM,CACJ,IAAI,KAAK,CACP,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,mCAAmC,GAAG,CAAC,OAAO,EAAE,CACzE,CACF,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAEhD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,+CAA+C;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;gBAChC,IAAI,CAAC;oBACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxB,CAAC;gBAAC,MAAM,CAAC;oBACP,kBAAkB;gBACpB,CAAC;YACH,CAAC,EAAE,KAAK,CAAC,CAAC;YACV,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACpB,YAAY,CAAC,SAAS,CAAC,CAAC;gBACxB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,oDAAoD;IACpD,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC;IACnC,CAAC;CACF;AAED,8BAA8B;AAE9B;;;GAGG;AACH,SAAS,gBAAgB,CAAC,GAAQ,EAAE,KAAe;IACjD,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,OAAO,CAAC,QAAQ,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,MAAM;QACnB,OAAO,GAAG,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CACjC,IAAgB,EAChB,OAAgB,EAChB,UAAkB,EAClB,QAAmB,EACnB,OAAuB,EACvB,KAAe;IAEf,OAAO,KAAK,EAAE,GAAQ,EAAiB,EAAE;QACvC,MAAM,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEhD,sDAAsD;QACtD,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,MAAM,CAAC,IAAI,QAAQ;gBAAE,CAAC,CAAC,KAAK,EAAE,CAAC;YACpC,IAAI,OAAO;gBAAE,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC;YACH,6DAA6D;YAC7D,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;YAEvC,MAAM,GAAG,GAAkB;gBACzB,IAAI,EAAE,KAAK;gBACX,OAAO;gBACP,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,UAAU,EAAE,GAAG,CAAC,OAAO;gBACvB,UAAU;gBACV,GAAG,CAAC,QAAQ;oBACV,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE;oBACzD,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACnF,IAAI,MAAM,EAAE,cAAc;gBAAE,GAAG,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;YAEvE,+DAA+D;YAC/D,8DAA8D;YAC9D,MAAM,aAAa,EAAE,CAAC;QACxB,CAAC;gBAAS,CAAC;YACT,gDAAgD;YAChD,IAAI,WAAW,EAAE,CAAC;gBAChB,KAAK,MAAM,CAAC,IAAI,QAAQ;oBAAE,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC;gBAC3C,IAAI,OAAO;oBAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/jobs/worker.d.ts

@@ -0,0 +1,11 @@
+import type { JobHandler, JobType } from "./types.js";
+/**
+ * Worker registry. All execution is in-process for now.
+ * Child-process forking deferred to Slice 19.
+ */
+export declare class WorkerRegistry {
+    private handlers;
+    register(type: JobType, handler: JobHandler): void;
+    get(type: JobType): JobHandler | undefined;
+}
+//# sourceMappingURL=worker.d.ts.map

package/dist/src/jobs/worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../../src/jobs/worker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAEtD;;;GAGG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAkC;IAElD,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,GAAG,IAAI;IAIlD,GAAG,CAAC,IAAI,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS;CAG3C"}

package/dist/src/jobs/worker.js

@@ -0,0 +1,14 @@
+/**
+ * Worker registry. All execution is in-process for now.
+ * Child-process forking deferred to Slice 19.
+ */
+export class WorkerRegistry {
+    handlers = new Map();
+    register(type, handler) {
+        this.handlers.set(type, handler);
+    }
+    get(type) {
+        return this.handlers.get(type);
+    }
+}
+//# sourceMappingURL=worker.js.map

package/dist/src/jobs/worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/jobs/worker.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,QAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;IAElD,QAAQ,CAAC,IAAa,EAAE,OAAmB;QACzC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACnC,CAAC;IAED,GAAG,CAAC,IAAa;QACf,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;CACF"}

package/dist/src/main.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=main.d.ts.map

package/dist/src/main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/main.ts"],"names":[],"mappings":""}