@getrift/rift 0.0.0 → 0.1.0-beta.0

package/dist/src/ingestion/watcher.js

@@ -0,0 +1,182 @@
+/**
+ * File watcher for mode: "watch" source directories.
+ *
+ * Uses chokidar with awaitWriteFinish. Debounces events per path.
+ * Exposes pause()/resume(): while paused, debounced events are
+ * buffered (not dropped). resume() drains the buffer.
+ *
+ * Required by Live-Indexer / Repair-Job Coordination Rule (PRD Section 2).
+ */
+import { watch as chokidarWatch } from "chokidar";
+import path from "node:path";
+/**
+ * Directory basenames that are ignored anywhere in the tree because
+ * they are engine/build/tooling junk, never user content. Matched as
+ * path segments (via `DEFAULT_IGNORED_PATTERNS` below), not just at
+ * the source root, so nested copies are also skipped — e.g.
+ * `src/foo/__pycache__` inside a deeply nested Python subpackage,
+ * or a `.venv` created inside a sub-project.
+ */
+export const GLOBAL_JUNK_DIR_NAMES = [
+    "dist",
+    "logs",
+    ".codex-worktrees",
+    ".venv",
+    "__pycache__",
+    ".next",
+    ".obsidian",
+];
+function matchDirSegment(name) {
+    const escaped = name.replace(/[.+?^${}()|[\]\\]/g, "\\$&");
+    return new RegExp(`(^|[/\\\\])${escaped}([/\\\\]|$)`);
+}
+const DEFAULT_IGNORED_PATTERNS = [
+    /(^|[/\\])\.git([/\\]|$)/,
+    /(^|[/\\])node_modules([/\\]|$)/,
+    /(^|[/\\])\.DS_Store$/,
+    ...GLOBAL_JUNK_DIR_NAMES.map(matchDirSegment),
+];
+export function isWatcherPathIgnored(candidatePath, ignoredPaths = []) {
+    const normalized = path.resolve(candidatePath);
+    if (DEFAULT_IGNORED_PATTERNS.some((pattern) => pattern.test(normalized))) {
+        return true;
+    }
+    return ignoredPaths.some((ignoredPath) => {
+        const normalizedIgnoredPath = path.resolve(ignoredPath);
+        return normalized === normalizedIgnoredPath ||
+            normalized.startsWith(normalizedIgnoredPath + path.sep);
+    });
+}
+export class Watcher {
+    fsWatcher = null;
+    paused = false;
+    draining = false;
+    buffer = [];
+    debounceTimers = new Map();
+    debounceMs;
+    handler;
+    paths;
+    awfConfig;
+    ignoredPaths;
+    constructor(paths, handler, opts = {}) {
+        this.paths = paths;
+        this.handler = handler;
+        this.debounceMs = opts.debounceMs ?? 300;
+        this.awfConfig = opts.awaitWriteFinish ?? {
+            stabilityThreshold: 200,
+            pollInterval: 100,
+        };
+        this.ignoredPaths = opts.ignoredPaths ?? [];
+    }
+    /**
+     * Start watching. Returns a promise that resolves once chokidar
+     * has finished the initial scan (ready event), or rejects if
+     * chokidar emits an error before becoming ready.
+     */
+    start() {
+        return new Promise((resolve, reject) => {
+            this.fsWatcher = chokidarWatch([...this.paths], {
+                ignoreInitial: true,
+                awaitWriteFinish: this.awfConfig,
+                ignored: (candidatePath) => isWatcherPathIgnored(candidatePath, this.ignoredPaths),
+            });
+            let ready = false;
+            this.fsWatcher.on("add", (p) => this.onRawEvent({ type: "add", path: p }));
+            this.fsWatcher.on("change", (p) => this.onRawEvent({ type: "change", path: p }));
+            this.fsWatcher.on("unlink", (p) => this.onRawEvent({ type: "unlink", path: p }));
+            this.fsWatcher.on("error", (err) => {
+                const msg = err instanceof Error ? err.message : String(err);
+                if (!ready) {
+                    reject(err instanceof Error ? err : new Error(msg));
+                }
+                else {
+                    process.stderr.write(`Watcher error: ${msg}\n`);
+                }
+            });
+            this.fsWatcher.on("ready", () => {
+                ready = true;
+                resolve();
+            });
+        });
+    }
+    /** Stop watching and clear all pending debounce timers. */
+    async stop() {
+        for (const timer of this.debounceTimers.values()) {
+            clearTimeout(timer);
+        }
+        this.debounceTimers.clear();
+        this.buffer.length = 0;
+        if (this.fsWatcher) {
+            await this.fsWatcher.close();
+            this.fsWatcher = null;
+        }
+    }
+    /** Pause event dispatch — events are debounced then buffered. */
+    pause() {
+        this.paused = true;
+    }
+    /**
+     * Resume event dispatch. Drains all buffered events sequentially
+     * while still paused, so new events arriving during drain are
+     * buffered behind the backlog. Only unpauses after drain completes.
+     */
+    async resume() {
+        // Drain the backlog while still paused. Any debounce timer
+        // that fires during drain sees paused=true and appends to buffer,
+        // so the while-loop catches it before we go live.
+        this.draining = true;
+        try {
+            while (this.buffer.length > 0) {
+                const event = this.buffer.shift();
+                await this.dispatch(event);
+            }
+        }
+        finally {
+            // Always unpause, even if a buffered event throws.
+            // Without this, a single failure would wedge the watcher
+            // in paused+draining mode permanently.
+            this.draining = false;
+            this.paused = false;
+        }
+    }
+    /** Whether the watcher is currently paused. */
+    get isPaused() {
+        return this.paused;
+    }
+    /** Number of events currently buffered (while paused). */
+    get bufferedCount() {
+        return this.buffer.length;
+    }
+    /**
+     * Inject a synthetic event for testing. Bypasses chokidar but
+     * goes through the same debounce -> dispatch/buffer path.
+     */
+    _injectEvent(event) {
+        this.onRawEvent(event);
+    }
+    /**
+     * Raw event from chokidar (or _injectEvent). Debounces per path:
+     * only the last event for a given path within the debounce window fires.
+     */
+    onRawEvent(event) {
+        const existing = this.debounceTimers.get(event.path);
+        if (existing !== undefined)
+            clearTimeout(existing);
+        const timer = setTimeout(() => {
+            this.debounceTimers.delete(event.path);
+            if (this.paused || this.draining) {
+                this.buffer.push(event);
+            }
+            else {
+                this.dispatch(event).catch((err) => {
+                    process.stderr.write(`Watcher handler error for ${event.path}: ${err instanceof Error ? err.message : String(err)}\n`);
+                });
+            }
+        }, this.debounceMs);
+        this.debounceTimers.set(event.path, timer);
+    }
+    async dispatch(event) {
+        await this.handler(event);
+    }
+}
+//# sourceMappingURL=watcher.js.map

package/dist/src/ingestion/watcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"watcher.js","sourceRoot":"","sources":["../../../src/ingestion/watcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,KAAK,IAAI,aAAa,EAAkB,MAAM,UAAU,CAAC;AAClE,OAAO,IAAI,MAAM,WAAW,CAAC;AAW7B;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,MAAM;IACN,MAAM;IACN,kBAAkB;IAClB,OAAO;IACP,aAAa;IACb,OAAO;IACP,WAAW;CACH,CAAC;AAEX,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;IAC3D,OAAO,IAAI,MAAM,CAAC,cAAc,OAAO,aAAa,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,wBAAwB,GAAsB;IAClD,yBAAyB;IACzB,gCAAgC;IAChC,sBAAsB;IACtB,GAAG,qBAAqB,CAAC,GAAG,CAAC,eAAe,CAAC;CAC9C,CAAC;AAEF,MAAM,UAAU,oBAAoB,CAClC,aAAqB,EACrB,eAAkC,EAAE;IAEpC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C,IAAI,wBAAwB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QACzE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;QACvC,MAAM,qBAAqB,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACxD,OAAO,UAAU,KAAK,qBAAqB;YACzC,UAAU,CAAC,UAAU,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,OAAO;IACV,SAAS,GAAqB,IAAI,CAAC;IACnC,MAAM,GAAG,KAAK,CAAC;IACf,QAAQ,GAAG,KAAK,CAAC;IACR,MAAM,GAAgB,EAAE,CAAC;IACzB,cAAc,GAAG,IAAI,GAAG,EAAyC,CAAC;IAClE,UAAU,CAAS;IACnB,OAAO,CAAmB;IAC1B,KAAK,CAAoB;IACzB,SAAS,CAAiE;IAC1E,YAAY,CAAoB;IAEjD,YACE,KAAwB,EACxB,OAAyB,EACzB,OAAuB,EAAE;QAEzB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,GAAG,CAAC;QACzC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,gBAAgB,IAAI;YACxC,kBAAkB,EAAE,GAAG;YACvB,YAAY,EAAE,GAAG;SAClB,CAAC;QACF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;IAC9C,CAAC;IAED;;;;OAIG;IACH,KAAK;QACH,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,CAAC,SAAS,GAAG,aAAa,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE;gBAC9C,aAAa,EAAE,IAAI;gBACnB,gBAAgB,EAAE,IAAI,CAAC,SAAS;gBAChC,OAAO,EAAE,CAAC,aAAqB,EAAE,EAAE,CACjC,oBAAoB,CAAC,aAAa,EAAE,IAAI,CAAC,YAAY,CAAC;aACzD,CAAC,CAAC;YAEH,IAAI,KAAK,GAAG,KAAK,CAAC;YAElB,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACnF,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACzF,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAEzF,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAY,EAAE,EAAE;gBAC1C,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;gBACtD,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC;gBAClD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBAC9B,KAAK,GAAG,IAAI,CAAC;gBACb,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,2DAA2D;IAC3D,KAAK,CAAC,IAAI;QACR,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC;YACjD,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACvB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,KAAK;QACH,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM;QACV,2DAA2D;QAC3D,kEAAkE;QAClE,kDAAkD;QAClD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAG,CAAC;gBACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,mDAAmD;YACnD,yDAAyD;YACzD,uCAAuC;YACvC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;YACtB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,0DAA0D;IAC1D,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,KAAgB;QAC3B,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED;;;OAGG;IACK,UAAU,CAAC,KAAgB;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,QAAQ,KAAK,SAAS;YAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;QAEnD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACjC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;oBAC1C,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6BAA6B,KAAK,CAAC,IAAI,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CACjG,CAAC;gBACJ,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEpB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,KAAgB;QACrC,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;CACF"}

package/dist/src/jobs/codex-override-handler.d.ts

@@ -0,0 +1,3 @@
+import type { JobHandler } from "./types.js";
+export declare function createCodexOverrideAwareHandler(workerHandler: JobHandler, inlineOverrideHandler: JobHandler): JobHandler;
+//# sourceMappingURL=codex-override-handler.d.ts.map

package/dist/src/jobs/codex-override-handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex-override-handler.d.ts","sourceRoot":"","sources":["../../../src/jobs/codex-override-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAO,UAAU,EAAE,MAAM,YAAY,CAAC;AAelD,wBAAgB,+BAA+B,CAC7C,aAAa,EAAE,UAAU,EACzB,qBAAqB,EAAE,UAAU,GAChC,UAAU,CASZ"}

package/dist/src/jobs/codex-override-handler.js

@@ -0,0 +1,16 @@
+import { CODEX_CLI_OVERRIDE_PROVIDER } from "../providers/operator-overrides.js";
+function isCodexCliOverrideJob(job) {
+    const payload = (job.payload ?? {});
+    return (payload.provider_override === CODEX_CLI_OVERRIDE_PROVIDER ||
+        payload.extraction_provider_override === CODEX_CLI_OVERRIDE_PROVIDER);
+}
+export function createCodexOverrideAwareHandler(workerHandler, inlineOverrideHandler) {
+    return async (job) => {
+        if (isCodexCliOverrideJob(job)) {
+            await inlineOverrideHandler(job);
+            return;
+        }
+        await workerHandler(job);
+    };
+}
+//# sourceMappingURL=codex-override-handler.js.map

package/dist/src/jobs/codex-override-handler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex-override-handler.js","sourceRoot":"","sources":["../../../src/jobs/codex-override-handler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAE,MAAM,oCAAoC,CAAC;AAEjF,SAAS,qBAAqB,CAAC,GAAQ;IACrC,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAGjC,CAAC;IAEF,OAAO,CACL,OAAO,CAAC,iBAAiB,KAAK,2BAA2B;QACzD,OAAO,CAAC,4BAA4B,KAAK,2BAA2B,CACrE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,aAAyB,EACzB,qBAAiC;IAEjC,OAAO,KAAK,EAAE,GAAQ,EAAiB,EAAE;QACvC,IAAI,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACjC,OAAO;QACT,CAAC;QAED,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/jobs/handlers/compact.d.ts

@@ -0,0 +1,30 @@
+import type { JobHandler } from "../types.js";
+import type { EmbeddingProvider, DigestSummarizer } from "../../providers/types.js";
+import type { Watcher } from "../../ingestion/watcher.js";
+import type { Scanner } from "../../ingestion/scanner.js";
+export interface CompactDeps {
+    embeddingProvider: EmbeddingProvider;
+    digestSummarizer: DigestSummarizer;
+    dataDir: string;
+    ageThresholdDays: number;
+    watchers: Watcher[];
+    scanner: Scanner | null;
+}
+export interface CompactReport {
+    moved: number;
+    digest_id: string | null;
+    dry_run: boolean;
+    rollback: boolean;
+}
+export interface RollbackManifest {
+    id: string;
+    created_at: string;
+    conversation_ids: string[];
+    digest_id: string;
+    digest_filename: string;
+    eligible: boolean;
+}
+/** Check if a rollback-eligible manifest exists on disk. */
+export declare function hasEligibleRollback(dataDir: string): boolean;
+export declare function createCompactHandler(deps: CompactDeps): JobHandler;
+//# sourceMappingURL=compact.d.ts.map

package/dist/src/jobs/handlers/compact.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compact.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/compact.ts"],"names":[],"mappings":"AAgCA,OAAO,KAAK,EAAO,UAAU,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAqB,MAAM,0BAA0B,CAAC;AACvG,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAM1D,MAAM,WAAW,WAAW;IAC1B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,OAAO,CAAC;CACnB;AASD,4DAA4D;AAC5D,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAE5D;AAMD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,WAAW,GAAG,UAAU,CAwBlE"}

package/dist/src/jobs/handlers/compact.js

@@ -0,0 +1,329 @@
+/**
+ * Compaction job handler — Slice 11.
+ *
+ * Hot->cold tier migration, weekly digest generation, and rollback.
+ *
+ * Compaction:
+ * 1. Select conversations older than age_threshold_days from conversations_hot
+ * 2. Write rollback manifest with eligible:false BEFORE any table mutation (crash safety)
+ * 3. Insert into conversations_cold, delete from conversations_hot
+ * 4. Generate weekly digest via DigestSummarizer, embed
+ * 5. Write raw digest artifact to data/raw/digests/ (before table insert — crash safety)
+ * 6. Insert digest row into digests table
+ * 7. Flip manifest to eligible:true only after all mutations succeed
+ *
+ * Rollback:
+ * 1. Read the eligible rollback manifest
+ * 2. Move entries from cold -> hot, remove digest row AND raw digest file
+ * 3. Clear rollback eligibility
+ *
+ * Dry run: report what would change without mutating.
+ *
+ * Implements Live-Indexer / Repair-Job Coordination Rule:
+ * Pauses watchers + scanner before mutations, drains buffer in post-pass, resumes.
+ *
+ * Crash safety: manifest is written before any table mutation. If compaction
+ * crashes mid-way, reconcile (Slice 10) handles recovery:
+ * - Cross-tier duplicates (copied to cold, not yet removed from hot) -> reconcile removes hot copy
+ * - Missing digest (not yet written) -> reconcile re-indexes from raw file if it exists
+ */
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { getTable } from "../../storage/tables.js";
+import { atomicWrite } from "../../storage/atomic.js";
+const COMPACTION_DIR = "compaction";
+const MANIFEST_SUFFIX = "_manifest.json";
+// ---------------------------------------------------------------------------
+// Public helpers (used by route for pre-check)
+// ---------------------------------------------------------------------------
+/** Check if a rollback-eligible manifest exists on disk. */
+export function hasEligibleRollback(dataDir) {
+    return findEligibleManifest(dataDir) !== null;
+}
+// ---------------------------------------------------------------------------
+// Handler factory
+// ---------------------------------------------------------------------------
+export function createCompactHandler(deps) {
+    return async (job) => {
+        const payload = job.payload ?? { rollback: false, dry_run: false };
+        const isDryRun = payload.dry_run;
+        const isRollback = payload.rollback;
+        // --- 1. Pause all live indexers ---
+        for (const w of deps.watchers)
+            w.pause();
+        if (deps.scanner)
+            deps.scanner.pause();
+        try {
+            if (isRollback) {
+                const report = await handleRollback(deps, isDryRun);
+                job.payload = report;
+            }
+            else {
+                const report = await handleCompaction(deps, isDryRun);
+                job.payload = report;
+            }
+        }
+        finally {
+            // --- Post-pass: resume watchers (drains buffered events) ---
+            for (const w of deps.watchers)
+                await w.resume();
+            if (deps.scanner)
+                deps.scanner.resume();
+        }
+    };
+}
+// ---------------------------------------------------------------------------
+// Compaction logic
+// ---------------------------------------------------------------------------
+async function handleCompaction(deps, dryRun) {
+    const hotTable = getTable("conversations_hot");
+    const coldTable = getTable("conversations_cold");
+    const digestTable = getTable("digests");
+    // Find conversations older than threshold
+    const cutoff = new Date();
+    cutoff.setDate(cutoff.getDate() - deps.ageThresholdDays);
+    const cutoffIso = cutoff.toISOString();
+    const allHot = (await hotTable.query().toArray());
+    const toMove = allHot.filter((row) => row.indexed_at < cutoffIso);
+    if (toMove.length === 0) {
+        return { moved: 0, digest_id: null, dry_run: dryRun, rollback: false };
+    }
+    if (dryRun) {
+        return {
+            moved: toMove.length,
+            digest_id: null,
+            dry_run: true,
+            rollback: false,
+        };
+    }
+    // Generate digest from conversations being moved
+    const entries = toMove.map(rowToEntry);
+    const digestContent = await deps.digestSummarizer.summarize(entries);
+    const digestEmbedding = await deps.embeddingProvider.embed(digestContent.summary);
+    const now = new Date();
+    const digestId = crypto.randomUUID();
+    const conversationIds = toMove.map((r) => r.id);
+    // Compute period bounds from the moved conversations
+    const timestamps = toMove.map((r) => new Date(r.indexed_at).getTime());
+    const periodStart = new Date(Math.min(...timestamps)).toISOString();
+    const periodEnd = new Date(Math.max(...timestamps)).toISOString();
+    const timestamp = now.toISOString().replace(/[:.]/g, "-");
+    // --- CRASH SAFETY: write manifest with eligible:false BEFORE any table mutation ---
+    // Eligible is flipped to true only after all mutations succeed. If we crash
+    // before that flip, the failed compaction is NOT rollback-eligible.
+    const manifest = {
+        id: timestamp,
+        created_at: now.toISOString(),
+        conversation_ids: conversationIds,
+        digest_id: digestId,
+        digest_filename: "", // Set after raw digest is written
+        eligible: false,
+    };
+    await writeManifest(deps.dataDir, manifest);
+    // Clear previous rollback eligibility
+    clearPriorEligibility(deps.dataDir, timestamp);
+    // --- Table mutations ---
+    // 1. Insert into cold (strip LanceDB internal fields)
+    await coldTable.add(toMove.map(cleanConversationRow));
+    // 2. Remove from hot
+    for (const row of toMove) {
+        await hotTable.delete(`id = '${row.id}'`);
+    }
+    // 3. Write raw digest artifact BEFORE table insert (crash safety:
+    //    if we crash after table insert but before raw write, reconcile
+    //    would delete the orphaned row. Writing raw first means reconcile
+    //    can always re-index from the file.)
+    const periodLabel = `${periodStart.slice(0, 10)}_${periodEnd.slice(0, 10)}`;
+    const rawDigest = {
+        id: digestId,
+        content: digestContent.summary,
+        summary: digestContent.summary,
+        topics: digestContent.topics,
+        period_start: periodStart,
+        period_end: periodEnd,
+        digest_type: "weekly",
+        created_at: now.toISOString(),
+        conversation_ids: conversationIds,
+    };
+    const digestFileName = `${periodLabel}_${timestamp}.json`;
+    const digestPath = path.join(deps.dataDir, "raw", "digests", digestFileName);
+    await atomicWrite(digestPath, JSON.stringify(rawDigest, null, 2));
+    // 4. Insert digest row into table
+    const digestRow = {
+        id: digestId,
+        content: digestContent.summary,
+        summary: digestContent.summary,
+        embedding: digestEmbedding,
+        period_start: periodStart,
+        period_end: periodEnd,
+        digest_type: "weekly",
+        created_at: now.toISOString(),
+    };
+    await digestTable.add([digestRow]);
+    // --- Mark this compaction as rollback-eligible (all mutations succeeded) ---
+    manifest.eligible = true;
+    manifest.digest_filename = digestFileName;
+    await writeManifest(deps.dataDir, manifest);
+    return {
+        moved: toMove.length,
+        digest_id: digestId,
+        dry_run: false,
+        rollback: false,
+    };
+}
+// ---------------------------------------------------------------------------
+// Rollback logic
+// ---------------------------------------------------------------------------
+async function handleRollback(deps, dryRun) {
+    const manifest = findEligibleManifest(deps.dataDir);
+    if (!manifest) {
+        // Should not reach here — route pre-checks. But guard defensively.
+        throw new Error("no_eligible_rollback");
+    }
+    if (dryRun) {
+        return {
+            moved: manifest.conversation_ids.length,
+            digest_id: manifest.digest_id,
+            dry_run: true,
+            rollback: true,
+        };
+    }
+    const hotTable = getTable("conversations_hot");
+    const coldTable = getTable("conversations_cold");
+    const digestTable = getTable("digests");
+    // Move conversations from cold back to hot
+    for (const convId of manifest.conversation_ids) {
+        const rows = (await coldTable
+            .query()
+            .where(`id = '${convId}'`)
+            .toArray());
+        if (rows.length > 0) {
+            await hotTable.add(rows.map(cleanConversationRow));
+            await coldTable.delete(`id = '${convId}'`);
+        }
+    }
+    // Remove the digest row from table
+    await digestTable.delete(`id = '${manifest.digest_id}'`);
+    // Remove the raw digest file so reconcile cannot resurrect it
+    if (manifest.digest_filename) {
+        const digestPath = path.join(deps.dataDir, "raw", "digests", manifest.digest_filename);
+        try {
+            fs.unlinkSync(digestPath);
+        }
+        catch {
+            // File may already be gone — that's fine
+        }
+    }
+    // Clear rollback eligibility
+    manifest.eligible = false;
+    await writeManifest(deps.dataDir, manifest);
+    return {
+        moved: manifest.conversation_ids.length,
+        digest_id: manifest.digest_id,
+        dry_run: false,
+        rollback: true,
+    };
+}
+// ---------------------------------------------------------------------------
+// Manifest helpers
+// ---------------------------------------------------------------------------
+function compactionDir(dataDir) {
+    return path.join(dataDir, COMPACTION_DIR);
+}
+async function writeManifest(dataDir, manifest) {
+    const dir = compactionDir(dataDir);
+    const filePath = path.join(dir, `${manifest.id}${MANIFEST_SUFFIX}`);
+    await atomicWrite(filePath, JSON.stringify(manifest, null, 2));
+}
+function findEligibleManifest(dataDir) {
+    const dir = compactionDir(dataDir);
+    if (!fs.existsSync(dir))
+        return null;
+    const files = fs
+        .readdirSync(dir)
+        .filter((f) => f.endsWith(MANIFEST_SUFFIX))
+        .sort()
+        .reverse(); // Most recent first
+    for (const file of files) {
+        try {
+            const raw = JSON.parse(fs.readFileSync(path.join(dir, file), "utf-8"));
+            if (raw.eligible)
+                return raw;
+        }
+        catch {
+            // Skip malformed manifests
+        }
+    }
+    return null;
+}
+function clearPriorEligibility(dataDir, currentId) {
+    const dir = compactionDir(dataDir);
+    if (!fs.existsSync(dir))
+        return;
+    const files = fs.readdirSync(dir).filter((f) => f.endsWith(MANIFEST_SUFFIX));
+    for (const file of files) {
+        const filePath = path.join(dir, file);
+        try {
+            const raw = JSON.parse(fs.readFileSync(filePath, "utf-8"));
+            if (raw.eligible && raw.id !== currentId) {
+                raw.eligible = false;
+                fs.writeFileSync(filePath, JSON.stringify(raw, null, 2), "utf-8");
+            }
+        }
+        catch {
+            // Skip malformed manifests
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function rowToEntry(row) {
+    return {
+        id: row.id,
+        content: row.content,
+        summary: row.summary,
+        source: row.source,
+        domain: row.domain,
+        intent: row.intent,
+        quality: row.quality,
+        topics: safeParseArray(row.topics),
+        decisions: safeParseArray(row.decisions),
+        key_outputs: safeParseArray(row.key_outputs),
+    };
+}
+/**
+ * Strip LanceDB internal fields (e.g. embedding.isValid) that appear
+ * in query results but aren't part of the table schema.
+ */
+function cleanConversationRow(row) {
+    // LanceDB query results include internal Arrow metadata (e.g. embedding.isValid).
+    // Convert embedding to a plain number[] to strip those properties.
+    const embedding = Array.from(row.embedding);
+    return {
+        id: row.id,
+        content: row.content,
+        summary: row.summary,
+        embedding,
+        source: row.source,
+        domain: row.domain,
+        intent: row.intent,
+        quality: row.quality,
+        topics: row.topics,
+        decisions: row.decisions,
+        key_outputs: row.key_outputs,
+        indexed_at: row.indexed_at,
+        idempotency_key: row.idempotency_key,
+    };
+}
+function safeParseArray(json) {
+    try {
+        const parsed = JSON.parse(json);
+        return Array.isArray(parsed) ? parsed : [];
+    }
+    catch {
+        return [];
+    }
+}
+//# sourceMappingURL=compact.js.map

package/dist/src/jobs/handlers/compact.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compact.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/compact.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAM7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AA4BtD,MAAM,cAAc,GAAG,YAAY,CAAC;AACpC,MAAM,eAAe,GAAG,gBAAgB,CAAC;AAEzC,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E,4DAA4D;AAC5D,MAAM,UAAU,mBAAmB,CAAC,OAAe;IACjD,OAAO,oBAAoB,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC;AAChD,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,UAAU,oBAAoB,CAAC,IAAiB;IACpD,OAAO,KAAK,EAAE,GAAQ,EAAiB,EAAE;QACvC,MAAM,OAAO,GAAI,GAAG,CAAC,OAA0B,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;QACjC,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEpC,qCAAqC;QACrC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ;YAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QACzC,IAAI,IAAI,CAAC,OAAO;YAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAEvC,IAAI,CAAC;YACH,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBACpD,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBACtD,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC;YACvB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,8DAA8D;YAC9D,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ;gBAAE,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC;YAChD,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC1C,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,KAAK,UAAU,gBAAgB,CAC7B,IAAiB,EACjB,MAAe;IAEf,MAAM,QAAQ,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IACjD,MAAM,WAAW,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IAExC,0CAA0C;IAC1C,MAAM,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;IAC1B,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IAEvC,MAAM,MAAM,GAAG,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAsB,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAElE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IACzE,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,SAAS,EAAE,IAAI;YACf,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE,KAAK;SAChB,CAAC;IACJ,CAAC;IAED,iDAAiD;IACjD,MAAM,OAAO,GAAwB,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACrE,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAElF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACrC,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEhD,qDAAqD;IACrD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IACvE,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACpE,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAElE,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAE1D,qFAAqF;IACrF,4EAA4E;IAC5E,oEAAoE;IACpE,MAAM,QAAQ,GAAqB;QACjC,EAAE,EAAE,SAAS;QACb,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE;QAC7B,gBAAgB,EAAE,eAAe;QACjC,SAAS,EAAE,QAAQ;QACnB,eAAe,EAAE,EAAE,EAAG,kCAAkC;QACxD,QAAQ,EAAE,KAAK;KAChB,CAAC;IACF,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAE5C,sCAAsC;IACtC,qBAAqB,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAE/C,0BAA0B;IAC1B,sDAAsD;IACtD,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAEtD,qBAAqB;IACrB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,QAAQ,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;IAC5C,CAAC;IAED,kEAAkE;IAClE,oEAAoE;IACpE,sEAAsE;IACtE,yCAAyC;IACzC,MAAM,WAAW,GAAG,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAC5E,MAAM,SAAS,GAAG;QAChB,EAAE,EAAE,QAAQ;QACZ,OAAO,EAAE,aAAa,CAAC,OAAO;QAC9B,OAAO,EAAE,aAAa,CAAC,OAAO;QAC9B,MAAM,EAAE,aAAa,CAAC,MAAM;QAC5B,YAAY,EAAE,WAAW;QACzB,UAAU,EAAE,SAAS;QACrB,WAAW,EAAE,QAAQ;QACrB,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE;QAC7B,gBAAgB,EAAE,eAAe;KAClC,CAAC;IACF,MAAM,cAAc,GAAG,GAAG,WAAW,IAAI,SAAS,OAAO,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;IAC7E,MAAM,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAElE,kCAAkC;IAClC,MAAM,SAAS,GAAc;QAC3B,EAAE,EAAE,QAAQ;QACZ,OAAO,EAAE,aAAa,CAAC,OAAO;QAC9B,OAAO,EAAE,aAAa,CAAC,OAAO;QAC9B,SAAS,EAAE,eAAe;QAC1B,YAAY,EAAE,WAAW;QACzB,UAAU,EAAE,SAAS;QACrB,WAAW,EAAE,QAAQ;QACrB,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE;KAC9B,CAAC;IACF,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnC,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,eAAe,GAAG,cAAc,CAAC;IAC1C,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAE5C,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,SAAS,EAAE,QAAQ;QACnB,OAAO,EAAE,KAAK;QACd,QAAQ,EAAE,KAAK;KAChB,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,KAAK,UAAU,cAAc,CAC3B,IAAiB,EACjB,MAAe;IAEf,MAAM,QAAQ,GAAG,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,mEAAmE;QACnE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,gBAAgB,CAAC,MAAM;YACvC,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IACjD,MAAM,WAAW,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IAExC,2CAA2C;IAC3C,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,CAAC,MAAM,SAAS;aAC1B,KAAK,EAAE;aACP,KAAK,CAAC,SAAS,MAAM,GAAG,CAAC;aACzB,OAAO,EAAE,CAAsB,CAAC;QACnC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;YACnD,MAAM,SAAS,CAAC,MAAM,CAAC,SAAS,MAAM,GAAG,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,mCAAmC;IACnC,MAAM,WAAW,CAAC,MAAM,CAAC,SAAS,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC;IAEzD,8DAA8D;IAC9D,IAAI,QAAQ,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,eAAe,CAAC,CAAC;QACvF,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,yCAAyC;QAC3C,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,QAAQ,CAAC,QAAQ,GAAG,KAAK,CAAC;IAC1B,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAE5C,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,gBAAgB,CAAC,MAAM;QACvC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,OAAO,EAAE,KAAK;QACd,QAAQ,EAAE,IAAI;KACf,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,OAAe,EACf,QAA0B;IAE1B,MAAM,GAAG,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,EAAE,GAAG,eAAe,EAAE,CAAC,CAAC;IACpE,MAAM,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAe;IAC3C,MAAM,GAAG,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAErC,MAAM,KAAK,GAAG,EAAE;SACb,WAAW,CAAC,GAAG,CAAC;SAChB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;SAC1C,IAAI,EAAE;SACN,OAAO,EAAE,CAAC,CAAC,oBAAoB;IAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAC3B,CAAC;YACtB,IAAI,GAAG,CAAC,QAAQ;gBAAE,OAAO,GAAG,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,2BAA2B;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe,EAAE,SAAiB;IAC/D,MAAM,GAAG,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO;IAEhC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;IAC7E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CACf,CAAC;YACtB,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;gBACzC,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC;gBACrB,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,2BAA2B;QAC7B,CAAC;IACH,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,UAAU,CAAC,GAAoB;IACtC,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC;QAClC,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC;QACxC,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC,WAAW,CAAC;KAC7C,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,GAAoB;IAChD,kFAAkF;IAClF,mEAAmE;IACnE,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAA6B,CAAC,CAAC;IAChE,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,SAAS;QACT,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,eAAe,EAAE,GAAG,CAAC,eAAe;KACrC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}

package/dist/src/jobs/handlers/ingest.d.ts

@@ -0,0 +1,13 @@
+import type { JobHandler } from "../types.js";
+import type { JobQueue } from "../queue.js";
+import type { EmbeddingProvider, MetadataExtractor } from "../../providers/types.js";
+import type { IngestPayload } from "../../server/routes/ingest.js";
+export interface IngestHandlerDeps {
+    embeddingProvider: EmbeddingProvider;
+    metadataExtractor: MetadataExtractor;
+    dataDir: string;
+    queue: JobQueue;
+    metadataExtractorFactory?: (payload: IngestPayload, deps: IngestHandlerDeps) => MetadataExtractor;
+}
+export declare function createIngestHandler(deps: IngestHandlerDeps): JobHandler;
+//# sourceMappingURL=ingest.d.ts.map

package/dist/src/jobs/handlers/ingest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/ingest.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAO,UAAU,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EACV,iBAAiB,EACjB,iBAAiB,EAElB,MAAM,0BAA0B,CAAC;AAUlC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAGnE,MAAM,WAAW,iBAAiB;IAChC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,QAAQ,CAAC;IAChB,wBAAwB,CAAC,EAAE,CACzB,OAAO,EAAE,aAAa,EACtB,IAAI,EAAE,iBAAiB,KACpB,iBAAiB,CAAC;CACxB;AA6DD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,UAAU,CA4JvE"}