@getrift/rift 0.0.0 → 0.1.0-beta.1

package/dist/src/jobs/handlers/save.js

@@ -0,0 +1,206 @@
+/**
+ * Save job handler — Slice 8.
+ *
+ * Processes a POST /save payload:
+ * 1. Write raw payload to data/raw/conversations/{source}/{timestamp}_{id}.json
+ * 2. Generate embedding via EmbeddingProvider
+ * 3. Insert row into conversations_hot
+ *
+ * On failure (after raw write): quarantine to data/quarantine/{source}_{timestamp}_{hash}.json
+ * Raw file is written BEFORE table insert (crash safety).
+ *
+ * Replay safety:
+ * - Row ID is the root of the retry chain (walks `retry_of` to the
+ *   first ancestor), so multi-hop retries all converge on one ID.
+ * - Before inserting, checks if the row already exists. If it does
+ *   (prior run succeeded), the handler returns early — no mutation.
+ * - Raw file is written once per stable ID; replays detect the existing
+ *   file, skip the write, and use the raw file's payload as the
+ *   authoritative content. This prevents drift between the canonical
+ *   raw artifact and the indexed row.
+ */
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { getTable } from "../../storage/tables.js";
+import { atomicWrite } from "../../storage/atomic.js";
+/**
+ * Walk the `retry_of` chain to the root job.
+ *
+ * Queue retries point to the immediate failed parent, not the
+ * original. A -> B (retry_of=A) -> C (retry_of=B). This function
+ * returns A in all three cases, so all retries share one stable ID
+ * and one canonical payload.
+ */
+function resolveChainRoot(job, queue) {
+    let current = job;
+    let depth = 0;
+    while (current.retry_of && depth < 50) {
+        const parent = queue.get(current.retry_of);
+        if (!parent)
+            break; // Parent not in queue (e.g. pruned) — stop here.
+        current = parent;
+        depth++;
+    }
+    return current;
+}
+/**
+ * Find the raw file for `rowId` under `rawDir`, if it exists.
+ * Files are named `{timestamp}_{rowId}.json`, so we match the suffix.
+ * Returns the full path, or undefined if not found.
+ */
+function findRawFile(rawDir, rowId) {
+    if (!fs.existsSync(rawDir))
+        return undefined;
+    const suffix = `_${rowId}.json`;
+    const match = fs.readdirSync(rawDir).find((f) => f.endsWith(suffix));
+    return match ? path.join(rawDir, match) : undefined;
+}
+export function createSaveHandler(deps) {
+    return async (job) => {
+        const jobPayload = job.payload;
+        if (!jobPayload || !jobPayload.source || !jobPayload.summary) {
+            throw new Error("Invalid save job payload: missing source or summary");
+        }
+        // Resolve the chain root to get a stable ID and the original
+        // payload's source — the raw directory is always keyed to the
+        // first job's source, not the retry's.
+        const rootJob = resolveChainRoot(job, deps.queue);
+        const rootPayload = rootJob.payload ?? jobPayload;
+        const rowId = rootJob.id;
+        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+        // --- 1. Write raw payload to disk (before table insert) ---
+        // Always use the root payload's source for the directory path so
+        // retries with a changed source still find the canonical artifact.
+        const rawDir = path.join(deps.dataDir, "raw", "conversations", rootPayload.source);
+        let payload;
+        const existingRawPath = findRawFile(rawDir, rowId);
+        // Track the path if we write a NEW raw file (not a replay).
+        // Used in the catch block to clean up on failed replacement saves.
+        let newRawPath;
+        if (existingRawPath) {
+            // Replay: read the canonical raw file so the indexed row always
+            // agrees with the raw artifact, even if the retry carried a
+            // different payload (including a different source).
+            payload = JSON.parse(fs.readFileSync(existingRawPath, "utf-8"));
+        }
+        else {
+            // First run: persist the raw artifact using the original payload.
+            payload = rootPayload;
+            const rawFileName = `${timestamp}_${rowId}.json`;
+            newRawPath = path.join(rawDir, rawFileName);
+            await atomicWrite(newRawPath, JSON.stringify(payload, null, 2));
+        }
+        // --- 2. Generate embedding + 3. Insert into conversations_hot ---
+        // Both steps are wrapped so ANY failure after raw write is quarantined.
+        try {
+            const table = getTable("conversations_hot");
+            // Replay guard: if the row already exists from a prior successful
+            // run, return early. No delete, so no risk of data loss.
+            const existing = await table
+                .query()
+                .where(`id = '${rowId}'`)
+                .toArray();
+            if (existing.length > 0) {
+                return; // Already indexed — nothing to do.
+            }
+            // Generate embedding FIRST — before any deletions so the old row
+            // is preserved if embedding fails (data-safety invariant).
+            const textForEmbedding = payload.content ?? payload.summary;
+            const embedding = await deps.embeddingProvider.embed(textForEmbedding);
+            const row = {
+                id: rowId,
+                content: payload.content ?? payload.summary,
+                summary: payload.summary,
+                embedding,
+                source: payload.source,
+                domain: payload.domain ?? "tech",
+                intent: "build", // Default — no LLM extraction in this slice
+                quality: "medium", // Default — no LLM extraction in this slice
+                topics: JSON.stringify(payload.topics ?? []),
+                decisions: JSON.stringify(payload.decisions ?? []),
+                key_outputs: JSON.stringify(payload.key_outputs ?? []),
+                indexed_at: new Date().toISOString(),
+                idempotency_key: payload.idempotency_key ?? "",
+            };
+            // Insert the new row FIRST. Old-version cleanup happens after the
+            // insert succeeds, so a failed insert never destroys the old version.
+            await table.add([row]);
+            // Clean up old version — best-effort AFTER new row is safely stored.
+            // Failure here means a temporary duplicate (old + new coexist),
+            // which is recoverable; the alternative (deleting old before insert
+            // succeeds) risks losing both versions on insert failure.
+            if (payload.replace_idempotency_key) {
+                try {
+                    const coldTable = getTable("conversations_cold");
+                    const hotOldRows = await table
+                        .query()
+                        .where(`idempotency_key = '${payload.replace_idempotency_key}'`)
+                        .toArray();
+                    const coldOldRows = await coldTable
+                        .query()
+                        .where(`idempotency_key = '${payload.replace_idempotency_key}'`)
+                        .toArray();
+                    const oldRow = hotOldRows[0] ?? coldOldRows[0];
+                    if (oldRow) {
+                        const oldRawDir = path.join(deps.dataDir, "raw", "conversations", oldRow.source);
+                        const oldRawPath = findRawFile(oldRawDir, oldRow.id);
+                        if (oldRawPath) {
+                            try {
+                                fs.unlinkSync(oldRawPath);
+                            }
+                            catch { /* best-effort */ }
+                        }
+                    }
+                    if (hotOldRows.length > 0) {
+                        await table.delete(`idempotency_key = '${payload.replace_idempotency_key}'`);
+                    }
+                    if (coldOldRows.length > 0) {
+                        await coldTable.delete(`idempotency_key = '${payload.replace_idempotency_key}'`);
+                    }
+                }
+                catch (cleanupErr) {
+                    // Log but don't throw — new version is safely saved.
+                    // Temporary duplicate is not data loss; reconcile-level
+                    // dedup can address it if needed.
+                    process.stderr.write(`save: warning: failed to clean up replaced version (key=${payload.replace_idempotency_key}): ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}\n`);
+                }
+            }
+        }
+        catch (err) {
+            // If this was a replacement save and we wrote a new raw file,
+            // clean it up so reconcile/rebuild won't resurrect the failed
+            // version alongside the still-valid old version. For non-
+            // replacement first-time saves the raw file is the intended
+            // safety net — reconcile recovers from it.
+            if (newRawPath && payload.replace_idempotency_key) {
+                try {
+                    fs.unlinkSync(newRawPath);
+                }
+                catch { /* best-effort */ }
+            }
+            await quarantine(deps.dataDir, payload, err);
+            throw err; // Re-throw so the job is marked failed
+        }
+    };
+}
+/**
+ * Write a quarantine file for a failed save.
+ * Path: data/quarantine/{source}_{timestamp}_{hash}.json
+ */
+async function quarantine(dataDir, payload, error) {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const hash = crypto
+        .createHash("sha256")
+        .update(JSON.stringify(payload))
+        .digest("hex")
+        .slice(0, 12);
+    const quarantinePath = path.join(dataDir, "quarantine", `${payload.source}_${timestamp}_${hash}.json`);
+    const quarantineData = {
+        payload,
+        error: error instanceof Error ? error.message : String(error),
+        quarantined_at: new Date().toISOString(),
+    };
+    await atomicWrite(quarantinePath, JSON.stringify(quarantineData, null, 2));
+}
+//# sourceMappingURL=save.js.map

package/dist/src/jobs/handlers/save.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"save.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/save.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAK7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAStD;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,GAAQ,EAAE,KAAe;IACjD,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,OAAO,CAAC,QAAQ,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,MAAM,CAAC,iDAAiD;QACrE,OAAO,GAAG,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,MAAc,EAAE,KAAa;IAChD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;IAChC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACrE,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAqB;IACrD,OAAO,KAAK,EAAE,GAAQ,EAAiB,EAAE;QACvC,MAAM,UAAU,GAAG,GAAG,CAAC,OAAsB,CAAC;QAC9C,IAAI,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QAED,6DAA6D;QAC7D,8DAA8D;QAC9D,uCAAuC;QACvC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,WAAW,GAAI,OAAO,CAAC,OAAmC,IAAI,UAAU,CAAC;QAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAEjE,6DAA6D;QAC7D,iEAAiE;QACjE,mEAAmE;QACnE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CACtB,IAAI,CAAC,OAAO,EACZ,KAAK,EACL,eAAe,EACf,WAAW,CAAC,MAAM,CACnB,CAAC;QAEF,IAAI,OAAoB,CAAC;QACzB,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACnD,4DAA4D;QAC5D,mEAAmE;QACnE,IAAI,UAA8B,CAAC;QAEnC,IAAI,eAAe,EAAE,CAAC;YACpB,gEAAgE;YAChE,4DAA4D;YAC5D,oDAAoD;YACpD,OAAO,GAAG,IAAI,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAC3B,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,kEAAkE;YAClE,OAAO,GAAG,WAAW,CAAC;YACtB,MAAM,WAAW,GAAG,GAAG,SAAS,IAAI,KAAK,OAAO,CAAC;YACjD,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAC5C,MAAM,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAClE,CAAC;QAED,mEAAmE;QACnE,wEAAwE;QACxE,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;YAE5C,kEAAkE;YAClE,yDAAyD;YACzD,MAAM,QAAQ,GAAG,MAAM,KAAK;iBACzB,KAAK,EAAE;iBACP,KAAK,CAAC,SAAS,KAAK,GAAG,CAAC;iBACxB,OAAO,EAAE,CAAC;YACb,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,OAAO,CAAC,mCAAmC;YAC7C,CAAC;YAED,iEAAiE;YACjE,2DAA2D;YAC3D,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;YAC5D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;YAEvE,MAAM,GAAG,GAAoB;gBAC3B,EAAE,EAAE,KAAK;gBACT,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO;gBAC3C,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,SAAS;gBACT,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM;gBAChC,MAAM,EAAE,OAAO,EAAE,4CAA4C;gBAC7D,OAAO,EAAE,QAAQ,EAAE,4CAA4C;gBAC/D,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;gBAC5C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;gBAClD,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;gBACtD,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;aAC/C,CAAC;YAEF,kEAAkE;YAClE,sEAAsE;YACtE,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAEvB,qEAAqE;YACrE,gEAAgE;YAChE,oEAAoE;YACpE,0DAA0D;YAC1D,IAAI,OAAO,CAAC,uBAAuB,EAAE,CAAC;gBACpC,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;oBACjD,MAAM,UAAU,GAAG,MAAM,KAAK;yBAC3B,KAAK,EAAE;yBACP,KAAK,CAAC,sBAAsB,OAAO,CAAC,uBAAuB,GAAG,CAAC;yBAC/D,OAAO,EAAuB,CAAC;oBAClC,MAAM,WAAW,GAAG,MAAM,SAAS;yBAChC,KAAK,EAAE;yBACP,KAAK,CAAC,sBAAsB,OAAO,CAAC,uBAAuB,GAAG,CAAC;yBAC/D,OAAO,EAAuB,CAAC;oBAElC,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;oBAC/C,IAAI,MAAM,EAAE,CAAC;wBACX,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CACzB,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC,MAAM,CACpD,CAAC;wBACF,MAAM,UAAU,GAAG,WAAW,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;wBACrD,IAAI,UAAU,EAAE,CAAC;4BACf,IAAI,CAAC;gCAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;4BAAC,CAAC;4BAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;wBAChE,CAAC;oBACH,CAAC;oBACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC1B,MAAM,KAAK,CAAC,MAAM,CAAC,sBAAsB,OAAO,CAAC,uBAAuB,GAAG,CAAC,CAAC;oBAC/E,CAAC;oBACD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC3B,MAAM,SAAS,CAAC,MAAM,CAAC,sBAAsB,OAAO,CAAC,uBAAuB,GAAG,CAAC,CAAC;oBACnF,CAAC;gBACH,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,qDAAqD;oBACrD,wDAAwD;oBACxD,kCAAkC;oBAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,2DAA2D,OAAO,CAAC,uBAAuB,MAAM,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAC1K,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,8DAA8D;YAC9D,8DAA8D;YAC9D,0DAA0D;YAC1D,4DAA4D;YAC5D,2CAA2C;YAC3C,IAAI,UAAU,IAAI,OAAO,CAAC,uBAAuB,EAAE,CAAC;gBAClD,IAAI,CAAC;oBAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAChE,CAAC;YACD,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;YAC7C,MAAM,GAAG,CAAC,CAAC,uCAAuC;QACpD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,UAAU,CACvB,OAAe,EACf,OAAoB,EACpB,KAAc;IAEd,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,MAAM;SAChB,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;SAC/B,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEhB,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAC9B,OAAO,EACP,YAAY,EACZ,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,IAAI,IAAI,OAAO,CAC9C,CAAC;IAEF,MAAM,cAAc,GAAG;QACrB,OAAO;QACP,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QAC7D,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACzC,CAAC;IAEF,MAAM,WAAW,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC"}

package/dist/src/jobs/handlers/triage.d.ts

@@ -0,0 +1,47 @@
+import type { JobHandler } from "../types.js";
+import { type TriageResult } from "../../capture/triage-provider.js";
+import type { OpenAIConfig } from "../../config/schema.js";
+import { type HistoricalTriageMeteringMode, type OperatorOverrideProvider } from "../../providers/operator-overrides.js";
+import type { CostReportingTriageProvider } from "../../capture/batch-budget.js";
+export interface TriageJobItem {
+    id: string;
+    content: string;
+    source?: string;
+}
+export interface TriageJobPayload {
+    items: TriageJobItem[];
+    /**
+     * Retained for route/backfill compatibility. Codex CLI is currently the
+     * only live triage provider, so provider_override is accepted but does not
+     * change provider selection.
+     */
+    provider_override?: OperatorOverrideProvider;
+}
+export interface TriageJobItemResult {
+    id: string;
+    triage?: TriageResult;
+    error?: string;
+}
+export interface TriageJobOutput {
+    job_id: string;
+    completed_at: string;
+    total: number;
+    succeeded: number;
+    failed: number;
+    cost_cap_exceeded: boolean;
+    metering_mode: HistoricalTriageMeteringMode;
+    spend_note?: string;
+    spent_usd: number;
+    cost_cap_usd: number;
+    items: TriageJobItemResult[];
+}
+export interface TriageHandlerDeps {
+    dataDir: string;
+    openai: OpenAIConfig;
+    /** Override the API-key resolver (tests). Defaults to macOS Keychain. */
+    apiKey?: () => Promise<string>;
+    /** Override the provider factory (tests). */
+    providerFactory?: (deps: TriageHandlerDeps, payload: TriageJobPayload) => CostReportingTriageProvider;
+}
+export declare function createTriageHandler(deps: TriageHandlerDeps): JobHandler;
+//# sourceMappingURL=triage.d.ts.map

package/dist/src/jobs/handlers/triage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"triage.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/triage.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAO,UAAU,EAAE,MAAM,aAAa,CAAC;AAOnD,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAE1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAGL,KAAK,4BAA4B,EACjC,KAAK,wBAAwB,EAC9B,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAEjF,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,wBAAwB,CAAC;CAC9C;AAED,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,EAAE,4BAA4B,CAAC;IAC5C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,mBAAmB,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,YAAY,CAAC;IACrB,yEAAyE;IACzE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/B,6CAA6C;IAC7C,eAAe,CAAC,EAAE,CAChB,IAAI,EAAE,iBAAiB,EACvB,OAAO,EAAE,gBAAgB,KACtB,2BAA2B,CAAC;CAClC;AAYD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,UAAU,CAkFvE"}

package/dist/src/jobs/handlers/triage.js

@@ -0,0 +1,95 @@
+/**
+ * Triage job handler — Slice 5.
+ *
+ * Triage is a first-class job type so batches can be queued, retried,
+ * and observed independently of ingest. The handler runs every item in
+ * the payload through a single guarded TriageProvider so a per-batch
+ * cost cap and parallelism limit are enforced at the job boundary.
+ *
+ * Output: a JSON file at `data/triage-results/{jobId}.json` containing
+ * one entry per input item with the triage decision + provenance
+ * (decision, confidence, rationale, model, provider, version). Callers
+ * (e.g. follow-up routing) read this file. We deliberately don't fan
+ * out save/review-queue side effects from inside this handler — Slice 5
+ * is about the runtime, not a redesign of the routing pipeline.
+ *
+ * BatchCostExceededError aborts the job: the failure is surfaced via
+ * the job's error field, which is operator-visible.
+ */
+import path from "node:path";
+import { atomicWrite } from "../../storage/atomic.js";
+import { BatchBudget, Semaphore, withBatchGuards, } from "../../capture/batch-budget.js";
+import { BatchCostExceededError, } from "../../capture/triage-provider.js";
+import { CodexCliTriageProvider } from "../../capture/codex-cli-triage-provider.js";
+import { resolveOperatorOverrideProvider, triageMeteringModeForProvider, } from "../../providers/operator-overrides.js";
+function newDefaultProvider(_deps, _payload) {
+    // OpenAI remains available only via explicit test/providerFactory injection;
+    // the production default is intentionally Codex CLI so live capture uses the
+    // operator's existing CLI auth instead of a separate Keychain secret.
+    return new CodexCliTriageProvider();
+}
+export function createTriageHandler(deps) {
+    return async (job) => {
+        const payload = (job.payload ?? {});
+        const items = Array.isArray(payload.items) ? payload.items : [];
+        const providerOverride = resolveOperatorOverrideProvider(payload.provider_override);
+        if (items.length === 0) {
+            throw new Error("Triage job payload missing items");
+        }
+        const effectivePayload = {
+            items,
+            ...(providerOverride ? { provider_override: providerOverride } : {}),
+        };
+        const provider = (deps.providerFactory ?? newDefaultProvider)(deps, effectivePayload);
+        const meteringMode = triageMeteringModeForProvider(providerOverride ?? (deps.providerFactory ? undefined : "codex-cli"));
+        const budget = new BatchBudget(deps.openai.max_cost_per_batch_usd);
+        const semaphore = new Semaphore(deps.openai.max_parallel_requests);
+        const guarded = withBatchGuards(provider, budget, semaphore);
+        const results = [];
+        let costCapExceeded = false;
+        // Run all items in parallel — the semaphore enforces the in-flight cap.
+        await Promise.all(items.map(async (item) => {
+            try {
+                const triage = await guarded.triage({
+                    content: item.content,
+                    ...(item.source ? { source: item.source } : {}),
+                });
+                results.push({ id: item.id, triage });
+            }
+            catch (err) {
+                if (err instanceof BatchCostExceededError) {
+                    costCapExceeded = true;
+                }
+                results.push({
+                    id: item.id,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }));
+        const output = {
+            job_id: job.id,
+            completed_at: new Date().toISOString(),
+            total: items.length,
+            succeeded: results.filter((r) => r.triage !== undefined).length,
+            failed: results.filter((r) => r.error !== undefined).length,
+            cost_cap_exceeded: costCapExceeded,
+            metering_mode: meteringMode,
+            ...(meteringMode === "unmetered_codex_cli"
+                ? {
+                    spend_note: "Codex CLI historical backfill override is unmetered by OpenAI USD.",
+                }
+                : {}),
+            spent_usd: meteringMode === "openai_usd" ? budget.spent : 0,
+            cost_cap_usd: meteringMode === "openai_usd" ? budget.capUsd : 0,
+            items: results,
+        };
+        const outPath = path.join(deps.dataDir, "triage-results", `${job.id}.json`);
+        await atomicWrite(outPath, JSON.stringify(output, null, 2));
+        if (costCapExceeded) {
+            throw new BatchCostExceededError(`Triage job ${job.id} aborted: batch cost cap of $${budget.capUsd.toFixed(2)} exceeded ` +
+                `(spent $${budget.spent.toFixed(4)}, ${output.succeeded}/${output.total} completed before cap). ` +
+                `Partial results written to ${outPath}.`, budget.spent, budget.capUsd);
+        }
+    };
+}
+//# sourceMappingURL=triage.js.map

package/dist/src/jobs/handlers/triage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"triage.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/triage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EACL,WAAW,EACX,SAAS,EACT,eAAe,GAChB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,sBAAsB,GAEvB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,sBAAsB,EAAE,MAAM,4CAA4C,CAAC;AAEpF,OAAO,EACL,+BAA+B,EAC/B,6BAA6B,GAG9B,MAAM,uCAAuC,CAAC;AAmD/C,SAAS,kBAAkB,CACzB,KAAwB,EACxB,QAA0B;IAE1B,6EAA6E;IAC7E,6EAA6E;IAC7E,sEAAsE;IACtE,OAAO,IAAI,sBAAsB,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,OAAO,KAAK,EAAE,GAAQ,EAAiB,EAAE;QACvC,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAA8B,CAAC;QACjE,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,MAAM,gBAAgB,GAAG,+BAA+B,CACtD,OAAO,CAAC,iBAAiB,CAC1B,CAAC;QACF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,gBAAgB,GAAqB;YACzC,KAAK;YACL,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC;QACF,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAC3D,IAAI,EACJ,gBAAgB,CACjB,CAAC;QACF,MAAM,YAAY,GAAG,6BAA6B,CAChD,gBAAgB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CACrE,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACnE,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACnE,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAE7D,MAAM,OAAO,GAA0B,EAAE,CAAC;QAC1C,IAAI,eAAe,GAAG,KAAK,CAAC;QAE5B,wEAAwE;QACxE,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC;oBAClC,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAChD,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,GAAG,YAAY,sBAAsB,EAAE,CAAC;oBAC1C,eAAe,GAAG,IAAI,CAAC;gBACzB,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC;oBACX,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACtC,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;YAC/D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;YAC3D,iBAAiB,EAAE,eAAe;YAClC,aAAa,EAAE,YAAY;YAC3B,GAAG,CAAC,YAAY,KAAK,qBAAqB;gBACxC,CAAC,CAAC;oBACE,UAAU,EACR,oEAAoE;iBACvE;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,SAAS,EAAE,YAAY,KAAK,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC3D,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/D,KAAK,EAAE,OAAO;SACf,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;QAC5E,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAE5D,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,IAAI,sBAAsB,CAC9B,cAAc,GAAG,CAAC,EAAE,gCAAgC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY;gBACtF,WAAW,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,KAAK,0BAA0B;gBACjG,8BAA8B,OAAO,GAAG,EAC1C,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,MAAM,CACd,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/jobs/queue.d.ts

@@ -0,0 +1,107 @@
+import type { Job, JobType, CreateJobResult, JobHandler } from "./types.js";
+/**
+ * In-process job queue with:
+ * - Persistence to `queue.json` (atomic write: temp + rename)
+ * - State machine: queued -> running -> completed | failed, plus cancelled, interrupted
+ * - Idempotency checked before mutual exclusion (PRD 4.1)
+ * - Mutual exclusion for {compact, reindex, reconcile}
+ * - Crash recovery: running -> interrupted -> re-queued on startup
+ */
+export declare class JobQueue {
+    private queuePath;
+    private jobs;
+    private handlers;
+    private processing;
+    private held;
+    constructor(queuePath: string);
+    /**
+     * Hold processing — registerHandler and create won't auto-start queued
+     * work until releaseHold() is called. Used at startup to prevent
+     * recovered jobs from executing before the integrity check completes.
+     */
+    hold(): void;
+    /** Release hold and kick processing for any waiting jobs. */
+    releaseHold(): void;
+    /** Register a handler for a job type. Kicks processing for any queued work. */
+    registerHandler(type: JobType, handler: JobHandler): void;
+    /** Check if a handler is registered for a job type. */
+    hasHandler(type: JobType): boolean;
+    /**
+     * Load persisted queue from disk. Mark any `running` jobs as
+     * `interrupted` and re-queue them — timers/progress don't survive restart.
+     *
+     * Only jobs that were `running` at crash time get re-queued. Jobs that
+     * were already `interrupted` (from a previous recovery) are left as-is
+     * so repeated loads don't multiply work.
+     */
+    load(): Promise<void>;
+    /**
+     * Create a job. Returns `{ job, duplicate }`.
+     *
+     * Idempotency is checked first (PRD 4.1):
+     * - If a job with the same `idempotency_key` exists in
+     *   `queued | running | completed` -> return it with `duplicate: true`.
+     * - If it exists in `failed` -> create a new retry job with `retry_of`.
+     *
+     * Mutual exclusion is checked second:
+     * - If the job type is in the exclusive set and another exclusive job
+     *   is `queued | running`, return null (caller should 409).
+     */
+    create(type: JobType, opts?: {
+        idempotency_key?: string;
+        payload?: unknown;
+    }): Promise<CreateJobResult | {
+        conflict: true;
+    }>;
+    /** Get a job by ID, or undefined. */
+    get(id: string): Job | undefined;
+    /**
+     * Check whether an idempotency key matches an existing active job.
+     * Returns the job if found in queued/running/completed state, undefined otherwise.
+     * Does NOT create or mutate anything — safe for pre-flight checks.
+     */
+    findDuplicate(idempotencyKey: string): Job | undefined;
+    /**
+     * Cancel a job.
+     * - `queued` or `interrupted` -> `cancelled`, returns the job.
+     * - `running` -> returns `{ conflict: true }`.
+     * - Others -> returns undefined (not found or not cancellable).
+     */
+    cancel(id: string): Promise<Job | {
+        conflict: true;
+    } | undefined>;
+    /** Atomic write: temp file + rename. */
+    private persist;
+    /**
+     * Find the active job for a given idempotency key.
+     * Skip superseded jobs (those with `retried_by` set) so we always
+     * find the leaf of the retry chain, not the old parent.
+     */
+    private findByIdempotencyKey;
+    /**
+     * Create a job that bypasses the mutual exclusion check and is
+     * inserted at the front of the queue (runs before other queued work).
+     *
+     * Used exclusively for startup corruption repair: the mandatory
+     * rebuild must run before any other queued work touches storage,
+     * regardless of what else is in the queue. Other exclusive jobs
+     * naturally wait (one-at-a-time processing) and run after the
+     * rebuild against healthy storage.
+     */
+    createUrgent(type: JobType, opts?: {
+        payload?: unknown;
+    }): Promise<CreateJobResult>;
+    /** Whether any exclusive job is queued or running. */
+    hasActiveExclusiveJob(): boolean;
+    /**
+     * Return the exclusive job that is currently running, or null.
+     * Only returns jobs with status "running" — queued exclusive jobs
+     * have not entered their coordinated maintenance phase.
+     */
+    getRunningExclusiveJob(): Job | null;
+    private createRetryJob;
+    /** Process the next queued job if no job is currently running. */
+    private processNext;
+    private findNextQueued;
+}
+//# sourceMappingURL=queue.d.ts.map

package/dist/src/jobs/queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../../../src/jobs/queue.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,GAAG,EAAE,OAAO,EAAa,eAAe,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGvF;;;;;;;GAOG;AACH,qBAAa,QAAQ;IAMP,OAAO,CAAC,SAAS;IAL7B,OAAO,CAAC,IAAI,CAA0B;IACtC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,IAAI,CAAS;gBAED,SAAS,EAAE,MAAM;IAErC;;;;OAIG;IACH,IAAI,IAAI,IAAI;IAIZ,6DAA6D;IAC7D,WAAW,IAAI,IAAI;IAKnB,+EAA+E;IAC/E,eAAe,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,GAAG,IAAI;IAKzD,uDAAuD;IACvD,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO;IAIlC;;;;;;;OAOG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA8C3B;;;;;;;;;;;OAWG;IACG,MAAM,CACV,IAAI,EAAE,OAAO,EACb,IAAI,CAAC,EAAE;QAAE,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,GACrD,OAAO,CAAC,eAAe,GAAG;QAAE,QAAQ,EAAE,IAAI,CAAA;KAAE,CAAC;IA0ChD,qCAAqC;IACrC,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS;IAIhC;;;;OAIG;IACH,aAAa,CAAC,cAAc,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS;IAatD;;;;;OAKG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG;QAAE,QAAQ,EAAE,IAAI,CAAA;KAAE,GAAG,SAAS,CAAC;IAkBvE,wCAAwC;YAC1B,OAAO;IAOrB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAO5B;;;;;;;;;OASG;IACG,YAAY,CAChB,IAAI,EAAE,OAAO,EACb,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,GAC3B,OAAO,CAAC,eAAe,CAAC;IAwB3B,sDAAsD;IACtD,qBAAqB,IAAI,OAAO;IAYhC;;;;OAIG;IACH,sBAAsB,IAAI,GAAG,GAAG,IAAI;YAStB,cAAc;IA6B5B,kEAAkE;IAClE,OAAO,CAAC,WAAW;IAgCnB,OAAO,CAAC,cAAc;CAMvB"}