@getrift/rift 0.1.0-beta.2 → 0.1.0-beta.21

package/dist/src/server/routes/triage.js

@@ -10,6 +10,7 @@
  */
 import { z } from "zod";
 import { toJobStatusView } from "../../jobs/types.js";
+import { enqueueJob } from "./enqueue.js";
 import { HISTORICAL_BACKFILL_CAPABILITY_HEADER, HISTORICAL_BACKFILL_OVERRIDE_HEADER, INTERNAL_HISTORICAL_TRIAGE_CODEX_ROUTE, CODEX_CLI_OVERRIDE_PROVIDER, hasHistoricalBackfillCapability, isHistoricalTriageIdempotencyKey, } from "../../providers/operator-overrides.js";
 const TriageItemSchema = z
     .object({
@@ -75,7 +76,9 @@ export function registerTriageRoute(server, queue, dataDir) {
         if (parsed.data.idempotency_key !== undefined) {
             opts.idempotency_key = parsed.data.idempotency_key;
         }
-        const result = await queue.create("triage", opts);
+        const result = await enqueueJob(reply, queue, "triage", opts);
+        if (!result)
+            return;
         if ("conflict" in result) {
             return reply.code(409).send({ error: "conflict" });
         }

package/dist/src/server/routes/triage.js.map

@@ -1 +1 @@
-{"version":3,"file":"triage.js","sourceRoot":"","sources":["../../../../src/server/routes/triage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EACL,qCAAqC,EACrC,mCAAmC,EACnC,sCAAsC,EACtC,2BAA2B,EAC3B,+BAA+B,EAC/B,gCAAgC,GAEjC,MAAM,uCAAuC,CAAC;AAE/C,MAAM,gBAAgB,GAAG,CAAC;KACvB,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACrC,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,mBAAmB,GAAG,CAAC;KAC1B,MAAM,CAAC;IACN,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;IACjD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACvC,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,UAAU,mBAAmB,CACjC,MAAuB,EACvB,KAAe,EACf,OAAe;IAEf,MAAM,YAAY,GAAG,KAAK,EACxB,OAAuB,EACvB,KAAmB,EACnB,gBAA2C,EAC3C,2BAA2B,GAAG,KAAK,EACnC,EAAE;QACF,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBAC1B,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;aAC7B,CAAC,CAAC;QACL,CAAC;QAED,IACE,gBAAgB,KAAK,SAAS;YAC9B,CAAC,OAAO,CAAC,OAAO,CAAC,mCAAmC,CAAC,KAAK,SAAS;gBACjE,OAAO,CAAC,OAAO,CAAC,qCAAqC,CAAC,KAAK,SAAS,CAAC,EACvE,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBAC1B,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE;4BACJ,OAAO,CAAC,OAAO,CAAC,qCAAqC,CAAC,KAAK,SAAS;gCAClE,CAAC,CAAC,qCAAqC;gCACvC,CAAC,CAAC,mCAAmC;yBACxC;wBACD,OAAO,EACL,qFAAqF;qBACxF;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,IACE,2BAA2B;YAC3B,CAAC,+BAA+B,CAC9B,OAAO,EACP,OAAO,CAAC,OAAO,CAAC,qCAAqC,CAAC,CACvD,EACD,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACtD,CAAC;QAED,IACE,2BAA2B;YAC3B,CAAC,gCAAgC,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,EAC9D,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBAC1B,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,CAAC,iBAAiB,CAAC;wBACzB,OAAO,EACL,qEAAqE;qBACxE;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,GAMN;YACF,OAAO,EAAE;gBACP,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;gBACxB,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrE;SACF,CAAC;QACF,IAAI,MAAM,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAC9C,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC;QACrD,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAElD,IAAI,UAAU,IAAI,MAAM,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBAC1B,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;gBACrB,SAAS,EAAE,IAAI;gBACf,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IACzD,CAAC,CAAC;IAEF,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,CAC9C,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,CAC7B,CAAC;IAEF,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,CAC3E,YAAY,CAAC,OAAO,EAAE,KAAK,EAAE,2BAA2B,EAAE,IAAI,CAAC,CAChE,CAAC;AACJ,CAAC"}
+{"version":3,"file":"triage.js","sourceRoot":"","sources":["../../../../src/server/routes/triage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EACL,qCAAqC,EACrC,mCAAmC,EACnC,sCAAsC,EACtC,2BAA2B,EAC3B,+BAA+B,EAC/B,gCAAgC,GAEjC,MAAM,uCAAuC,CAAC;AAE/C,MAAM,gBAAgB,GAAG,CAAC;KACvB,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACrC,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,mBAAmB,GAAG,CAAC;KAC1B,MAAM,CAAC;IACN,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;IACjD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACvC,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,UAAU,mBAAmB,CACjC,MAAuB,EACvB,KAAe,EACf,OAAe;IAEf,MAAM,YAAY,GAAG,KAAK,EACxB,OAAuB,EACvB,KAAmB,EACnB,gBAA2C,EAC3C,2BAA2B,GAAG,KAAK,EACnC,EAAE;QACF,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBAC1B,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;aAC7B,CAAC,CAAC;QACL,CAAC;QAED,IACE,gBAAgB,KAAK,SAAS;YAC9B,CAAC,OAAO,CAAC,OAAO,CAAC,mCAAmC,CAAC,KAAK,SAAS;gBACjE,OAAO,CAAC,OAAO,CAAC,qCAAqC,CAAC,KAAK,SAAS,CAAC,EACvE,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBAC1B,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE;4BACJ,OAAO,CAAC,OAAO,CAAC,qCAAqC,CAAC,KAAK,SAAS;gCAClE,CAAC,CAAC,qCAAqC;gCACvC,CAAC,CAAC,mCAAmC;yBACxC;wBACD,OAAO,EACL,qFAAqF;qBACxF;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,IACE,2BAA2B;YAC3B,CAAC,+BAA+B,CAC9B,OAAO,EACP,OAAO,CAAC,OAAO,CAAC,qCAAqC,CAAC,CACvD,EACD,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACtD,CAAC;QAED,IACE,2BAA2B;YAC3B,CAAC,gCAAgC,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,EAC9D,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBAC1B,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,CAAC,iBAAiB,CAAC;wBACzB,OAAO,EACL,qEAAqE;qBACxE;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,GAMN;YACF,OAAO,EAAE;gBACP,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;gBACxB,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrE;SACF,CAAC;QACF,IAAI,MAAM,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAC9C,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC;QACrD,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC9D,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,UAAU,IAAI,MAAM,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBAC1B,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;gBACrB,SAAS,EAAE,IAAI;gBACf,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IACzD,CAAC,CAAC;IAEF,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,CAC9C,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,CAC7B,CAAC;IAEF,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,CAC3E,YAAY,CAAC,OAAO,EAAE,KAAK,EAAE,2BAA2B,EAAE,IAAI,CAAC,CAChE,CAAC;AACJ,CAAC"}

package/dist/src/storage/rebuild.d.ts

@@ -1,6 +1,18 @@
 import type { Config } from "../config/schema.js";
 import type { EmbeddingProvider } from "../providers/types.js";
 import type { TableName } from "./tables.js";
+/**
+ * Shared duplicate-resolution policy for conversation raw files.
+ *
+ * Both repair paths — reconcile and full rebuild — must pick the SAME winner
+ * when several raw files compete for one row id or idempotency_key, otherwise
+ * the canonical conversation row could flip depending on which tool last ran,
+ * which is corrosive to trust and debugging. Policy: most-recent raw wins,
+ * ordered by filename (`<timestamp>_<id>.json`, ISO-8601-derived → lexicographic
+ * == chronological). Compare basenames so the ordering is stable regardless of
+ * which source folder a file lives in. Sorts most-recent FIRST (descending).
+ */
+export declare function compareRawRecencyDesc(aPath: string, bPath: string): number;
 export interface RebuildDeps {
     config: Config;
     embeddingProvider: EmbeddingProvider;
@@ -11,6 +23,15 @@ export interface RebuildDeps {
 export interface RebuildReport {
     tables_rebuilt: TableName[];
     rows_reindexed: number;
+    /**
+     * Conversation raw files skipped because a row with the same non-empty
+     * `idempotency_key` was already inserted into the rebuilt index under a
+     * different row id. Mirrors reconcile's `missing_skipped_duplicate_key`:
+     * without it, an orphan raw left on disk (e.g. one reconcile already
+     * skipped) would resurface as a duplicate every time the index is rebuilt
+     * from raw — rebuild groups by row id and would otherwise miss the dup.
+     */
+    conversations_skipped_duplicate_key: number;
 }
 /**
  * Full rebuild from raw disk state via shadow-table swap.
@@ -26,7 +47,20 @@ export interface ReembedOptions {
     config?: Config;
     localEmbeddingProvider?: EmbeddingProvider;
 }
-export declare function reembed(embeddingProvider: EmbeddingProvider, scope: ReindexScope, config?: Config, localEmbeddingProvider?: EmbeddingProvider): Promise<number>;
+export declare function reembed(embeddingProvider: EmbeddingProvider, scope: ReindexScope, dataDir: string, config?: Config, localEmbeddingProvider?: EmbeddingProvider): Promise<number>;
+/**
+ * Remove structured-doc rows whose source file no longer exists on disk.
+ * Scoped to filesystem-sourced rows (watched + scheduled_scan).
+ *
+ * Why: deletions between scheduled scans (or while a watcher is offline)
+ * leave ghost rows in the index, so search can return content that no
+ * longer exists. This pass closes that gap.
+ *
+ * Safety: skips rows whose owning source root is unreachable, so an
+ * unmounted drive cannot trigger mass deletion. If `config` is omitted
+ * the function still runs, but without the unmounted-root guard.
+ */
+export declare function removeStructuredDocOrphans(dataDir: string, config?: Config): Promise<number>;
 export declare function extractIdFromFilename(filename: string): string | undefined;
 /**
  * Extract the ISO timestamp from a raw conversation filename.

package/dist/src/storage/rebuild.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rebuild.d.ts","sourceRoot":"","sources":["../../../src/storage/rebuild.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAM/D,OAAO,KAAK,EAIV,SAAS,EACV,MAAM,aAAa,CAAC;AAMrB,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,sBAAsB,CAAC,EAAE,iBAAiB,CAAC;CAC5C;AAED,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,SAAS,EAAE,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAoB3E;AAuMD,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,eAAe,GAAG,WAAW,GAAG,iBAAiB,CAAC;AAErF,MAAM,WAAW,cAAc;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,uDAAuD;IACvD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sBAAsB,CAAC,EAAE,iBAAiB,CAAC;CAC5C;AAED,wBAAsB,OAAO,CAC3B,iBAAiB,EAAE,iBAAiB,EACpC,KAAK,EAAE,YAAY,EACnB,MAAM,CAAC,EAAE,MAAM,EACf,sBAAsB,CAAC,EAAE,iBAAiB,GACzC,OAAO,CAAC,MAAM,CAAC,CAcjB;AA4JD,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAG1E;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAa/E"}
+{"version":3,"file":"rebuild.d.ts","sourceRoot":"","sources":["../../../src/storage/rebuild.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAW/D,OAAO,KAAK,EAIV,SAAS,EACV,MAAM,aAAa,CAAC;AAWrB;;;;;;;;;;GAUG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE1E;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,sBAAsB,CAAC,EAAE,iBAAiB,CAAC;CAC5C;AAED,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,SAAS,EAAE,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB;;;;;;;OAOG;IACH,mCAAmC,EAAE,MAAM,CAAC;CAC7C;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAgD3E;AA2TD,MAAM,MAAM,YAAY,GACpB,KAAK,GACL,eAAe,GACf,WAAW,GACX,iBAAiB,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,uDAAuD;IACvD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sBAAsB,CAAC,EAAE,iBAAiB,CAAC;CAC5C;AAED,wBAAsB,OAAO,CAC3B,iBAAiB,EAAE,iBAAiB,EACpC,KAAK,EAAE,YAAY,EACnB,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,MAAM,EACf,sBAAsB,CAAC,EAAE,iBAAiB,GACzC,OAAO,CAAC,MAAM,CAAC,CA2CjB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAoDjB;AAyPD,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAG1E;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,GACf,IAAI,GAAG,SAAS,CAclB"}

package/dist/src/storage/rebuild.js

@@ -9,12 +9,30 @@
 import fs from "node:fs";
 import path from "node:path";
 import { CONVERSATION_SOURCES } from "../providers/types.js";
+import { RIFT_NONE_EMBEDDING_PROVIDER, isPlaceholderEmbeddingProvider, } from "../providers/placeholder-embed.js";
 import { walkDirectory } from "../ingestion/scanner.js";
 import { isSupported } from "../ingestion/extractor.js";
 import { buildIgnoredPaths } from "../ingestion/ignored-paths.js";
 import { fileId } from "../ingestion/indexer.js";
+import { conversationContentFingerprint } from "../ingestion/inbox-core/conversation-fingerprint.js";
 import { createShadowSwap } from "./shadow-swap.js";
+import { recordEmbed } from "../observability/embedding-events.js";
+import { appendIndexEvent, recordIndexWrite, } from "../observability/index-events.js";
 const ALL_CONV_SOURCES = [...CONVERSATION_SOURCES, "inbox"];
+/**
+ * Shared duplicate-resolution policy for conversation raw files.
+ *
+ * Both repair paths — reconcile and full rebuild — must pick the SAME winner
+ * when several raw files compete for one row id or idempotency_key, otherwise
+ * the canonical conversation row could flip depending on which tool last ran,
+ * which is corrosive to trust and debugging. Policy: most-recent raw wins,
+ * ordered by filename (`<timestamp>_<id>.json`, ISO-8601-derived → lexicographic
+ * == chronological). Compare basenames so the ordering is stable regardless of
+ * which source folder a file lives in. Sorts most-recent FIRST (descending).
+ */
+export function compareRawRecencyDesc(aPath, bPath) {
+    return path.basename(bPath).localeCompare(path.basename(aPath));
+}
 /**
  * Full rebuild from raw disk state via shadow-table swap.
  * Creates shadow tables for all four tables, populates them from
@@ -22,19 +40,45 @@ const ALL_CONV_SOURCES = [...CONVERSATION_SOURCES, "inbox"];
  */
 export async function fullRebuild(deps) {
     const swap = createShadowSwap(deps.dataDir);
-    const report = { tables_rebuilt: [], rows_reindexed: 0 };
+    const report = {
+        tables_rebuilt: [],
+        rows_reindexed: 0,
+        conversations_skipped_duplicate_key: 0,
+    };
     const swaps = new Map();
-    await populateStructuredDocsShadows(deps, swap, swaps, report);
-    await populateConversationShadows(deps, swap, swaps, report);
-    await populateDigestShadow(deps, swap, swaps, report);
+    // Per-table populate counts. Used to emit one post-commit "live"
+    // index-event per table so `index.last_update_at` is only credited
+    // after the swap actually installs the shadows. If `commitAllSwaps`
+    // throws, we never emit the live events and the lane correctly
+    // shows shadow-only successes (which `computeIndexHealth` ignores).
+    const populated = new Map();
+    await populateStructuredDocsShadows(deps, swap, swaps, populated, report);
+    await populateConversationShadows(deps, swap, swaps, populated, report);
+    await populateDigestShadow(deps, swap, swaps, populated, report);
     await swap.commitAllSwaps(swaps);
+    // Post-commit: credit the lane with one "live" event per swapped
+    // table. The shadow events stay in the lane for debugging but do
+    // not bump `last_update_at` (see computeIndexHealth).
+    const commitTs = new Date().toISOString();
+    for (const [tableName, rowCount] of populated) {
+        appendIndexEvent(deps.dataDir, {
+            ts: commitTs,
+            table: tableName,
+            pipeline: "backfill",
+            operation: "shadow_swap_commit",
+            outcome: "success",
+            row_count: rowCount,
+            ms: 0,
+            phase: "live",
+        });
+    }
     report.tables_rebuilt.push("structured_docs", "structured_docs_local", "conversations_hot", "conversations_cold", "digests");
     return report;
 }
 // ---------------------------------------------------------------------------
 // Populate shadow tables from raw state
 // ---------------------------------------------------------------------------
-async function populateStructuredDocsShadows(deps, swap, swaps, report) {
+async function populateStructuredDocsShadows(deps, swap, swaps, populated, report) {
     const cloudShadow = await swap.createShadow("structured_docs");
     const localShadow = await swap.createShadow("structured_docs_local");
     swaps.set("structured_docs", cloudShadow);
@@ -75,7 +119,11 @@ async function populateStructuredDocsShadows(deps, swap, swaps, report) {
             const targetShadow = isLocal ? localShadow : cloudShadow;
             try {
                 const content = fs.readFileSync(canonical, "utf-8");
-                const embedding = await provider.embed(content);
+                const embedding = await recordEmbed(deps.dataDir, provider, {
+                    pipeline: "backfill",
+                    operation: "structured_doc_embedding",
+                    input_count: 1,
+                }, () => provider.embed(content));
                 const row = {
                     id,
                     source_path: canonical,
@@ -89,7 +137,17 @@ async function populateStructuredDocsShadows(deps, swap, swaps, report) {
                     indexed_at: new Date().toISOString(),
                     metadata: "{}",
                 };
-                await targetShadow.add([row]);
+                const targetTableName = isLocal
+                    ? "structured_docs_local"
+                    : "structured_docs";
+                await recordIndexWrite(deps.dataDir, {
+                    table: targetTableName,
+                    pipeline: "backfill",
+                    operation: "structured_doc_upsert",
+                    row_count: 1,
+                    phase: "shadow",
+                }, () => targetShadow.add([row]));
+                populated.set(targetTableName, (populated.get(targetTableName) ?? 0) + 1);
                 report.rows_reindexed++;
             }
             catch (err) {
@@ -98,60 +156,112 @@ async function populateStructuredDocsShadows(deps, swap, swaps, report) {
         }
     }
 }
-async function populateConversationShadows(deps, swap, swaps, report) {
+async function populateConversationShadows(deps, swap, swaps, populated, report) {
     const hotShadow = await swap.createShadow("conversations_hot");
     const coldShadow = await swap.createShadow("conversations_cold");
     swaps.set("conversations_hot", hotShadow);
     swaps.set("conversations_cold", coldShadow);
     const cutoff = new Date();
     cutoff.setDate(cutoff.getDate() - (deps.config.compaction.age_threshold_days ?? 30));
+    // Durable dedup across the whole rebuild: a non-empty idempotency_key may
+    // only be inserted once. Mirrors reconcile's key-skip so a raw file that
+    // reconcile already declined to revive cannot reappear as a duplicate here.
+    const seenKeys = new Set();
+    // Group raw files by stable row id ACROSS ALL SOURCES first. Refresh-on-grow
+    // may leave more than one raw artifact for the same id on disk if pruning was
+    // interrupted; reinserting all of them duplicates the conversation. Keep only
+    // the most-recent raw per id (shared recency policy). Collecting globally —
+    // rather than per source folder — matters because `seenKeys` is global: if we
+    // sorted within each source folder, an older raw in an earlier folder could
+    // claim a key before a newer same-key raw in a later folder was ever seen.
+    const latestByRowId = new Map();
     for (const source of ALL_CONV_SOURCES) {
         const rawDir = path.join(deps.dataDir, "raw", "conversations", source);
         if (!fs.existsSync(rawDir))
             continue;
-        const files = fs.readdirSync(rawDir).filter((f) => f.endsWith(".json"));
-        for (const file of files) {
+        for (const file of fs.readdirSync(rawDir)) {
+            if (!file.endsWith(".json"))
+                continue;
             const id = extractIdFromFilename(file);
             if (!id)
                 continue;
             const rawPath = path.join(rawDir, file);
-            try {
-                const raw = JSON.parse(fs.readFileSync(rawPath, "utf-8"));
-                const summary = raw.summary ?? "";
-                const content = raw.content ?? summary;
-                const vec = await deps.embeddingProvider.embed(content || summary);
-                const row = {
-                    id,
-                    content,
-                    summary,
-                    embedding: vec,
-                    source: raw.source ?? "claude_code",
-                    domain: raw.domain ?? "tech",
-                    intent: raw.intent ?? "build",
-                    quality: raw.quality ?? "medium",
-                    topics: JSON.stringify(raw.topics ?? []),
-                    decisions: JSON.stringify(raw.decisions ?? []),
-                    key_outputs: JSON.stringify(raw.key_outputs ?? []),
-                    indexed_at: new Date().toISOString(),
-                    idempotency_key: raw.idempotency_key ?? "",
-                };
-                const filenameTime = extractTimestampFromFilename(file);
-                const rowTime = filenameTime ?? fs.statSync(rawPath).mtime;
-                if (rowTime >= cutoff) {
-                    await hotShadow.add([row]);
-                }
-                else {
-                    await coldShadow.add([row]);
-                }
-                report.rows_reindexed++;
+            const prior = latestByRowId.get(id);
+            if (!prior || compareRawRecencyDesc(rawPath, prior) < 0) {
+                latestByRowId.set(id, rawPath);
             }
-            catch (err) {
-                process.stderr.write(`rebuild: failed to index conversation ${id}: ${err instanceof Error ? err.message : String(err)}\n`);
+        }
+    }
+    // Process most-recent raw first (global order) so that when two row ids share
+    // one idempotency_key the newest wins and older duplicates are skipped —
+    // identical winner to reconcile via the shared recency comparator.
+    const ordered = [...latestByRowId.entries()].sort(([, a], [, b]) => compareRawRecencyDesc(a, b));
+    for (const [id, rawPath] of ordered) {
+        const file = path.basename(rawPath);
+        try {
+            const raw = JSON.parse(fs.readFileSync(rawPath, "utf-8"));
+            const key = typeof raw.idempotency_key === "string" ? raw.idempotency_key : "";
+            if (key && seenKeys.has(key)) {
+                report.conversations_skipped_duplicate_key++;
+                continue;
             }
+            const summary = raw.summary ?? "";
+            const content = raw.content ?? summary;
+            const vec = await recordEmbed(deps.dataDir, deps.embeddingProvider, {
+                pipeline: "backfill",
+                operation: "conversation_embedding",
+                input_count: 1,
+            }, () => deps.embeddingProvider.embed(content || summary));
+            const fingerprint = typeof raw.conversation_fingerprint === "string" &&
+                raw.conversation_fingerprint.length > 0
+                ? raw.conversation_fingerprint
+                : content
+                    ? conversationContentFingerprint(content)
+                    : "";
+            const row = {
+                id,
+                content,
+                summary,
+                embedding: vec,
+                source: raw.source ?? "claude_code",
+                domain: raw.domain ?? "tech",
+                intent: raw.intent ?? "build",
+                quality: raw.quality ?? "medium",
+                topics: JSON.stringify(raw.topics ?? []),
+                decisions: JSON.stringify(raw.decisions ?? []),
+                key_outputs: JSON.stringify(raw.key_outputs ?? []),
+                indexed_at: new Date().toISOString(),
+                idempotency_key: raw.idempotency_key ?? "",
+                conversation_fingerprint: fingerprint,
+                // Metadata restored from raw → preserve its provider marker. Embedding
+                // recomputed here → marker reflects the provider that just ran.
+                metadata_provider: typeof raw.metadata_provider === "string" ? raw.metadata_provider : "",
+                embedding_provider: isPlaceholderEmbeddingProvider(deps.embeddingProvider)
+                    ? RIFT_NONE_EMBEDDING_PROVIDER
+                    : "",
+            };
+            const filenameTime = extractTimestampFromFilename(file);
+            const rowTime = filenameTime ?? fs.statSync(rawPath).mtime;
+            const targetTable = rowTime >= cutoff ? "conversations_hot" : "conversations_cold";
+            const targetShadow = rowTime >= cutoff ? hotShadow : coldShadow;
+            await recordIndexWrite(deps.dataDir, {
+                table: targetTable,
+                pipeline: "backfill",
+                operation: "conversation_upsert",
+                row_count: 1,
+                phase: "shadow",
+            }, () => targetShadow.add([row]));
+            if (key)
+                seenKeys.add(key);
+            populated.set(targetTable, (populated.get(targetTable) ?? 0) + 1);
+            report.rows_reindexed++;
+        }
+        catch (err) {
+            process.stderr.write(`rebuild: failed to index conversation ${id}: ${err instanceof Error ? err.message : String(err)}\n`);
         }
     }
 }
-async function populateDigestShadow(deps, swap, swaps, report) {
+async function populateDigestShadow(deps, swap, swaps, populated, report) {
     const shadow = await swap.createShadow("digests");
     swaps.set("digests", shadow);
     const digestDir = path.join(deps.dataDir, "raw", "digests");
@@ -164,7 +274,11 @@ async function populateDigestShadow(deps, swap, swaps, report) {
                 if (typeof raw.id !== "string" || !raw.id)
                     continue;
                 const summary = raw.summary ?? raw.content ?? "";
-                const vec = await deps.embeddingProvider.embed(summary);
+                const vec = await recordEmbed(deps.dataDir, deps.embeddingProvider, {
+                    pipeline: "backfill",
+                    operation: "digest_embedding",
+                    input_count: 1,
+                }, () => deps.embeddingProvider.embed(summary));
                 const row = {
                     id: raw.id,
                     content: raw.content ?? "",
@@ -175,7 +289,14 @@ async function populateDigestShadow(deps, swap, swaps, report) {
                     digest_type: raw.digest_type ?? "weekly",
                     created_at: raw.created_at ?? new Date().toISOString(),
                 };
-                await shadow.add([row]);
+                await recordIndexWrite(deps.dataDir, {
+                    table: "digests",
+                    pipeline: "backfill",
+                    operation: "digest_upsert",
+                    row_count: 1,
+                    phase: "shadow",
+                }, () => shadow.add([row]));
+                populated.set("digests", (populated.get("digests") ?? 0) + 1);
                 report.rows_reindexed++;
             }
             catch (err) {
@@ -184,27 +305,86 @@ async function populateDigestShadow(deps, swap, swaps, report) {
         }
     }
 }
-export async function reembed(embeddingProvider, scope, config, localEmbeddingProvider) {
+export async function reembed(embeddingProvider, scope, dataDir, config, localEmbeddingProvider) {
     let count = 0;
     if (scope === "all" || scope === "structured_docs" || scope === "documents") {
-        count += await reembedStructuredDocs(embeddingProvider, config, localEmbeddingProvider);
+        count += await reembedStructuredDocs(embeddingProvider, dataDir, config, localEmbeddingProvider);
         if (localEmbeddingProvider) {
-            count += await reembedLocalStructuredDocs(localEmbeddingProvider);
+            count += await reembedLocalStructuredDocs(localEmbeddingProvider, dataDir);
         }
     }
     if (scope === "all" || scope === "conversations") {
-        count += await reembedTable("conversations_hot", embeddingProvider, (r) => r.content || r.summary, "indexed_at");
-        count += await reembedTable("conversations_cold", embeddingProvider, (r) => r.content || r.summary, "indexed_at");
-        count += await reembedTable("digests", embeddingProvider, (r) => r.summary || r.content, "created_at");
+        count += await reembedTable("conversations_hot", embeddingProvider, dataDir, "conversation_embedding", (r) => r.content || r.summary, "indexed_at");
+        count += await reembedTable("conversations_cold", embeddingProvider, dataDir, "conversation_embedding", (r) => r.content || r.summary, "indexed_at");
+        count += await reembedTable("digests", embeddingProvider, dataDir, "digest_embedding", (r) => r.summary || r.content, "created_at");
     }
     return count;
 }
+/**
+ * Remove structured-doc rows whose source file no longer exists on disk.
+ * Scoped to filesystem-sourced rows (watched + scheduled_scan).
+ *
+ * Why: deletions between scheduled scans (or while a watcher is offline)
+ * leave ghost rows in the index, so search can return content that no
+ * longer exists. This pass closes that gap.
+ *
+ * Safety: skips rows whose owning source root is unreachable, so an
+ * unmounted drive cannot trigger mass deletion. If `config` is omitted
+ * the function still runs, but without the unmounted-root guard.
+ */
+export async function removeStructuredDocOrphans(dataDir, config) {
+    const { getTable } = await import("./tables.js");
+    const tableNames = ["structured_docs", "structured_docs_local"];
+    let removed = 0;
+    for (const tableName of tableNames) {
+        let table;
+        try {
+            table = getTable(tableName);
+        }
+        catch {
+            continue;
+        }
+        const rows = (await table.query().toArray());
+        const rootReachable = new Map();
+        const isReachable = (root) => {
+            const cached = rootReachable.get(root);
+            if (cached !== undefined)
+                return cached;
+            const ok = fs.existsSync(root);
+            rootReachable.set(root, ok);
+            return ok;
+        };
+        for (const row of rows) {
+            if (row.source_type !== "filesystem_watched" &&
+                row.source_type !== "filesystem_scheduled_scan")
+                continue;
+            if (config) {
+                const owner = findOwningSource(row.source_path, config);
+                if (!owner)
+                    continue;
+                if (!isReachable(owner.path))
+                    continue;
+            }
+            if (fs.existsSync(row.source_path))
+                continue;
+            try {
+                await table.delete(`id = '${row.id}'`);
+                removed++;
+                process.stderr.write(`reindex: removed orphan ${tableName} row ${row.id} (source missing: ${row.source_path})\n`);
+            }
+            catch (err) {
+                process.stderr.write(`reindex: failed to remove orphan ${row.id} in ${tableName}: ${err instanceof Error ? err.message : String(err)}\n`);
+            }
+        }
+    }
+    return removed;
+}
 /**
  * Re-embed structured_docs with per-source provider routing.
  * Rows whose source_path falls under a local-extraction source use
  * localEmbeddingProvider; everything else uses the cloud provider.
  */
-async function reembedStructuredDocs(cloudProvider, config, localProvider) {
+async function reembedStructuredDocs(cloudProvider, dataDir, config, localProvider) {
     const { getTable } = await import("./tables.js");
     const table = getTable("structured_docs");
     const allRows = (await table.query().toArray());
@@ -212,10 +392,21 @@ async function reembedStructuredDocs(cloudProvider, config, localProvider) {
     for (const row of allRows) {
         try {
             const provider = resolveProviderForPath(row.source_path, cloudProvider, config, localProvider);
-            const newEmbedding = await provider.embed(row.content);
+            const newEmbedding = await recordEmbed(dataDir, provider, {
+                pipeline: "reindex",
+                operation: "structured_doc_embedding",
+                input_count: 1,
+            }, () => provider.embed(row.content));
             const oldTimestamp = row.indexed_at;
             const newTimestamp = new Date().toISOString();
-            await table.add([{ ...row, embedding: newEmbedding, indexed_at: newTimestamp }]);
+            await recordIndexWrite(dataDir, {
+                table: "structured_docs",
+                pipeline: "reindex",
+                operation: "structured_doc_upsert",
+                row_count: 1,
+            }, () => table.add([
+                { ...row, embedding: newEmbedding, indexed_at: newTimestamp },
+            ]));
             try {
                 await table.delete(`id = '${row.id}' AND indexed_at = '${oldTimestamp}'`);
                 count++;
@@ -224,7 +415,9 @@ async function reembedStructuredDocs(cloudProvider, config, localProvider) {
                 try {
                     await table.delete(`id = '${row.id}' AND indexed_at = '${newTimestamp}'`);
                 }
-                catch { /* duplicate remains — reconcile will repair */ }
+                catch {
+                    /* duplicate remains — reconcile will repair */
+                }
             }
         }
         catch (err) {
@@ -237,17 +430,28 @@ async function reembedStructuredDocs(cloudProvider, config, localProvider) {
  * Re-embed all rows in structured_docs_local using the local provider.
  * Simpler than the cloud path: all rows use the same provider.
  */
-async function reembedLocalStructuredDocs(localProvider) {
+async function reembedLocalStructuredDocs(localProvider, dataDir) {
     const { getTable } = await import("./tables.js");
     const table = getTable("structured_docs_local");
     const allRows = (await table.query().toArray());
     let count = 0;
     for (const row of allRows) {
         try {
-            const newEmbedding = await localProvider.embed(row.content);
+            const newEmbedding = await recordEmbed(dataDir, localProvider, {
+                pipeline: "reindex",
+                operation: "structured_doc_local_embedding",
+                input_count: 1,
+            }, () => localProvider.embed(row.content));
             const oldTimestamp = row.indexed_at;
             const newTimestamp = new Date().toISOString();
-            await table.add([{ ...row, embedding: newEmbedding, indexed_at: newTimestamp }]);
+            await recordIndexWrite(dataDir, {
+                table: "structured_docs_local",
+                pipeline: "reindex",
+                operation: "structured_doc_local_upsert",
+                row_count: 1,
+            }, () => table.add([
+                { ...row, embedding: newEmbedding, indexed_at: newTimestamp },
+            ]));
             try {
                 await table.delete(`id = '${row.id}' AND indexed_at = '${oldTimestamp}'`);
                 count++;
@@ -256,7 +460,9 @@ async function reembedLocalStructuredDocs(localProvider) {
                 try {
                     await table.delete(`id = '${row.id}' AND indexed_at = '${newTimestamp}'`);
                 }
-                catch { /* duplicate remains — reconcile will repair */ }
+                catch {
+                    /* duplicate remains — reconcile will repair */
+                }
             }
         }
         catch (err) {
@@ -278,22 +484,31 @@ function resolveProviderForPath(sourcePath, cloudProvider, config, localProvider
         return localProvider;
     return cloudProvider;
 }
+/**
+ * True when `file` is the same as `root` or lives beneath it. Uses
+ * `path.relative` so sibling paths that merely share a string prefix
+ * (e.g. `/docs/app-old` vs root `/docs/app`) are correctly rejected.
+ */
+function isPathWithin(file, root) {
+    const rel = path.relative(root, file);
+    return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
+}
 /**
  * Find the most-specific source root that owns a given file path.
- * Longest prefix wins, so nested roots are correctly attributed.
+ * Longest matching root wins, so nested roots are correctly attributed.
  */
 function findOwningSource(filePath, config) {
     let best;
     let bestLen = 0;
     for (const s of config.sources) {
-        if (filePath.startsWith(s.path) && s.path.length > bestLen) {
+        if (isPathWithin(filePath, s.path) && s.path.length > bestLen) {
             best = s;
             bestLen = s.path.length;
         }
     }
     return best;
 }
-async function reembedTable(tableName, embeddingProvider, getText, timestampField) {
+async function reembedTable(tableName, embeddingProvider, dataDir, operation, getText, timestampField) {
     const { getTable } = await import("./tables.js");
     const table = getTable(tableName);
     const allRows = (await table.query().toArray());
@@ -301,10 +516,17 @@ async function reembedTable(tableName, embeddingProvider, getText, timestampFiel
     for (const row of allRows) {
         try {
             const text = getText(row);
-            const newEmbedding = await embeddingProvider.embed(text);
+            const newEmbedding = await recordEmbed(dataDir, embeddingProvider, { pipeline: "reindex", operation, input_count: 1 }, () => embeddingProvider.embed(text));
             const oldTimestamp = row[timestampField];
             const newTimestamp = new Date().toISOString();
-            await table.add([{ ...row, embedding: newEmbedding, [timestampField]: newTimestamp }]);
+            await recordIndexWrite(dataDir, {
+                table: tableName,
+                pipeline: "reindex",
+                operation: `${operation.replace(/_embedding$/, "")}_upsert`,
+                row_count: 1,
+            }, () => table.add([
+                { ...row, embedding: newEmbedding, [timestampField]: newTimestamp },
+            ]));
             try {
                 await table.delete(`id = '${row.id}' AND ${timestampField} = '${oldTimestamp}'`);
                 count++;
@@ -313,7 +535,9 @@ async function reembedTable(tableName, embeddingProvider, getText, timestampFiel
                 try {
                     await table.delete(`id = '${row.id}' AND ${timestampField} = '${newTimestamp}'`);
                 }
-                catch { /* duplicate remains — reconcile will repair */ }
+                catch {
+                    /* duplicate remains — reconcile will repair */
+                }
             }
         }
         catch (err) {