@getrift/rift 0.1.0-beta.21 → 0.1.0-beta.23

package/dist/src/server/serving-marker.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Explicit capability flags. Capability booleans — not a commit/version string —
+ * are the contract: `package.json` still reads `beta.21` on chunk-aware `main`,
+ * so a version string cannot distinguish builds. Each build declares what IT can
+ * do; a future build that drops a capability flips the flag rather than relying
+ * on commit-ancestry math.
+ */
+export interface ServingCapabilities {
+    /** The conversation tables carry `parent_id`/`chunk_index`/`chunk_count`. */
+    conversation_chunk_columns: boolean;
+    /** Retrieval reassembles chunk sets on the expand path (`detail="full"`). */
+    expand_stitching: boolean;
+    /** Safe for `chunk-backfill` to write a chunk-column conversation set here. */
+    chunk_backfill_write_compatible: boolean;
+}
+export interface ServingMarker {
+    version: string;
+    commit: string;
+    booted_at: string;
+    capabilities: ServingCapabilities;
+}
+/**
+ * Capabilities of THIS build. All true: this code defines the chunk columns in
+ * the conversation seed/migration, reassembles chunk sets on expand, and writes
+ * chunk-column sets coherently. A future build that regresses any of these must
+ * set the corresponding flag false.
+ */
+export declare const CURRENT_SERVING_CAPABILITIES: ServingCapabilities;
+/** Path of the marker within a data dir (under `observability/`, daemon-owned). */
+export declare function servingMarkerPath(dataDir: string): string;
+/** Assemble the marker for this build at boot time. */
+export declare function buildServingMarker(bootedAt: string): ServingMarker;
+/**
+ * Stamp the serving-build marker into the data dir. Atomic (write-temp +
+ * rename) so a crash mid-write never leaves a half-written marker that would
+ * read as malformed → refused. Best-effort: any failure is logged and
+ * swallowed so it can never break daemon boot.
+ *
+ * MUST be called only from the serving daemon boot path, never from a CLI tool.
+ */
+export declare function writeServingMarker(dataDir: string, bootedAt?: string): void;
+/** Read the marker. Returns null if absent, unreadable, or malformed. */
+export declare function readServingMarker(dataDir: string): ServingMarker | null;
+export interface CompatibilityVerdict {
+    ok: boolean;
+    /** Operator-facing reason when `ok` is false. */
+    reason: string;
+}
+/**
+ * Maximum age of a serving-build marker the write guard will trust (24h). The
+ * marker certifies a RECENT boot, not an ancient one — see the downgrade note in
+ * the file header. The backfill tool requires the daemon stopped, so the owner
+ * naturally does a boot→stop cycle right before backfilling; that boot re-stamps
+ * the marker fresh. A window of a day gives slack ("booted this morning, backfill
+ * this evening") while still rejecting a months-old, possibly downgrade-era
+ * marker. Small future clock skew is tolerated up to {@link MAX_FUTURE_SKEW_MS}.
+ */
+export declare const MAX_MARKER_AGE_MS: number;
+/** Clock-skew slack for a marker dated slightly in the future (5 min). */
+export declare const MAX_FUTURE_SKEW_MS: number;
+/**
+ * Decide whether a destructive `chunk-backfill` WRITE is safe against the
+ * install that serves `dataDir`. Refuses unless the serving daemon has stamped
+ * a well-formed, RECENT marker in which EVERY capability in
+ * {@link REQUIRED_WRITE_CAPABILITIES} is true.
+ *
+ * Refusal cases:
+ *   - Absent/malformed marker: a pre-chunk daemon never writes one, so we cannot
+ *     prove the install that restarts after the write can serve chunk rows.
+ *   - Unparseable `booted_at`: a marker without a real boot time cannot prove
+ *     freshness, so it is treated as malformed.
+ *   - Missing capability: the serving build self-declares it cannot serve a
+ *     chunk-column set.
+ *   - Stale `booted_at` (older than `maxAgeMs`): proves only that SOME chunk-aware
+ *     daemon booted this dir long ago, not that the CURRENT install is chunk-aware
+ *     (closes the silent-downgrade gap — see file header).
+ *
+ * `now`/`maxAgeMs` are injectable for deterministic tests; production callers use
+ * the wall clock and {@link MAX_MARKER_AGE_MS}.
+ */
+export declare function assertChunkBackfillWriteCompatible(dataDir: string, opts?: {
+    now?: Date;
+    maxAgeMs?: number;
+}): CompatibilityVerdict;
+//# sourceMappingURL=serving-marker.d.ts.map

package/dist/src/server/serving-marker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"serving-marker.d.ts","sourceRoot":"","sources":["../../../src/server/serving-marker.ts"],"names":[],"mappings":"AA8CA;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,6EAA6E;IAC7E,0BAA0B,EAAE,OAAO,CAAC;IACpC,6EAA6E;IAC7E,gBAAgB,EAAE,OAAO,CAAC;IAC1B,+EAA+E;IAC/E,+BAA+B,EAAE,OAAO,CAAC;CAC1C;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,mBAAmB,CAAC;CACnC;AAED;;;;;GAKG;AACH,eAAO,MAAM,4BAA4B,EAAE,mBAI1C,CAAC;AAEF,mFAAmF;AACnF,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,uDAAuD;AACvD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,CAQlE;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,MAAM,EACf,QAAQ,GAAE,MAAiC,GAC1C,IAAI,CAcN;AAsBD,yEAAyE;AACzE,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAQvE;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,OAAO,CAAC;IACZ,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;CAChB;AAgBD;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,QAAsB,CAAC;AAErD,0EAA0E;AAC1E,eAAO,MAAM,kBAAkB,QAAgB,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,kCAAkC,CAChD,OAAO,EAAE,MAAM,EACf,IAAI,GAAE;IAAE,GAAG,CAAC,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAC3C,oBAAoB,CAoEtB"}

package/dist/src/server/serving-marker.js

@@ -0,0 +1,226 @@
+/**
+ * Serving-build capability marker.
+ *
+ * The serving daemon stamps a small JSON marker into the data dir on every
+ * boot, recording WHICH code last served this data dir and what that code is
+ * capable of. This is the contract that lets an OFFLINE tool — one that runs
+ * with the daemon stopped, so it cannot probe `GET /version` — decide whether a
+ * destructive, schema-touching operation is safe against the install that will
+ * restart and serve this dir.
+ *
+ * Why a capability marker and not the obvious alternatives:
+ *   - On-disk table schema is the very thing that gets accidentally mutated
+ *     (the chunk-backfill `--dry-run` incident added the chunk columns to a
+ *     pre-chunk dir), so the schema cannot certify the serving code.
+ *   - Observability events are an indirect history signal — they tell you what
+ *     happened, not what the currently-installed code can do.
+ *   - `RIFT_CHUNKING` gates whether NEW conversations are split, not whether the
+ *     permanent chunk columns / expand read path exist in the serving build.
+ * Only a marker written BY the serving build answers the question we actually
+ * care about: "what code last served this dir, and can it handle a
+ * chunk-backfill write?".
+ *
+ * Discipline:
+ *   - Written ONLY by the serving daemon (never by CLI tools). A CLI tool that
+ *     stamped the marker would be certifying itself, defeating the purpose.
+ *   - Best-effort and non-fatal — a failed marker write must never break boot.
+ *   - Absent / unknown / malformed is treated by the write guard as "cannot
+ *     confirm compatible → refuse". A pre-chunk daemon (e.g. beta.21) never
+ *     writes the marker, so it always reads as absent → refused.
+ *   - STALE markers are refused too (freshness window, see
+ *     {@link MAX_MARKER_AGE_MS}). A marker proves "a chunk-aware daemon booted
+ *     this dir RECENTLY", not "a chunk-aware daemon booted this dir once, ever".
+ *     This closes the downgrade gap: if a chunk-aware daemon stamps a marker and
+ *     a pre-chunk daemon (beta.21) later serves the same dir, beta.21 neither
+ *     overwrites nor removes the stale chunk-aware marker — so without a
+ *     freshness check the offline write guard would still pass against an install
+ *     that can no longer serve chunk rows. Requiring a recent boot means the
+ *     owner must (re)boot the chunk-aware daemon immediately before backfilling,
+ *     which re-stamps the marker with the CURRENT install's real capabilities.
+ *     Daemon DOWNGRADE (chunk-aware → pre-chunk) after a marker write is
+ *     otherwise unsupported in this owner-only phase.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { getBuildInfo } from "./build-info.js";
+/**
+ * Capabilities of THIS build. All true: this code defines the chunk columns in
+ * the conversation seed/migration, reassembles chunk sets on expand, and writes
+ * chunk-column sets coherently. A future build that regresses any of these must
+ * set the corresponding flag false.
+ */
+export const CURRENT_SERVING_CAPABILITIES = {
+    conversation_chunk_columns: true,
+    expand_stitching: true,
+    chunk_backfill_write_compatible: true,
+};
+/** Path of the marker within a data dir (under `observability/`, daemon-owned). */
+export function servingMarkerPath(dataDir) {
+    return path.join(dataDir, "observability", "serving-build.json");
+}
+/** Assemble the marker for this build at boot time. */
+export function buildServingMarker(bootedAt) {
+    const info = getBuildInfo();
+    return {
+        version: info.version,
+        commit: info.commit,
+        booted_at: bootedAt,
+        capabilities: { ...CURRENT_SERVING_CAPABILITIES },
+    };
+}
+/**
+ * Stamp the serving-build marker into the data dir. Atomic (write-temp +
+ * rename) so a crash mid-write never leaves a half-written marker that would
+ * read as malformed → refused. Best-effort: any failure is logged and
+ * swallowed so it can never break daemon boot.
+ *
+ * MUST be called only from the serving daemon boot path, never from a CLI tool.
+ */
+export function writeServingMarker(dataDir, bootedAt = new Date().toISOString()) {
+    try {
+        const target = servingMarkerPath(dataDir);
+        fs.mkdirSync(path.dirname(target), { recursive: true });
+        const tmp = `${target}.tmp-${process.pid}`;
+        fs.writeFileSync(tmp, `${JSON.stringify(buildServingMarker(bootedAt), null, 2)}\n`);
+        fs.renameSync(tmp, target);
+    }
+    catch (err) {
+        process.stderr.write(`serving-marker: failed to write (non-fatal): ${err instanceof Error ? err.message : String(err)}\n`);
+    }
+}
+function isServingMarker(v) {
+    if (!v || typeof v !== "object")
+        return false;
+    const m = v;
+    if (typeof m.version !== "string" ||
+        typeof m.commit !== "string" ||
+        typeof m.booted_at !== "string") {
+        return false;
+    }
+    const c = m.capabilities;
+    if (!c || typeof c !== "object")
+        return false;
+    const caps = c;
+    return (typeof caps.conversation_chunk_columns === "boolean" &&
+        typeof caps.expand_stitching === "boolean" &&
+        typeof caps.chunk_backfill_write_compatible === "boolean");
+}
+/** Read the marker. Returns null if absent, unreadable, or malformed. */
+export function readServingMarker(dataDir) {
+    try {
+        const raw = fs.readFileSync(servingMarkerPath(dataDir), "utf8");
+        const parsed = JSON.parse(raw);
+        return isServingMarker(parsed) ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Every capability a chunk-backfill WRITE depends on. The write doesn't just
+ * need a serving build that *tolerates* a chunk-column write — it needs one that
+ * has the columns (`conversation_chunk_columns`), reassembles the sets it writes
+ * on the read path (`expand_stitching`), AND self-certifies the write as safe
+ * (`chunk_backfill_write_compatible`). A build that regresses ANY of these would
+ * serve the backfilled corpus incorrectly, so all must be true.
+ */
+const REQUIRED_WRITE_CAPABILITIES = [
+    "conversation_chunk_columns",
+    "expand_stitching",
+    "chunk_backfill_write_compatible",
+];
+/**
+ * Maximum age of a serving-build marker the write guard will trust (24h). The
+ * marker certifies a RECENT boot, not an ancient one — see the downgrade note in
+ * the file header. The backfill tool requires the daemon stopped, so the owner
+ * naturally does a boot→stop cycle right before backfilling; that boot re-stamps
+ * the marker fresh. A window of a day gives slack ("booted this morning, backfill
+ * this evening") while still rejecting a months-old, possibly downgrade-era
+ * marker. Small future clock skew is tolerated up to {@link MAX_FUTURE_SKEW_MS}.
+ */
+export const MAX_MARKER_AGE_MS = 24 * 60 * 60 * 1000;
+/** Clock-skew slack for a marker dated slightly in the future (5 min). */
+export const MAX_FUTURE_SKEW_MS = 5 * 60 * 1000;
+/**
+ * Decide whether a destructive `chunk-backfill` WRITE is safe against the
+ * install that serves `dataDir`. Refuses unless the serving daemon has stamped
+ * a well-formed, RECENT marker in which EVERY capability in
+ * {@link REQUIRED_WRITE_CAPABILITIES} is true.
+ *
+ * Refusal cases:
+ *   - Absent/malformed marker: a pre-chunk daemon never writes one, so we cannot
+ *     prove the install that restarts after the write can serve chunk rows.
+ *   - Unparseable `booted_at`: a marker without a real boot time cannot prove
+ *     freshness, so it is treated as malformed.
+ *   - Missing capability: the serving build self-declares it cannot serve a
+ *     chunk-column set.
+ *   - Stale `booted_at` (older than `maxAgeMs`): proves only that SOME chunk-aware
+ *     daemon booted this dir long ago, not that the CURRENT install is chunk-aware
+ *     (closes the silent-downgrade gap — see file header).
+ *
+ * `now`/`maxAgeMs` are injectable for deterministic tests; production callers use
+ * the wall clock and {@link MAX_MARKER_AGE_MS}.
+ */
+export function assertChunkBackfillWriteCompatible(dataDir, opts = {}) {
+    const now = opts.now ?? new Date();
+    const maxAgeMs = opts.maxAgeMs ?? MAX_MARKER_AGE_MS;
+    const marker = readServingMarker(dataDir);
+    if (!marker) {
+        return {
+            ok: false,
+            reason: `No serving-build marker at ${servingMarkerPath(dataDir)}. Cannot ` +
+                `confirm the install that serves this data dir is chunk-aware — a ` +
+                `pre-chunk daemon (e.g. beta.21) never writes this marker. Boot the ` +
+                `chunk-aware serving daemon on this dir first (it stamps the marker on ` +
+                `startup), or — if this is a DISPOSABLE COPY, never your live data dir — ` +
+                `re-run with --allow-uncertified-copy to override.`,
+        };
+    }
+    const missing = REQUIRED_WRITE_CAPABILITIES.filter((cap) => !marker.capabilities[cap]);
+    if (missing.length > 0) {
+        return {
+            ok: false,
+            reason: `Serving-build marker (version ${marker.version}, commit ` +
+                `${marker.commit.slice(0, 12)}) is missing required capabilities: ` +
+                `${missing.map((c) => `${c}=false`).join(", ")} — the serving install ` +
+                `cannot safely serve a chunk-column conversation set. Upgrade the ` +
+                `serving daemon before backfilling.`,
+        };
+    }
+    const bootedMs = Date.parse(marker.booted_at);
+    if (Number.isNaN(bootedMs)) {
+        return {
+            ok: false,
+            reason: `Serving-build marker has an invalid booted_at ("${marker.booted_at}") ` +
+                `— cannot prove the chunk-aware daemon booted recently. Reboot the ` +
+                `chunk-aware serving daemon on this dir to re-stamp the marker.`,
+        };
+    }
+    const ageMs = now.getTime() - bootedMs;
+    if (ageMs > maxAgeMs) {
+        const ageHours = Math.round(ageMs / 3_600_000);
+        const maxHours = Math.round(maxAgeMs / 3_600_000);
+        return {
+            ok: false,
+            reason: `Serving-build marker is stale (booted_at ${marker.booted_at}, ~${ageHours}h ` +
+                `old > ${maxHours}h freshness window). A stale marker only proves a ` +
+                `chunk-aware daemon booted this dir long ago — not that the CURRENT ` +
+                `install is chunk-aware (a since-downgraded daemon leaves the old marker ` +
+                `in place). (Re)boot the chunk-aware serving daemon on this dir ` +
+                `immediately before backfilling to re-stamp the marker, or — if this is ` +
+                `a DISPOSABLE COPY, never your live data dir — re-run with ` +
+                `--allow-uncertified-copy to override.`,
+        };
+    }
+    if (ageMs < -MAX_FUTURE_SKEW_MS) {
+        return {
+            ok: false,
+            reason: `Serving-build marker is dated in the future (booted_at ` +
+                `${marker.booted_at}) beyond tolerated clock skew — refusing rather than ` +
+                `trusting a marker that may have been hand-edited. Reboot the chunk-aware ` +
+                `serving daemon on this dir to re-stamp the marker.`,
+        };
+    }
+    return { ok: true, reason: "" };
+}
+//# sourceMappingURL=serving-marker.js.map

package/dist/src/server/serving-marker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"serving-marker.js","sourceRoot":"","sources":["../../../src/server/serving-marker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAyB/C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAwB;IAC/D,0BAA0B,EAAE,IAAI;IAChC,gBAAgB,EAAE,IAAI;IACtB,+BAA+B,EAAE,IAAI;CACtC,CAAC;AAEF,mFAAmF;AACnF,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,oBAAoB,CAAC,CAAC;AACnE,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,kBAAkB,CAAC,QAAgB;IACjD,MAAM,IAAI,GAAG,YAAY,EAAE,CAAC;IAC5B,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,SAAS,EAAE,QAAQ;QACnB,YAAY,EAAE,EAAE,GAAG,4BAA4B,EAAE;KAClD,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAChC,OAAe,EACf,WAAmB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IAE3C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAC1C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,MAAM,GAAG,GAAG,GAAG,MAAM,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;QAC3C,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACpF,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,gDACE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,IAAI,CACL,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,CAAU;IACjC,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9C,MAAM,CAAC,GAAG,CAA4B,CAAC;IACvC,IACE,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ;QAC7B,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ;QAC5B,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,EAC/B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC;IACzB,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9C,MAAM,IAAI,GAAG,CAA4B,CAAC;IAC1C,OAAO,CACL,OAAO,IAAI,CAAC,0BAA0B,KAAK,SAAS;QACpD,OAAO,IAAI,CAAC,gBAAgB,KAAK,SAAS;QAC1C,OAAO,IAAI,CAAC,+BAA+B,KAAK,SAAS,CAC1D,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,CAAC;QAC1C,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,2BAA2B,GAA6C;IAC5E,4BAA4B;IAC5B,kBAAkB;IAClB,iCAAiC;CAClC,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAErD,0EAA0E;AAC1E,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,kCAAkC,CAChD,OAAe,EACf,OAA0C,EAAE;IAE5C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACpD,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EACJ,8BAA8B,iBAAiB,CAAC,OAAO,CAAC,WAAW;gBACnE,mEAAmE;gBACnE,qEAAqE;gBACrE,wEAAwE;gBACxE,0EAA0E;gBAC1E,mDAAmD;SACtD,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,2BAA2B,CAAC,MAAM,CAChD,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CACnC,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EACJ,iCAAiC,MAAM,CAAC,OAAO,WAAW;gBAC1D,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,sCAAsC;gBACnE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB;gBACvE,mEAAmE;gBACnE,oCAAoC;SACvC,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC9C,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3B,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EACJ,mDAAmD,MAAM,CAAC,SAAS,KAAK;gBACxE,oEAAoE;gBACpE,gEAAgE;SACnE,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC;IACvC,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;QAClD,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EACJ,4CAA4C,MAAM,CAAC,SAAS,MAAM,QAAQ,IAAI;gBAC9E,SAAS,QAAQ,oDAAoD;gBACrE,qEAAqE;gBACrE,0EAA0E;gBAC1E,iEAAiE;gBACjE,yEAAyE;gBACzE,4DAA4D;gBAC5D,uCAAuC;SAC1C,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,kBAAkB,EAAE,CAAC;QAChC,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EACJ,yDAAyD;gBACzD,GAAG,MAAM,CAAC,SAAS,uDAAuD;gBAC1E,2EAA2E;gBAC3E,oDAAoD;SACvD,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;AAClC,CAAC"}

package/dist/src/storage/chunk-backfill.d.ts

@@ -0,0 +1,39 @@
+import type { EmbeddingProvider } from "../providers/types.js";
+export interface ChunkBackfillOptions {
+    embedding: EmbeddingProvider;
+    dataDir: string;
+    /** Report what would change without embedding or writing anything. */
+    dryRun?: boolean;
+    /** Stop after this many conversations have been (re)chunked. 0/undefined = no cap. */
+    limit?: number;
+    /** Sleep this many ms between conversations to spare the embedding API. */
+    throttleMs?: number;
+    /** Cooperative cancel — checked between conversations (Ctrl-C safe). */
+    signal?: AbortSignal;
+    /** Progress sink (one line per meaningful event). */
+    log?: (msg: string) => void;
+}
+export interface ChunkBackfillSummary {
+    /** Distinct conversations examined (most-recent raw per id). */
+    scanned: number;
+    /** Conversations split into a multi-chunk set this run (or that would be, in dryRun). */
+    rechunked: number;
+    /** Conversations already in the desired multi-chunk shape — skipped. */
+    alreadyChunked: number;
+    /** Conversations short enough to stay a single unmarked row — skipped. */
+    singleChunk: number;
+    /** Raw artifacts with no matching indexed row — skipped (not resurrected). */
+    orphaned: number;
+    /** Raw artifacts that could not be read/parsed — skipped. */
+    errored: number;
+    /** True if an abort signal stopped the run before all conversations were seen. */
+    aborted: boolean;
+}
+/**
+ * Re-chunk historical conversations from raw into chunk sets. Returns a summary
+ * of what changed. Idempotent: a conversation already in its desired shape (a
+ * complete set of the right count, or a single unmarked row for a short
+ * conversation) is skipped, so re-running only touches the remainder.
+ */
+export declare function backfillConversationChunks(opts: ChunkBackfillOptions): Promise<ChunkBackfillSummary>;
+//# sourceMappingURL=chunk-backfill.d.ts.map

package/dist/src/storage/chunk-backfill.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-backfill.d.ts","sourceRoot":"","sources":["../../../src/storage/chunk-backfill.ts"],"names":[],"mappings":"AA8BA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AA0B/D,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,iBAAiB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,sEAAsE;IACtE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,sFAAsF;IACtF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,qDAAqD;IACrD,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,oBAAoB;IACnC,gEAAgE;IAChE,OAAO,EAAE,MAAM,CAAC;IAChB,yFAAyF;IACzF,SAAS,EAAE,MAAM,CAAC;IAClB,wEAAwE;IACxE,cAAc,EAAE,MAAM,CAAC;IACvB,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC;IACpB,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,CAAC;IACjB,6DAA6D;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,kFAAkF;IAClF,OAAO,EAAE,OAAO,CAAC;CAClB;AAyGD;;;;;GAKG;AACH,wBAAsB,0BAA0B,CAC9C,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,oBAAoB,CAAC,CAuH/B"}

package/dist/src/storage/chunk-backfill.js

@@ -0,0 +1,295 @@
+/**
+ * Owner-only historical conversation chunk backfill.
+ *
+ * Index-time chunking (behind `RIFT_CHUNKING`) only splits NEW conversations as
+ * they are saved/ingested; conversations indexed before the flag was on are still
+ * stored as one whole row each. This module re-chunks those historical
+ * conversations from their raw artifacts so the owner can dogfood retrieval over a
+ * fully-chunked corpus and gather OFF-vs-ON eval evidence before any product
+ * default flip.
+ *
+ * Deliberately CRUDE and owner-grade — NOT the productized beta-user migration:
+ *   - explicit, one-shot (run from the offline CLI; no background daemon job),
+ *   - `dryRun` reports what would change without embedding or writing,
+ *   - lightly throttleable, and safe to Ctrl-C (an `AbortSignal` is checked
+ *     between conversations, so the in-flight conversation's atomic write always
+ *     finishes before we stop — never a half-written set on interrupt),
+ *   - no resumable status UI, pause/cancel, or rollback tooling.
+ *
+ * Data-safety mirrors the save/ingest chunk-write path: every chunk of a
+ * conversation is embedded BEFORE any destructive delete, and the raw artifact
+ * that drove the re-chunk is left on disk, so an interrupt between delete and
+ * insert is fully repairable by re-running (or by reconcile/rebuild).
+ *
+ * Reads are compatible with the `expand` stitching slice: a backfilled set uses
+ * the same `parent_id` / `chunk_index` / `chunk_count` markers and `convChunkId`
+ * scheme, so `detail="full"` reassembles a backfilled conversation through the
+ * exact same path as a natively-chunked one.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { CONVERSATION_SOURCES } from "../providers/types.js";
+import { RIFT_NONE_EMBEDDING_PROVIDER, isPlaceholderEmbeddingProvider, } from "../providers/placeholder-embed.js";
+import { recordEmbed } from "../observability/embedding-events.js";
+import { recordIndexWrite } from "../observability/index-events.js";
+import { eqFilter } from "./filter.js";
+import { getTable } from "./tables.js";
+import { chunkConversation } from "../ingestion/chunk-text.js";
+import { convChunkId, conversationChunkColumns, chunkSetComplete, expectedChunkCount, } from "../ingestion/chunk-meta.js";
+import { conversationContentFingerprint } from "../ingestion/inbox-core/conversation-fingerprint.js";
+import { extractIdFromFilename, compareRawRecencyDesc } from "./rebuild.js";
+const ALL_CONV_SOURCES = [...CONVERSATION_SOURCES, "inbox"];
+const CONV_TABLES = [
+    "conversations_hot",
+    "conversations_cold",
+];
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Collect the most-recent raw artifact per conversation id across every source.
+ * Same recency policy as reconcile/rebuild (`compareRawRecencyDesc`) so all three
+ * tools canonicalise a duplicated id to the same winner.
+ */
+function collectRawConversations(dataDir) {
+    const byId = new Map();
+    for (const source of ALL_CONV_SOURCES) {
+        const rawDir = path.join(dataDir, "raw", "conversations", source);
+        if (!fs.existsSync(rawDir))
+            continue;
+        for (const file of fs.readdirSync(rawDir)) {
+            if (!file.endsWith(".json"))
+                continue;
+            const id = extractIdFromFilename(file);
+            if (!id)
+                continue;
+            const rawPath = path.join(rawDir, file);
+            const prior = byId.get(id);
+            if (!prior || compareRawRecencyDesc(rawPath, prior) < 0) {
+                byId.set(id, rawPath);
+            }
+        }
+    }
+    return [...byId.entries()].map(([id, rawPath]) => ({ id, rawPath }));
+}
+/**
+ * All currently-indexed rows of a conversation (the whole row OR its chunk set),
+ * gathered across BOTH tiers. The same id can appear in hot AND cold after a
+ * compaction crash ("copied to cold but not yet removed from hot"); we dedupe by
+ * id with the cold copy winning, and pick cold as the write target so the
+ * rechunk collapses the duplicate rather than leaving a stale copy in the other
+ * tier.
+ */
+async function fetchConversationRows(id) {
+    const idFilter = eqFilter("id", id, { validateAsRowId: true });
+    const parentFilter = eqFilter("parent_id", id, { validateAsRowId: true });
+    const uniq = new Map();
+    let hasCold = false;
+    let hasCrossTierRows = false;
+    // Iterate hot → cold so the canonical cold row overwrites its hot duplicate.
+    for (const table of CONV_TABLES) {
+        const handle = getTable(table);
+        // Schema-aware filter. A legacy pre-chunk table (e.g. beta.21) has NO
+        // `parent_id` column; filtering on it would throw. When the column is
+        // absent we match by id only and the rows are necessarily legacy unchunked
+        // (no chunk set can exist without the column). This lets dry-run recon run
+        // against the exact incident shape WITHOUT migrating the columns on — which
+        // is the whole point of the read-only path.
+        const hasParentId = (await handle.schema()).fields.some((f) => f.name === "parent_id");
+        const setFilter = hasParentId
+            ? `(${idFilter}) OR (${parentFilter})`
+            : idFilter;
+        const rows = (await handle
+            .query()
+            .where(setFilter)
+            .toArray());
+        if (rows.length === 0)
+            continue;
+        if (table === "conversations_cold")
+            hasCold = true;
+        for (const r of rows) {
+            // Already seen this id in an earlier tier → it lives in both → duplicate.
+            if (uniq.has(r.id))
+                hasCrossTierRows = true;
+            uniq.set(r.id, r);
+        }
+    }
+    if (uniq.size === 0)
+        return null;
+    return {
+        targetTable: hasCold ? "conversations_cold" : "conversations_hot",
+        rows: [...uniq.values()],
+        hasCrossTierRows,
+    };
+}
+/**
+ * Re-chunk historical conversations from raw into chunk sets. Returns a summary
+ * of what changed. Idempotent: a conversation already in its desired shape (a
+ * complete set of the right count, or a single unmarked row for a short
+ * conversation) is skipped, so re-running only touches the remainder.
+ */
+export async function backfillConversationChunks(opts) {
+    const { embedding, dataDir } = opts;
+    const dryRun = opts.dryRun ?? false;
+    const throttleMs = opts.throttleMs ?? 0;
+    const limit = opts.limit ?? 0;
+    const log = opts.log ?? (() => { });
+    const summary = {
+        scanned: 0,
+        rechunked: 0,
+        alreadyChunked: 0,
+        singleChunk: 0,
+        orphaned: 0,
+        errored: 0,
+        aborted: false,
+    };
+    const raws = collectRawConversations(dataDir);
+    log(`Found ${raws.length} historical conversation(s) to examine.`);
+    for (const { id, rawPath } of raws) {
+        if (opts.signal?.aborted) {
+            summary.aborted = true;
+            log("Aborted — stopping before the next conversation.");
+            break;
+        }
+        if (limit > 0 && summary.rechunked >= limit) {
+            log(`Reached --limit ${limit}; stopping.`);
+            break;
+        }
+        summary.scanned++;
+        let raw;
+        try {
+            raw = JSON.parse(fs.readFileSync(rawPath, "utf-8"));
+        }
+        catch {
+            summary.errored++;
+            log(`! ${id}: unreadable raw artifact — skipped.`);
+            continue;
+        }
+        // Treat whitespace-only as empty: a raw with `"content": "   "` is truthy
+        // but `chunkConversation` trims it to zero chunks, which would otherwise
+        // drive an empty replacement set straight to the delete path below.
+        const pickContent = (v) => typeof v === "string" && v.trim().length > 0 ? v : "";
+        const content = pickContent(raw.content) || pickContent(raw.summary);
+        if (content.trim().length === 0) {
+            summary.errored++;
+            log(`! ${id}: raw has no content — skipped.`);
+            continue;
+        }
+        const existing = await fetchConversationRows(id);
+        if (!existing) {
+            // Raw with no indexed row: an orphan (failed save / pruned row). Do NOT
+            // resurrect it here — that's reconcile/rebuild's job, with their full
+            // dedup machinery. Backfill only re-shapes already-indexed conversations.
+            summary.orphaned++;
+            continue;
+        }
+        const chunks = chunkConversation(content);
+        const desiredCount = chunks.length;
+        // Hard guard against ever reaching the delete path with an empty
+        // replacement set. The content check above already rejects blank raws (the
+        // only input for which `chunkConversation` returns []), so this is purely
+        // defensive — but a zero-chunk set means `rechunkConversation` would delete
+        // the existing rows from both tiers and insert nothing, silently dropping an
+        // indexed conversation. Skip and count it as an error instead.
+        if (desiredCount === 0) {
+            summary.errored++;
+            log(`! ${id}: produced no chunks — skipped (existing rows untouched).`);
+            continue;
+        }
+        // Already in the desired shape? (complete set of the right count, or a
+        // single unmarked row for a short conversation.) Skip — keeps re-runs cheap
+        // and avoids needless re-embedding. But NEVER skip a cross-tier duplicate:
+        // the deduped `existing.rows` can look complete while a stale copy survives
+        // in the other tier, so it still needs the delete-both-tiers repair below.
+        if (!existing.hasCrossTierRows &&
+            chunkSetComplete(existing.rows) &&
+            expectedChunkCount(existing.rows) === desiredCount) {
+            if (desiredCount > 1)
+                summary.alreadyChunked++;
+            else
+                summary.singleChunk++;
+            continue;
+        }
+        if (dryRun) {
+            summary.rechunked++;
+            log(`~ ${id}: would re-chunk into ${desiredCount} section(s) [${existing.targetTable}] (dry-run).`);
+            if (throttleMs > 0)
+                await sleep(throttleMs);
+            continue;
+        }
+        await rechunkConversation(embedding, dataDir, id, content, chunks, existing);
+        summary.rechunked++;
+        log(`+ ${id}: re-chunked into ${desiredCount} section(s) [${existing.targetTable}].`);
+        if (throttleMs > 0)
+            await sleep(throttleMs);
+    }
+    return summary;
+}
+/**
+ * Embed the full chunk set, then atomically replace the conversation's rows in
+ * its current tier. Metadata (domain/intent/quality/topics/decisions/key_outputs/
+ * summary/idempotency_key/fingerprint/metadata_provider) is preserved from the
+ * existing indexed row so an enriched conversation is NOT downgraded to its raw
+ * artifact's basic metadata — only `content`/`embedding`/the chunk markers change.
+ *
+ * Ordering = embed-all → delete-old → insert-new (same invariant as reconcile):
+ * the embeddings exist before any delete, and the raw artifact survives, so an
+ * interrupt in the brief delete→insert window is repaired by re-running.
+ */
+async function rechunkConversation(embedding, dataDir, id, content, chunks, existing) {
+    const count = chunks.length;
+    const embeddings = await recordEmbed(dataDir, embedding, {
+        pipeline: "backfill",
+        operation: "rechunk_conversation",
+        input_count: count,
+    }, () => embedding.embedBatch(chunks));
+    // Representative = lowest chunk_index (or the lone row) — its metadata is the
+    // conversation-level metadata shared by every chunk.
+    const rep = [...existing.rows].sort((a, b) => Number(a.chunk_index ?? 0) - Number(b.chunk_index ?? 0))[0];
+    const fingerprint = typeof rep.conversation_fingerprint === "string" &&
+        rep.conversation_fingerprint.length > 0
+        ? rep.conversation_fingerprint
+        : conversationContentFingerprint(content);
+    const embeddingProviderMarker = isPlaceholderEmbeddingProvider(embedding)
+        ? RIFT_NONE_EMBEDDING_PROVIDER
+        : "";
+    const indexedAt = new Date().toISOString();
+    const rows = chunks.map((chunk, i) => ({
+        id: count > 1 ? convChunkId(id, i) : id,
+        content: chunk,
+        summary: rep.summary,
+        embedding: embeddings[i],
+        source: rep.source,
+        domain: rep.domain,
+        intent: rep.intent,
+        quality: rep.quality,
+        topics: rep.topics,
+        decisions: rep.decisions,
+        key_outputs: rep.key_outputs,
+        indexed_at: indexedAt,
+        idempotency_key: rep.idempotency_key,
+        conversation_fingerprint: fingerprint,
+        metadata_provider: typeof rep.metadata_provider === "string" ? rep.metadata_provider : "",
+        embedding_provider: embeddingProviderMarker,
+        ...conversationChunkColumns(id, i, count),
+    }));
+    // Delete every prior row for this conversation (the whole row keyed by id and
+    // any chunk set keyed by parent_id) from BOTH tiers — a compaction crash can
+    // leave the same id in hot AND cold, and deleting only one tier would leave a
+    // stale duplicate behind. Then insert the new set into the canonical target
+    // tier (cold if the conversation lives in cold, else hot), preserving the
+    // hot/cold placement rather than promoting old conversations into hot.
+    const idFilter = eqFilter("id", id, { validateAsRowId: true });
+    const parentFilter = eqFilter("parent_id", id, { validateAsRowId: true });
+    for (const table of CONV_TABLES) {
+        await getTable(table).delete(idFilter);
+        await getTable(table).delete(parentFilter);
+    }
+    await recordIndexWrite(dataDir, {
+        table: existing.targetTable,
+        pipeline: "backfill",
+        operation: "rechunk_conversation",
+        row_count: rows.length,
+    }, () => getTable(existing.targetTable).add(rows));
+}
+//# sourceMappingURL=chunk-backfill.js.map