akm-cli 0.9.0-beta.9 → 0.9.0-rc.0

package/dist/sources/wiki-fetchers/youtube.js

@@ -0,0 +1,239 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { fetchWithRetry } from "../../core/common.js";
+const YOUTUBE_HOSTS = new Set(["www.youtube.com", "youtube.com", "m.youtube.com", "youtu.be"]);
+const WATCH_HOST = "https://www.youtube.com/watch?v=";
+// Captions: the WEB watch-page caption baseUrls are now Proof-of-Origin-Token
+// gated (they carry `exp=xpe` and return an empty body without a `pot` token).
+// The ANDROID InnerTube `player` client still returns UNGATED caption baseUrls
+// that serve the transcript, so captions are sourced from there. See
+// https://github.com/yt-dlp/yt-dlp/issues/13075.
+const INNERTUBE_PLAYER_URL = "https://www.youtube.com/youtubei/v1/player";
+// Long-stable public InnerTube key; only authorizes the player call (not auth).
+const DEFAULT_INNERTUBE_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
+const ANDROID_INNERTUBE_CLIENT = {
+    clientName: "ANDROID",
+    clientVersion: "20.10.38",
+    androidSdkVersion: 34,
+    hl: "en",
+    gl: "US",
+};
+function extractVideoId(url) {
+    if (!YOUTUBE_HOSTS.has(url.hostname))
+        return null;
+    if (url.hostname === "youtu.be") {
+        const id = url.pathname.replace(/^\/+/, "").split("/")[0]?.trim();
+        return id || null;
+    }
+    if (url.pathname === "/watch") {
+        const id = url.searchParams.get("v")?.trim();
+        return id || null;
+    }
+    if (url.pathname.startsWith("/shorts/")) {
+        const id = url.pathname.slice("/shorts/".length).split("/")[0]?.trim();
+        return id || null;
+    }
+    if (url.pathname.startsWith("/embed/")) {
+        const id = url.pathname.slice("/embed/".length).split("/")[0]?.trim();
+        return id || null;
+    }
+    return null;
+}
+function canonicalWatchUrl(videoId) {
+    return `${WATCH_HOST}${encodeURIComponent(videoId)}`;
+}
+function decodeEntities(value) {
+    const safeFromCodePoint = (codePoint, fallback) => {
+        if (!Number.isFinite(codePoint) || codePoint < 0 || codePoint > 0x10ffff)
+            return fallback;
+        try {
+            return String.fromCodePoint(codePoint);
+        }
+        catch {
+            return fallback;
+        }
+    };
+    return value
+        .replace(/&#x([0-9a-f]+);/gi, (m, hex) => safeFromCodePoint(Number.parseInt(hex, 16), m))
+        .replace(/&#([0-9]+);/g, (m, dec) => safeFromCodePoint(Number.parseInt(dec, 10), m))
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;|&apos;/g, "'")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&amp;/g, "&");
+}
+function stripTags(value) {
+    return value.replace(/<[^>]+>/g, "");
+}
+function extractJsonObjectAfter(marker, source) {
+    const markerIndex = source.indexOf(marker);
+    if (markerIndex === -1)
+        return null;
+    const start = source.indexOf("{", markerIndex + marker.length);
+    if (start === -1)
+        return null;
+    let depth = 0;
+    let inString = false;
+    let escaped = false;
+    for (let i = start; i < source.length; i += 1) {
+        const char = source[i];
+        if (inString) {
+            if (escaped) {
+                escaped = false;
+            }
+            else if (char === "\\") {
+                escaped = true;
+            }
+            else if (char === '"') {
+                inString = false;
+            }
+            continue;
+        }
+        if (char === '"') {
+            inString = true;
+            continue;
+        }
+        if (char === "{")
+            depth += 1;
+        if (char === "}") {
+            depth -= 1;
+            if (depth === 0)
+                return source.slice(start, i + 1);
+        }
+    }
+    return null;
+}
+function parsePlayerResponse(html) {
+    const jsonText = extractJsonObjectAfter("ytInitialPlayerResponse =", html);
+    if (!jsonText)
+        return null;
+    try {
+        return JSON.parse(jsonText);
+    }
+    catch {
+        return null;
+    }
+}
+function chooseCaptionTrack(tracks) {
+    const normalized = tracks
+        .map((track) => ({
+        baseUrl: typeof track.baseUrl === "string" ? track.baseUrl : "",
+        languageCode: typeof track.languageCode === "string" ? track.languageCode : "",
+        kind: typeof track.kind === "string" ? track.kind : "",
+    }))
+        .filter((track) => track.baseUrl);
+    if (normalized.length === 0)
+        return null;
+    const preferred = normalized.find((track) => track.languageCode === "en" && track.kind !== "asr") ??
+        normalized.find((track) => track.languageCode.startsWith("en") && track.kind !== "asr") ??
+        normalized.find((track) => track.languageCode.startsWith("en")) ??
+        normalized[0];
+    return preferred.baseUrl;
+}
+async function fetchText(url, timeoutMs, signal) {
+    const response = await fetchWithRetry(url, {
+        headers: {
+            Accept: "text/html,application/json,application/xml,text/xml,text/plain;q=0.9,*/*;q=0.1",
+            "User-Agent": "akm-cli youtube fetcher",
+        },
+        signal,
+    }, { timeout: timeoutMs, retries: 1 });
+    if (!response.ok)
+        return null;
+    return response.text();
+}
+function parseTranscript(xml) {
+    // Two timedtext shapes: legacy srv1 `<text>` cues, and the format-3
+    // `<timedtext format="3">` `<p>` cues that the ANDROID InnerTube caption
+    // URLs return. Strip any nested `<s>` word tags via stripTags.
+    const matches = [...xml.matchAll(/<text\b[^>]*>([\s\S]*?)<\/text>/g), ...xml.matchAll(/<p\b[^>]*>([\s\S]*?)<\/p>/g)];
+    const texts = matches.map((match) => decodeEntities(stripTags(match[1] ?? "")).trim()).filter(Boolean);
+    if (texts.length === 0)
+        return null;
+    return texts.join("\n");
+}
+/**
+ * Fetch caption tracks via the ANDROID InnerTube `player` endpoint, whose
+ * caption baseUrls are not POT-gated. Returns [] on any failure so the caller
+ * can fall back to the (usually empty) watch-page tracks and degrade to a
+ * description-only snapshot.
+ */
+async function fetchInnertubeCaptionTracks(videoId, apiKey, timeoutMs, signal) {
+    try {
+        const response = await fetchWithRetry(`${INNERTUBE_PLAYER_URL}?key=${encodeURIComponent(apiKey)}`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "User-Agent": "com.google.android.youtube/20.10.38 (Linux; U; Android 14) gzip",
+            },
+            body: JSON.stringify({ context: { client: ANDROID_INNERTUBE_CLIENT }, videoId }),
+            signal,
+        }, { timeout: timeoutMs, retries: 1 });
+        if (!response.ok)
+            return [];
+        const json = (await response.json());
+        return json.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
+    }
+    catch {
+        return [];
+    }
+}
+/** Extract the page's InnerTube API key, falling back to the stable default. */
+function extractInnertubeKey(html) {
+    return html.match(/"INNERTUBE_API_KEY":"([^"]+)"/)?.[1] ?? DEFAULT_INNERTUBE_KEY;
+}
+const youtubeFetcher = {
+    name: "youtube-transcript",
+    matches(url) {
+        return extractVideoId(url) !== null;
+    },
+    async fetch(url, context) {
+        const videoId = extractVideoId(url);
+        if (!videoId)
+            return null;
+        const watchUrl = canonicalWatchUrl(videoId);
+        const watchHtml = await fetchText(watchUrl, context.timeoutMs, context.signal);
+        if (!watchHtml)
+            return null;
+        const playerResponse = parsePlayerResponse(watchHtml);
+        if (!playerResponse)
+            return null;
+        const title = typeof playerResponse.videoDetails?.title === "string" ? playerResponse.videoDetails.title.trim() : "";
+        const description = typeof playerResponse.videoDetails?.shortDescription === "string"
+            ? playerResponse.videoDetails.shortDescription.trim()
+            : "";
+        const sections = [];
+        let hasTranscript = false;
+        if (description) {
+            sections.push("## Description", "", description);
+        }
+        // Prefer ungated ANDROID InnerTube caption tracks; fall back to the
+        // (now usually empty) watch-page tracks so a description-only snapshot
+        // still works when InnerTube is unavailable.
+        const apiKey = extractInnertubeKey(watchHtml);
+        let tracks = await fetchInnertubeCaptionTracks(videoId, apiKey, context.timeoutMs, context.signal);
+        if (tracks.length === 0) {
+            tracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
+        }
+        const captionUrl = chooseCaptionTrack(tracks);
+        if (captionUrl) {
+            const transcriptXml = await fetchText(captionUrl, context.timeoutMs, context.signal);
+            const transcript = transcriptXml ? parseTranscript(transcriptXml) : null;
+            if (transcript) {
+                hasTranscript = true;
+                sections.push("## Transcript", "", transcript);
+            }
+        }
+        if (sections.length === 0)
+            return null;
+        return {
+            url: watchUrl,
+            title: title || `YouTube ${videoId}`,
+            markdown: sections.join("\n"),
+            preferredName: `youtube/${videoId}`,
+            tags: hasTranscript ? ["youtube", "video", "transcript"] : ["youtube", "video"],
+        };
+    },
+};
+export default youtubeFetcher;

package/dist/storage/database.js

@@ -35,18 +35,39 @@ const isBun = !!process.versions?.bun;
 // Bun built-in, unresolvable on Node) nor `better-sqlite3` (an optional native
 // dep, possibly absent under Bun) is statically imported.
 const nodeRequire = createRequire(import.meta.url);
+// bun:sqlite — Bun built-in, no native build. Cannot run on Node.
+const bunSqliteProvider = {
+    name: "bun:sqlite",
+    supported: () => isBun,
+    open: openBunDatabase,
+};
+// better-sqlite3 — native Node driver. Cannot run on Bun (oven-sh/bun#4290).
+const nodeSqliteProvider = {
+    name: "better-sqlite3",
+    supported: () => !isBun,
+    open: openNodeDatabase,
+};
 /**
- * Open a SQLite database handle at `path`, selecting the driver for the current
- * runtime. Returns a handle conforming to the structural {@link Database} type.
- *
- * The Node driver (`better-sqlite3`) is required lazily and ONLY when not on
- * Bun, so importing this module under Bun never touches `better-sqlite3`.
+ * Ordered provider registry. The factory selects the first supported provider.
+ * A future Postgres provider is appended here — and only here. Both SQLite
+ * engines are kept as distinct providers (not collapsed) because no single
+ * SQLite driver runs on both Bun and Node today.
  */
-export function openDatabase(path, opts) {
-    if (isBun) {
-        return openBunDatabase(path, opts);
+const PROVIDERS = [bunSqliteProvider, nodeSqliteProvider];
+/** Select the provider for the current runtime. */
+function selectProvider() {
+    const provider = PROVIDERS.find((p) => p.supported());
+    if (!provider) {
+        throw new Error(`No storage provider supports the current runtime (${isBun ? "Bun" : "Node"}).`);
     }
-    return openNodeDatabase(path, opts);
+    return provider;
+}
+/**
+ * Open a SQLite database handle at `path` via the active {@link StorageProvider}.
+ * Returns a handle conforming to the structural {@link Database} type.
+ */
+export function openDatabase(path, opts) {
+    return selectProvider().open(path, opts);
 }
 function openBunDatabase(path, opts) {
     const { Database: BunDatabase } = loadBunSqlite();
@@ -81,7 +102,21 @@ function loadBetterSqlite3() {
     if (!betterSqlite3Ctor) {
         // Runtime-gated dynamic require: only reached when NOT on Bun, so Bun never
         // resolves or loads the optional `better-sqlite3` native dependency.
-        const mod = nodeRequire("better-sqlite3");
+        // `better-sqlite3` is an optionalDependency, so `npm i` can succeed without
+        // it (or with a native build that failed). Convert the raw MODULE_NOT_FOUND
+        // into an actionable message instead of a cryptic onboarding crash.
+        let mod;
+        try {
+            mod = nodeRequire("better-sqlite3");
+        }
+        catch (err) {
+            throw new Error("akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
+                "  • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
+                "    'better-sqlite3' native binding rebuilds (a global `npm i -g better-sqlite3`\n" +
+                "    will NOT be resolved — Node loads it from akm's own node_modules).\n" +
+                "  • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.\n" +
+                `  Underlying load error: ${err instanceof Error ? err.message : String(err)}`);
+        }
         betterSqlite3Ctor = mod.default ?? mod;
     }
     return betterSqlite3Ctor;

package/dist/storage/managed-db.js

@@ -0,0 +1,82 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Managed-database seam — the single home for the SQLite open/lifecycle recipe.
+ *
+ * Before this module, two idioms were copy-pasted across state.db / logs.db /
+ * workflow.db / index.db and their consumers:
+ *
+ *   1. The open recipe: `mkdir(dir) → openDatabase(path) → applyStandardPragmas
+ *      → migrate`.
+ *   2. The borrow-or-own lifecycle: `const db = ctx?.db ?? open(); const owns =
+ *      !ctx?.db; try { … } finally { if (owns) db.close(); }`.
+ *
+ * {@link openManagedDatabase} owns (1); {@link withManagedDb} owns (2). Each DB
+ * module supplies only a path + initializer and gets a `withXDb` loan helper
+ * (see `withIndexDb`, `withStateDb`, etc.) so callers never hand-roll the
+ * ownership flag or the finally/close again. This is also the one place to add
+ * busy-timeout tuning, integrity checks, or test-isolation injection.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { openDatabase } from "./database.js";
+import { applyStandardPragmas } from "./sqlite-pragmas.js";
+/**
+ * Open a managed SQLite database: ensure the parent dir exists, open the handle,
+ * apply standard pragmas, then run the schema initializer. The single home for
+ * the open→pragmas→migrate recipe.
+ */
+export function openManagedDatabase(spec) {
+    const dir = path.dirname(spec.path);
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+    }
+    const db = openDatabase(spec.path);
+    applyStandardPragmas(db, spec.pragmas ?? { dataDir: dir });
+    spec.init?.(db);
+    return db;
+}
+/**
+ * Run `fn` against a managed database, owning its lifecycle.
+ *
+ * When `opts.borrowed` is supplied the caller already owns an open handle: it is
+ * passed straight through and NOT closed (borrow). Otherwise a fresh handle is
+ * opened via `open` and closed in a `finally` (own). This replaces the
+ * hand-rolled `ctx?.db ?? open()` + `ownsDb` flag + `finally`/close idiom — the
+ * ownership decision and the close live here, once.
+ *
+ * Synchronous by design: the DB consumers (telemetry writers, planners) finish
+ * all work within the tick, matching the inline blocks this replaces.
+ */
+export function withManagedDb(open, fn, opts) {
+    if (opts?.borrowed) {
+        return fn(opts.borrowed);
+    }
+    const db = open();
+    try {
+        return fn(db);
+    }
+    finally {
+        db.close();
+    }
+}
+/**
+ * Async sibling of {@link withManagedDb}. Use this — NOT `withManagedDb` — when
+ * `fn` holds the handle across an `await`: the sync version closes in its
+ * `finally` before the awaited work resolves (use-after-close). Here the handle
+ * is closed only after `fn`'s promise settles. Borrowed handles pass straight
+ * through and are not closed, as in the sync version.
+ */
+export async function withManagedDbAsync(open, fn, opts) {
+    if (opts?.borrowed) {
+        return fn(opts.borrowed);
+    }
+    const db = open();
+    try {
+        return await fn(db);
+    }
+    finally {
+        db.close();
+    }
+}

package/dist/storage/repositories/canaries-repository.js

@@ -0,0 +1,107 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/** Insert a freshly minted canary set (all rows active, one shared set id). */
+export function insertCanaries(db, canarySetId, canaries, now) {
+    if (canaries.length === 0)
+        return;
+    const ts = now ?? new Date().toISOString();
+    const stmt = db.prepare(`
+    INSERT INTO canary_queries (canary_set_id, anchor_ref, query, source, active, created_at)
+    VALUES (?, ?, ?, ?, 1, ?)
+  `);
+    db.transaction(() => {
+        for (const c of canaries) {
+            stmt.run(canarySetId, c.anchorRef, c.query, c.source ?? "auto", ts);
+        }
+    })();
+}
+/** Load the active canary set (empty array = never minted). */
+export function getActiveCanaries(db) {
+    // Scope to the NEWEST active set: if an interrupted refresh (or a bug) ever
+    // leaves two sets active, mixing their rows would silently corrupt the
+    // recall/entropy trend baselines. The newest set wins; stale-active rows are
+    // simply never returned.
+    return db
+        .prepare(`SELECT * FROM canary_queries
+       WHERE active = 1 AND canary_set_id = (
+         SELECT canary_set_id FROM canary_queries WHERE active = 1
+         ORDER BY created_at DESC, id DESC LIMIT 1
+       )
+       ORDER BY id`)
+        .all();
+}
+/** Load one canary set's rows by its exact set id (any active state), insertion order. */
+export function getCanariesBySetId(db, canarySetId) {
+    return db
+        .prepare(`SELECT * FROM canary_queries WHERE canary_set_id = ? ORDER BY id`)
+        .all(canarySetId);
+}
+/** List every distinct canary_set_id that still has active rows. */
+export function listActiveCanarySetIds(db) {
+    const rows = db.prepare(`SELECT DISTINCT canary_set_id FROM canary_queries WHERE active = 1`).all();
+    return rows.map((r) => r.canary_set_id);
+}
+/**
+ * Deactivate every canary row in a set. Rows are RETAINED (active = 0) so
+ * historical improve_cycle_metrics rows keyed on the old canary_set_id stay
+ * interpretable; only `akm improve canary --refresh` calls this.
+ */
+export function deactivateCanarySet(db, canarySetId) {
+    const result = db
+        .prepare(`UPDATE canary_queries SET active = 0 WHERE canary_set_id = ? AND active = 1`)
+        .run(canarySetId);
+    const changes = result.changes ?? 0;
+    return typeof changes === "bigint" ? Number(changes) : changes;
+}
+/** Persist one qualifying cycle's store-health snapshot. */
+export function insertCycleMetrics(db, row) {
+    db.prepare(`
+    INSERT INTO improve_cycle_metrics
+      (run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
+       canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
+       mean_bigram_diversity, over_generation_count, accepted_actions,
+       merge_floor_violations, alerts_json)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(row.run_id, row.ts, row.pass, row.canary_set_id, row.mean_recall, row.mean_ndcg, row.mean_mrr, row.canary_ranks_json, row.store_total, row.store_by_type_json, row.distinct_content_ratio, row.mean_bigram_diversity, row.over_generation_count, row.accepted_actions, row.merge_floor_violations, row.alerts_json);
+}
+/**
+ * Load the most recent cycle rows for one canary set, OLDEST-first (the alert
+ * evaluator's window order). Scoped by canary_set_id so trends never compare
+ * across canary re-mints.
+ */
+export function queryRecentCycleMetrics(db, canarySetId, limit) {
+    const rows = db
+        .prepare(`SELECT run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
+              canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
+              mean_bigram_diversity, over_generation_count, accepted_actions,
+              merge_floor_violations, alerts_json
+       FROM improve_cycle_metrics WHERE canary_set_id = ?
+       ORDER BY ts DESC, id DESC LIMIT ?`)
+        .all(canarySetId, Math.max(0, limit));
+    return rows.reverse();
+}
+/** Load the single most recent cycle row across all canary sets (health surface). */
+export function getLatestCycleMetrics(db) {
+    const row = db
+        .prepare(`SELECT run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
+              canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
+              mean_bigram_diversity, over_generation_count, accepted_actions,
+              merge_floor_violations, alerts_json
+       FROM improve_cycle_metrics ORDER BY ts DESC, id DESC LIMIT 1`)
+        .get();
+    return row == null ? undefined : row;
+}
+/**
+ * Delete cycle rows older than `retentionDays` (default 365 — owner-approved;
+ * a slow collapse needs a longer trend window than the 90-day events log).
+ * Returns the purged row count. canary_queries rows are never purged.
+ */
+export function purgeOldCycleMetrics(db, retentionDays = 365) {
+    if (!Number.isFinite(retentionDays) || retentionDays <= 0)
+        return 0;
+    const cutoff = new Date(Date.now() - retentionDays * 86_400_000).toISOString();
+    const result = db.prepare("DELETE FROM improve_cycle_metrics WHERE ts < ?").run(cutoff);
+    const changes = result.changes ?? 0;
+    return typeof changes === "bigint" ? Number(changes) : changes;
+}

package/dist/storage/repositories/consolidation-repository.js

@@ -0,0 +1,38 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Bulk-fetch the judged-state cache for a set of entry keys in one query.
+ * Returns a Map keyed by entry_key so the consolidate pool-selection loop can
+ * do O(1) "has this memory been judged at this content hash?" lookups.
+ * Empty input → empty map (no query issued).
+ */
+export function getConsolidationJudgedMap(db, entryKeys) {
+    const out = new Map();
+    if (entryKeys.length === 0)
+        return out;
+    // SQLite has a ~999 param ceiling; chunk if a caller ever exceeds that.
+    const CHUNK = 500;
+    for (let i = 0; i < entryKeys.length; i += CHUNK) {
+        const chunk = entryKeys.slice(i, i + CHUNK);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db
+            .prepare(`SELECT * FROM consolidation_judged WHERE entry_key IN (${placeholders})`)
+            .all(...chunk);
+        for (const row of rows)
+            out.set(row.entry_key, row);
+    }
+    return out;
+}
+/**
+ * Record (or update) the judged state for one memory. INSERT-OR-REPLACE so the
+ * row always reflects the most recent judge of that entry_key. Called once per
+ * memory the consolidate LLM saw in a successfully-judged chunk.
+ */
+export function upsertConsolidationJudged(db, input) {
+    db.prepare(`
+    INSERT OR REPLACE INTO consolidation_judged
+      (entry_key, content_hash, judged_at, outcome)
+    VALUES (?, ?, ?, ?)
+  `).run(input.entryKey, input.contentHash, input.judgedAt, input.outcome);
+}

package/dist/storage/repositories/embeddings-repository.js

@@ -0,0 +1,72 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Convert a `number[]` embedding vector to the `Float32Array` byte
+ * representation stored in the `body_embeddings.embedding` BLOB column.
+ */
+export function embeddingToBlob(vec) {
+    const f32 = new Float32Array(vec);
+    return new Uint8Array(f32.buffer);
+}
+/**
+ * Convert the raw `Uint8Array` bytes from the `body_embeddings.embedding`
+ * BLOB column back to a `number[]` embedding vector.
+ */
+export function blobToEmbedding(blob) {
+    // SQLite BLOB columns are returned as Uint8Array; re-interpret as Float32.
+    const f32 = new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
+    return Array.from(f32);
+}
+/**
+ * Bulk-fetch cached body embeddings for a set of content hashes.
+ * Returns a Map keyed by `content_hash` (embedding decoded to `number[]`).
+ * Empty input → empty map (no query issued).
+ *
+ * If the stored `model_id` does not match `expectedModelId` the entire table
+ * is cleared (drop-all on model mismatch) and an empty map is returned so
+ * callers re-embed everything on this run.
+ */
+export function getBodyEmbeddings(db, contentHashes, expectedModelId) {
+    const out = new Map();
+    if (contentHashes.length === 0)
+        return out;
+    // Model-id mismatch: vectors are in the wrong metric space — drop all rows.
+    const firstRow = db.prepare("SELECT model_id FROM body_embeddings LIMIT 1").get();
+    if (firstRow && firstRow.model_id !== expectedModelId) {
+        db.exec("DELETE FROM body_embeddings");
+        return out;
+    }
+    // SQLite has a ~999 param ceiling; chunk if needed.
+    const CHUNK = 500;
+    for (let i = 0; i < contentHashes.length; i += CHUNK) {
+        const chunk = contentHashes.slice(i, i + CHUNK);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db
+            .prepare(`SELECT content_hash, embedding FROM body_embeddings WHERE content_hash IN (${placeholders})`)
+            .all(...chunk);
+        for (const row of rows) {
+            out.set(row.content_hash, blobToEmbedding(row.embedding));
+        }
+    }
+    return out;
+}
+/**
+ * Upsert body-embedding rows in a single transaction.
+ * Each entry maps a `cacheHash` → `number[]` vector. `model_id` is stored
+ * so a future model change can trigger a drop-all purge.
+ */
+export function upsertBodyEmbeddings(db, entries) {
+    if (entries.length === 0)
+        return;
+    const now = Date.now();
+    const stmt = db.prepare(`
+    INSERT OR REPLACE INTO body_embeddings (content_hash, embedding, model_id, created_at)
+    VALUES (?, ?, ?, ?)
+  `);
+    db.transaction(() => {
+        for (const { contentHash, embedding, modelId } of entries) {
+            stmt.run(contentHash, embeddingToBlob(embedding), modelId, now);
+        }
+    })();
+}