@wirecat/cli-core 0.433.0 → 0.435.0

package/dist/store/sqlite/vector-index.d.ts

@@ -0,0 +1,76 @@
+import type { CacheDatabase } from "../driver.js";
+/**
+ * Every stored vector has length one, so the scan's dot product and the index's cosine rank alike. A model that
+ * already gives unit vectors (e5, the API models) changes by float rounding only. A zero vector (a text the model
+ * knows nothing of) stays zero: vec0 puts it first with no distance, so it is always among the candidates and scores
+ * 0 both ways.
+ */
+export declare const unitVector: (vector: Float32Array) => Float32Array;
+export interface VectorIndex {
+    table: string;
+    dims: number;
+    ready: boolean;
+}
+export declare const vectorIndexOf: (database: CacheDatabase, model: string) => VectorIndex | undefined;
+/**
+ * The model's index, made on its first vector. A model that already has vectors (an older store) is filled by
+ * `fillVectorIndexes` in short batches, and searched by scan until it is ready.
+ */
+export declare const ensureVectorIndex: (database: CacheDatabase, model: string, dims: number) => void;
+/**
+ * Indexes the vectors a store held before its index existed, in hash order from where the last batch stopped. Rows
+ * the triggers wrote meanwhile are skipped. Returns the models it filled.
+ */
+export declare const fillVectorIndexes: (database: CacheDatabase, { batch, notice }?: {
+    batch?: number;
+    notice?: (model: string) => void;
+}) => string[];
+export interface IndexFilter {
+    kind: string;
+    account: number;
+    chat?: number;
+    since?: number;
+    before?: number;
+}
+export interface Candidate {
+    id: number;
+    hash: string;
+    /** None for a zero vector. */
+    distance: number | null;
+}
+/**
+ * The best `limit` results by the index, or `undefined` when the caller must scan: no ready index for the model, or
+ * no exact answer within the most candidates vec0 returns. `survivors` applies the scan's own SQL to the candidates,
+ * in the scan's order (and checks each one's hash, so a reused chunk id is never taken for the old chunk), scores
+ * them, and names the chunks behind its best `limit` results. Those are exact once each is nearer than the last
+ * candidate by more than `NEAR_TIE`: every row nearer than that is among the candidates, and the rest cannot outrank
+ * them even where float32 and float64 disagree. A text many
+ * chunks share has one vector, so a tie that crosses the last candidate asks for more, as the scan's order decides it.
+ */
+export declare const searchIndex: <T>(database: CacheDatabase, model: string, filter: IndexFilter, query: Float32Array, limit: number, survivors: (candidates: Candidate[]) => {
+    results: T[];
+    best: number[];
+}) => T[] | undefined;
+/**
+ * Per model, `store check`'s safety net: the rows its index holds against the rows `source` derives now, compared
+ * whole (chunk, kind, account, chat, time, hash, vector) both ways. `missing` are rows the index lacks or holds
+ * wrong; `extra` rows it should not hold.
+ */
+export declare const vectorIndexCounts: (database: CacheDatabase) => {
+    model: string;
+    dims: number;
+    ready: boolean;
+    indexed: number;
+    expected: number;
+    missing: number;
+    extra: number;
+}[];
+/**
+ * The one rule for a vector whose text may be gone: it goes once no chunk of any kind holds its hash, for every model.
+ * Call it with the hashes a write released, after that write's new chunks are in, so a text that comes back keeps its
+ * vector. The key is (model, content_hash): naming each model reaches the row by the key; the hash alone scans every
+ * vector.
+ */
+export declare const dropUnusedVectors: (database: CacheDatabase, hashes: Iterable<string>) => void;
+/** Every vector no chunk holds, left by a build before `dropUnusedVectors` covered every write; how many. */
+export declare const dropAllUnusedVectors: (database: CacheDatabase) => number;

package/dist/store/sqlite/vector-index.js

@@ -0,0 +1,205 @@
+import { CliError } from "@wirecat/cli-base";
+import { writeBatch } from "./transaction.js";
+/**
+ * One sqlite-vec `vec0` table per embedding model, a row per chunk that has a vector of that model: rowid is
+ * `chunks.id`. The static triggers of store version 34 push every changed chunk id through `vector_index_redo`, and
+ * each model's own trigger re-derives that chunk's row, so every writer — the store's functions, the schema's own
+ * delete triggers, a raw `DELETE` — keeps the index in step inside its own transaction.
+ *
+ * `kind` and `account` are partition keys, `occurred_at` and `chat` metadata: the filters a KNN query applies before
+ * it ranks. `account`, `chat` and `occurred_at` are the columns each kind's scan filters on, so the index never
+ * leaves out a row the scan would keep; filters it cannot express are checked on the candidates after.
+ */
+const source = (model, dims) => `SELECT k.id, k.chunkable_type,
+    coalesce(CASE k.chunkable_type
+      WHEN 'image' THEN (SELECT ch.account_id FROM attachments a JOIN messages m ON m.id = a.attachable_id
+        JOIN chats ch ON ch.id = m.chat_id WHERE a.id = k.chunkable_id AND a.attachable_type = 'message')
+      WHEN 'conversation' THEN (SELECT ch.account_id FROM conversations c JOIN chats ch ON ch.id = c.chat_id
+        WHERE c.id = k.chunkable_id)
+      ELSE k.account_id END, 0),
+    coalesce(CASE k.chunkable_type
+      WHEN 'message' THEN (SELECT m.chat_id FROM messages m WHERE m.id = k.chunkable_id)
+      WHEN 'image' THEN (SELECT m.chat_id FROM attachments a JOIN messages m ON m.id = a.attachable_id
+        WHERE a.id = k.chunkable_id AND a.attachable_type = 'message')
+      WHEN 'conversation' THEN (SELECT c.chat_id FROM conversations c WHERE c.id = k.chunkable_id) END, 0),
+    coalesce(CASE k.chunkable_type
+      WHEN 'message' THEN (SELECT m.sent_at FROM messages m WHERE m.id = k.chunkable_id)
+      ELSE k.occurred_at END, 0),
+    v.vector, k.content_hash
+  FROM chunks k JOIN embeddings v ON v.model = ${literal(model)} AND v.content_hash = k.content_hash
+    AND length(v.vector) = ${dims * 4}`;
+/**
+ * Every stored vector has length one, so the scan's dot product and the index's cosine rank alike. A model that
+ * already gives unit vectors (e5, the API models) changes by float rounding only. A zero vector (a text the model
+ * knows nothing of) stays zero: vec0 puts it first with no distance, so it is always among the candidates and scores
+ * 0 both ways.
+ */
+export const unitVector = (vector) => {
+    let sum = 0;
+    for (const value of vector)
+        sum += value * value;
+    const length = Math.sqrt(sum);
+    if (!Number.isFinite(length))
+        throw new CliError("validation_error", "a vector must have finite values", { reason: "store.vector-invalid" });
+    return length === 0 ? vector : vector.map((value) => value / length);
+};
+const literal = (text) => `'${text.replaceAll("'", "''")}'`;
+const COLUMNS = "rowid, kind, account, chat, occurred_at, embedding, hash";
+const tableOf = (id) => `vector_index_${id}`;
+export const vectorIndexOf = (database, model) => {
+    const row = database.prepare("SELECT id, dims, ready FROM vector_indexes WHERE model = ?").get(model);
+    return row && { table: tableOf(Number(row.id)), dims: Number(row.dims), ready: row.ready === 1 };
+};
+/**
+ * The model's index, made on its first vector. A model that already has vectors (an older store) is filled by
+ * `fillVectorIndexes` in short batches, and searched by scan until it is ready.
+ */
+export const ensureVectorIndex = (database, model, dims) => {
+    if (vectorIndexOf(database, model))
+        return;
+    const held = database.prepare("SELECT 1 FROM embeddings WHERE model = ? LIMIT 1").get(model) !== undefined;
+    const id = Number(database
+        .prepare("INSERT INTO vector_indexes (model, dims, ready, filled_through) VALUES (?, ?, ?, '') RETURNING id")
+        .get(model, dims, held ? 0 : 1)?.id);
+    const table = tableOf(id);
+    database.exec(`CREATE VIRTUAL TABLE ${table} USING vec0(kind text partition key, account integer partition key,
+    chat integer, occurred_at integer, embedding float[${dims}] distance_metric=cosine, +hash text)`);
+    database.exec(`CREATE TRIGGER ${table}_redo AFTER INSERT ON vector_index_redo BEGIN
+    DELETE FROM ${table} WHERE rowid = new.chunk_id;
+    INSERT INTO ${table} (${COLUMNS}) ${source(model, dims)} WHERE k.id = new.chunk_id;
+  END`);
+};
+/** The models with vectors and no index, by a walk of the primary key rather than a scan of every vector. */
+const modelsWithoutIndex = (database) => database
+    .prepare(`WITH RECURSIVE m(model) AS (SELECT min(model) FROM embeddings
+         UNION ALL SELECT (SELECT min(model) FROM embeddings WHERE model > m.model) FROM m WHERE m.model IS NOT NULL)
+       SELECT m.model, (SELECT dims FROM embeddings WHERE model = m.model LIMIT 1) AS dims FROM m
+       WHERE m.model IS NOT NULL AND NOT EXISTS (SELECT 1 FROM vector_indexes i WHERE i.model = m.model)`)
+    .all();
+/** Hashes per batch: a batch is one short write transaction, so other processes wait less than their timeout. */
+const FILL_HASHES = 1_000;
+/**
+ * Indexes the vectors a store held before its index existed, in hash order from where the last batch stopped. Rows
+ * the triggers wrote meanwhile are skipped. Returns the models it filled.
+ */
+export const fillVectorIndexes = (database, { batch = FILL_HASHES, notice = () => { } } = {}) => {
+    for (const { model, dims } of modelsWithoutIndex(database))
+        writeBatch(database, () => ensureVectorIndex(database, model, Number(dims)));
+    const pending = database
+        .prepare("SELECT id, model, dims, filled_through FROM vector_indexes WHERE ready = 0 ORDER BY id")
+        .all();
+    for (const { id, model, dims, filled_through } of pending) {
+        notice(model);
+        const table = tableOf(id);
+        const step = database.prepare(`INSERT INTO ${table} (${COLUMNS}) ${source(model, dims)}
+        WHERE k.content_hash IN (SELECT value FROM json_each(?)) AND NOT EXISTS (SELECT 1 FROM ${table} WHERE rowid = k.id)`);
+        const hashes = database.prepare("SELECT content_hash AS hash FROM embeddings WHERE model = ? AND content_hash > ? ORDER BY content_hash LIMIT ?");
+        let after = filled_through;
+        for (;;) {
+            const done = writeBatch(database, () => {
+                const page = hashes.all(model, after, batch).map(({ hash }) => hash);
+                if (page.length > 0)
+                    step.run(JSON.stringify(page));
+                after = page.at(-1) ?? after;
+                const finished = page.length < batch;
+                // Another process may be filling too: neither moves the other back, nor un-readies a finished index.
+                database
+                    .prepare("UPDATE vector_indexes SET filled_through = ?, ready = ? WHERE id = ? AND ready = 0 AND filled_through <= ?")
+                    .run(after, finished ? 1 : 0, id, after);
+                return finished;
+            });
+            if (done)
+                break;
+        }
+    }
+    return pending.map(({ model }) => model);
+};
+/**
+ * vec0 measures in float32 and the scan in float64, so two results closer than this may swap. Far above float32's
+ * error at these lengths, far below any distance that matters for ranking.
+ */
+const NEAR_TIE = 1e-5;
+/** vec0 refuses a k above this. */
+const MAX_K = 4_096;
+const knn = (database, index, filter, query, k) => database
+    .prepare(`SELECT rowid AS id, hash, distance FROM ${index.table} WHERE embedding MATCH ? AND k = ? AND kind = ? AND account = ?
+        ${filter.chat === undefined ? "" : "AND chat = ?"}
+        ${filter.since === undefined ? "" : "AND occurred_at >= ?"}
+        ${filter.before === undefined ? "" : "AND occurred_at < ?"}`)
+    .all(new Uint8Array(query.buffer, query.byteOffset, query.byteLength), BigInt(k), filter.kind, 
+// node:sqlite binds a JS number as REAL, which vec0's integer columns refuse.
+BigInt(filter.account), ...[filter.chat, filter.since, filter.before].flatMap((value) => (value === undefined ? [] : [BigInt(value)])));
+/**
+ * The best `limit` results by the index, or `undefined` when the caller must scan: no ready index for the model, or
+ * no exact answer within the most candidates vec0 returns. `survivors` applies the scan's own SQL to the candidates,
+ * in the scan's order (and checks each one's hash, so a reused chunk id is never taken for the old chunk), scores
+ * them, and names the chunks behind its best `limit` results. Those are exact once each is nearer than the last
+ * candidate by more than `NEAR_TIE`: every row nearer than that is among the candidates, and the rest cannot outrank
+ * them even where float32 and float64 disagree. A text many
+ * chunks share has one vector, so a tie that crosses the last candidate asks for more, as the scan's order decides it.
+ */
+export const searchIndex = (database, model, filter, query, limit, survivors) => {
+    const index = vectorIndexOf(database, model);
+    if (!index?.ready || index.dims !== query.length)
+        return undefined;
+    let k = Math.min(MAX_K, Math.max(100, limit * 4));
+    for (;;) {
+        const candidates = knn(database, index, filter, query, k);
+        const { results, best } = survivors(candidates);
+        if (candidates.length < k)
+            return results;
+        const edge = candidates.at(-1)?.distance ?? null;
+        const distance = new Map(candidates.map((candidate) => [candidate.id, candidate.distance]));
+        // A last candidate with no distance is a zero vector: nothing is known to be nearer yet.
+        const nearer = (id) => edge !== null && (distance.get(id) ?? edge) < edge - NEAR_TIE;
+        if (best.length >= limit && best.every(nearer))
+            return results;
+        if (k === MAX_K)
+            return undefined;
+        k = Math.min(MAX_K, k * 4);
+    }
+};
+/**
+ * Per model, `store check`'s safety net: the rows its index holds against the rows `source` derives now, compared
+ * whole (chunk, kind, account, chat, time, hash, vector) both ways. `missing` are rows the index lacks or holds
+ * wrong; `extra` rows it should not hold.
+ */
+export const vectorIndexCounts = (database) => database.prepare("SELECT id, model, dims, ready FROM vector_indexes ORDER BY id").all().map(({ id, model, dims, ready }) => {
+    const held = `SELECT ${COLUMNS} FROM ${tableOf(id)}`;
+    const wanted = source(model, Number(dims));
+    const count = (sql) => Number(database.prepare(`SELECT count(*) AS n FROM (${sql})`).get()?.n ?? 0);
+    return {
+        model,
+        dims: Number(dims),
+        ready: ready === 1,
+        indexed: count(held),
+        expected: count(wanted),
+        missing: count(`${wanted} EXCEPT ${held}`),
+        extra: count(`${held} EXCEPT ${wanted}`),
+    };
+});
+/**
+ * The one rule for a vector whose text may be gone: it goes once no chunk of any kind holds its hash, for every model.
+ * Call it with the hashes a write released, after that write's new chunks are in, so a text that comes back keeps its
+ * vector. The key is (model, content_hash): naming each model reaches the row by the key; the hash alone scans every
+ * vector.
+ */
+export const dropUnusedVectors = (database, hashes) => {
+    const unique = new Set(hashes);
+    if (unique.size === 0)
+        return;
+    const models = database
+        .prepare(`WITH RECURSIVE m(model) AS (SELECT min(model) FROM embeddings
+         UNION ALL SELECT (SELECT min(model) FROM embeddings WHERE model > m.model) FROM m WHERE m.model IS NOT NULL)
+       SELECT model FROM m WHERE model IS NOT NULL`)
+        .all()
+        .map(({ model }) => String(model));
+    const drop = database.prepare("DELETE FROM embeddings WHERE model = ? AND content_hash = ? AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ?)");
+    for (const hash of unique)
+        for (const model of models)
+            drop.run(model, hash, hash);
+};
+/** Every vector no chunk holds, left by a build before `dropUnusedVectors` covered every write; how many. */
+export const dropAllUnusedVectors = (database) => database
+    .prepare("DELETE FROM embeddings WHERE NOT EXISTS (SELECT 1 FROM chunks k WHERE k.content_hash = embeddings.content_hash) RETURNING 1")
+    .all().length;

package/dist/store/sqlite/vectors.d.ts

@@ -36,8 +36,11 @@ export declare const conversationVectors: (context: StoreContext, conversationPk
     chunks: number;
     vectors: Float32Array[];
 };
-/** A vector already there for the same model and text is kept: the same text gives the same vector. */
-export declare const saveVectors: ({ orm, now }: StoreContext, model: string, dims: number, vectors: {
+/**
+ * A vector already there for the same model and text is kept: the same text gives the same vector. Each is stored at
+ * length one; a zero or non-finite vector is refused.
+ */
+export declare const saveVectors: ({ orm, now, database }: StoreContext, model: string, dims: number, vectors: {
     hash: string;
     vector: Float32Array;
 }[]) => void;
@@ -47,8 +50,8 @@ export declare const vectorStatus: ({ orm }: StoreContext, chatKey: number, mode
     embedded: number;
 };
 /**
- * Drops the vectors of the chat's chunks, or only one model's — but not one another chat's chunk still
- * points at: the same text in two chats has one vector. Messages are never touched.
+ * Drops the vectors of the chat's conversation and message chunks, or only one model's — but not one a chunk outside
+ * the chat still points at: the same text in two chats has one vector. The chunks themselves stay, to be embedded again.
  */
 export declare const clearVectors: ({ orm }: StoreContext, chatKey: number, model: string | undefined) => number;
 export interface NearestChunk {
@@ -65,7 +68,7 @@ export declare const dot: (query: Float32Array, blob: Uint8Array) => number;
  * The conversations of the current builds in scope nearest to `query`, best first: each scored by its
  * best chunk. Vectors are unit length, so the dot product is the cosine (phase 5 E3).
  */
-export declare const nearestChunks: ({ orm }: StoreContext, accountPk: number, { chatKey, model, since, limit, query, exclude, conversations, scope, projectId, before, personId, }: {
+export declare const nearestChunks: (context: StoreContext, accountPk: number, filter: {
     chatKey?: number;
     model: string;
     since?: number;

package/dist/store/sqlite/vectors.js

@@ -1,5 +1,6 @@
 import { chunkHash, chunkTextOf } from "../../conversations/chunks.js";
 import { sql } from "./drizzle/core.js";
+import { ensureVectorIndex, searchIndex, unitVector } from "./vector-index.js";
 const currentChunks = (chatKey) => sql `SELECT k.content_hash, k.chunkable_id AS conversation_id, r.first_message_id,
     r.last_message_id, r.text_start, r.text_end FROM chunks k JOIN chunk_messages r ON r.chunk_id = k.id
     JOIN conversations c ON k.chunkable_type = 'conversation' AND c.id = k.chunkable_id
@@ -113,10 +114,17 @@ export const conversationVectors = (context, conversationPk, model) => {
         : []);
     return { chunks: rows.length, vectors };
 };
-/** A vector already there for the same model and text is kept: the same text gives the same vector. */
-export const saveVectors = ({ orm, now }, model, dims, vectors) => {
+/**
+ * A vector already there for the same model and text is kept: the same text gives the same vector. Each is stored at
+ * length one; a zero or non-finite vector is refused.
+ */
+export const saveVectors = ({ orm, now, database }, model, dims, vectors) => {
+    if (vectors.length === 0)
+        return;
+    ensureVectorIndex(database, model, dims);
     const at = now();
-    for (const { hash, vector } of vectors) {
+    for (const { hash, vector: given } of vectors) {
+        const vector = unitVector(given);
         orm.run(sql `INSERT INTO embeddings (model, content_hash, dims, vector, created_at, updated_at)
         VALUES (${model}, ${hash}, ${dims}, ${Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength)}, ${at}, ${at})
         ON CONFLICT DO NOTHING`);
@@ -130,16 +138,19 @@ export const vectorStatus = ({ orm }, chatKey, model) => {
     return { chunks: Number(row?.chunks ?? 0), embedded: Number(row?.embedded ?? 0) };
 };
 /**
- * Drops the vectors of the chat's chunks, or only one model's — but not one another chat's chunk still
- * points at: the same text in two chats has one vector. Messages are never touched.
+ * Drops the vectors of the chat's conversation and message chunks, or only one model's — but not one a chunk outside
+ * the chat still points at: the same text in two chats has one vector. The chunks themselves stay, to be embedded again.
  */
-export const clearVectors = ({ orm }, chatKey, model) => orm.all(sql `DELETE FROM embeddings
+export const clearVectors = ({ orm }, chatKey, model) => orm.all(sql `WITH mine(id) AS (
+        SELECT k.id FROM chunks k JOIN conversations c ON k.chunkable_type = 'conversation' AND c.id = k.chunkable_id
+          WHERE c.chat_id = ${chatKey}
+        UNION ALL SELECT k.id FROM chunks k JOIN messages m ON k.chunkable_type = 'message' AND m.id = k.chunkable_id
+          WHERE m.chat_id = ${chatKey})
+      DELETE FROM embeddings
       WHERE ${model === undefined ? sql `1` : sql `model = ${model}`}
-        AND content_hash IN (SELECT k.content_hash FROM chunks k
-          JOIN conversations c ON k.chunkable_type = 'conversation' AND c.id = k.chunkable_id WHERE c.chat_id = ${chatKey})
-        AND content_hash NOT IN (SELECT k.content_hash FROM chunks k
-          LEFT JOIN conversations c ON k.chunkable_type = 'conversation' AND c.id = k.chunkable_id
-          WHERE c.id IS NULL OR c.chat_id <> ${chatKey})
+        AND content_hash IN (SELECT content_hash FROM chunks WHERE id IN (SELECT id FROM mine))
+        AND NOT EXISTS (SELECT 1 FROM chunks o WHERE o.content_hash = embeddings.content_hash
+          AND o.id NOT IN (SELECT id FROM mine))
       RETURNING 1 AS n`).length;
 /** Rows read per step of a search, so a chat of any size is scanned in bounded memory. */
 const SCAN_PAGE = 5_000;
@@ -157,14 +168,13 @@ export const dot = (query, blob) => {
  * The conversations of the current builds in scope nearest to `query`, best first: each scored by its
  * best chunk. Vectors are unit length, so the dot product is the cosine (phase 5 E3).
  */
-export const nearestChunks = ({ orm }, accountPk, { chatKey, model, since, limit, query, exclude, conversations, scope, projectId, before, personId, }) => {
-    const best = new Map();
-    let after = { conversation: 0, ordinal: -1 };
-    for (;;) {
-        const rows = orm.all(
-        // CROSS JOIN keeps the chunks first, so each page walks their key; led by the vectors, SQLite re-read and
-        // sorted every one of them per page — 1.3 s against 140 ms at 42k chunks (bench/embeddings/README.md).
-        sql `SELECT k.chunkable_id AS conversation, k.position AS ordinal, r.first_message_id AS first, r.last_message_id AS last,
+export const nearestChunks = (context, accountPk, filter) => {
+    const { orm } = context;
+    const { chatKey, model, since, limit, query, exclude, conversations, scope, projectId, before, personId } = filter;
+    const rows = (page) => orm.all(
+    // CROSS JOIN keeps the chunks first, so each page walks their key; led by the vectors, SQLite re-read and
+    // sorted every one of them per page — 1.3 s against 140 ms at 42k chunks (bench/embeddings/README.md).
+    sql `SELECT k.id, k.chunkable_id AS conversation, k.position AS ordinal, r.first_message_id AS first, r.last_message_id AS last,
           r.text_start AS start, r.text_end AS end, k.content_hash AS hash, v.vector FROM chunks k
         CROSS JOIN conversations c ON c.id = k.chunkable_id
         JOIN chunk_messages r ON r.chunk_id = k.id
@@ -180,29 +190,62 @@ export const nearestChunks = ({ orm }, accountPk, { chatKey, model, since, limit
           ${since === undefined ? sql `` : sql `AND k.occurred_at >= ${since}`}
           ${exclude === undefined ? sql `` : sql `AND c.id <> ${exclude}`}
           ${conversations === undefined ? sql `` : sql `AND c.id IN (SELECT value FROM json_each(${JSON.stringify(conversations)}))`}
-          AND (k.chunkable_id, k.position) > (${after.conversation}, ${after.ordinal})
-        ORDER BY k.chunkable_id, k.position LIMIT ${SCAN_PAGE}`);
-        for (const row of rows) {
-            const score = dot(query, row.vector);
-            const held = best.get(row.conversation);
-            if (!held || score > held.score) {
-                const range = rangeOf(row.start, row.end);
-                best.set(row.conversation, {
-                    conversationPk: row.conversation,
-                    firstMessagePk: row.first,
-                    lastMessagePk: row.last,
-                    ...(range ? { range } : {}),
-                    hash: row.hash,
-                    score,
-                });
-            }
-        }
-        const last = rows.at(-1);
-        if (!last || rows.length < SCAN_PAGE)
+          ${"ids" in page
+        ? sql `AND k.id IN (SELECT value FROM json_each(${JSON.stringify(page.ids)}))
+                  ORDER BY k.chunkable_id, k.position`
+        : sql `AND (k.chunkable_id, k.position) > (${page.after.conversation}, ${page.after.ordinal})
+                  ORDER BY k.chunkable_id, k.position LIMIT ${SCAN_PAGE}`}`);
+    const keepBest = (best, row) => {
+        const score = dot(query, row.vector);
+        const held = best.get(row.conversation);
+        if (held && score <= held.score)
+            return;
+        const range = rangeOf(row.start, row.end);
+        best.set(row.conversation, {
+            id: row.id,
+            conversationPk: row.conversation,
+            firstMessagePk: row.first,
+            lastMessagePk: row.last,
+            ...(range ? { range } : {}),
+            hash: row.hash,
+            score,
+        });
+    };
+    // Stable, so equal scores keep the scan's order, by conversation.
+    const ranked = (found) => found.sort((a, b) => b.score - a.score).slice(0, limit);
+    const plain = (found) => found.map(({ id: _, ...chunk }) => chunk);
+    // The index has no person, project, scope or conversation list; with one of those the scan stays.
+    const indexable = personId === undefined && projectId === undefined && scope === undefined && conversations === undefined;
+    const indexed = indexable &&
+        searchIndex(context.database, model, {
+            kind: "conversation",
+            account: accountPk,
+            ...(chatKey === undefined ? {} : { chat: chatKey }),
+            ...(since === undefined ? {} : { since }),
+            ...(before === undefined ? {} : { before }),
+        }, query, limit, (candidates) => {
+            const hashes = new Map(candidates.map(({ id, hash }) => [id, hash]));
+            const best = new Map();
+            for (const row of rows({ ids: [...hashes.keys()] }))
+                if (hashes.get(row.id) === row.hash)
+                    keepBest(best, row);
+            const results = ranked([...best.values()]);
+            return { results, best: results.map(({ id }) => id) };
+        });
+    if (indexed)
+        return plain(indexed);
+    const best = new Map();
+    let after = { conversation: 0, ordinal: -1 };
+    for (;;) {
+        const page = rows({ after });
+        for (const row of page)
+            keepBest(best, row);
+        const last = page.at(-1);
+        if (!last || page.length < SCAN_PAGE)
             break;
         after = { conversation: last.conversation, ordinal: last.ordinal };
     }
-    return [...best.values()].sort((a, b) => b.score - a.score).slice(0, limit);
+    return plain(ranked([...best.values()]));
 };
 const hasVectorOf = (models) => sql `EXISTS (SELECT 1 FROM chunks k
     JOIN conversations c ON k.chunkable_type = 'conversation' AND c.id = k.chunkable_id

package/dist/store/vector-extension.d.ts

@@ -0,0 +1,10 @@
+/**
+ * The sqlite-vec extension every store connection loads: the vector index's tables and triggers cannot be
+ * read or written without it. It ships in `@wirecat/cli-core-sqlite`, so a missing file is a broken install.
+ */
+export declare const vectorExtension: () => Promise<string>;
+/** Loads the extension into a connection that allows it, then closes the door for SQL's `load_extension()`. */
+export declare const loadVectorExtension: (database: {
+    loadExtension(path: string): void;
+    enableLoadExtension?(allow: boolean): void;
+}, path: string) => void;

package/dist/store/vector-extension.js

@@ -0,0 +1,53 @@
+import { readFileSync } from "node:fs";
+import { CliError } from "@wirecat/cli-base";
+const musl = () => {
+    if (process.platform !== "linux")
+        return false;
+    try {
+        return readFileSync("/proc/self/maps", "utf8").includes("ld-musl");
+    }
+    catch {
+        return false;
+    }
+};
+let found;
+/**
+ * The sqlite-vec extension every store connection loads: the vector index's tables and triggers cannot be
+ * read or written without it. It ships in `@wirecat/cli-core-sqlite`, so a missing file is a broken install.
+ */
+export const vectorExtension = () => {
+    found ??= locate();
+    return found;
+};
+const locate = async () => {
+    const onMusl = musl();
+    const target = `${process.platform}-${process.arch}${onMusl ? "-musl" : ""}`;
+    let path;
+    try {
+        const { extensionFor } = await import("@wirecat/cli-core-sqlite");
+        path = extensionFor({ musl: onMusl });
+    }
+    catch {
+        path = undefined;
+    }
+    if (path === undefined) {
+        found = undefined;
+        throw new CliError("configuration_error", `the message store needs the sqlite-vec extension, and @wirecat/cli-core-sqlite has none for ${target} — ` +
+            "reinstall the CLI so its dependencies install with it (an install that skips optional or native " +
+            "packages leaves it out)", { reason: "store.vector-extension-missing", target });
+    }
+    return path;
+};
+/** Loads the extension into a connection that allows it, then closes the door for SQL's `load_extension()`. */
+export const loadVectorExtension = (database, path) => {
+    try {
+        database.loadExtension(path);
+    }
+    catch (error) {
+        throw new CliError("configuration_error", `the sqlite-vec extension did not load on ${process.platform}-${process.arch} — reinstall the CLI; on macOS ` +
+            "under Bun, @wirecat/cli-core-sqlite's SQLite must be installed, since the system's cannot load extensions", { reason: "store.vector-extension-failed", cause: error instanceof Error ? error.message : String(error) });
+    }
+    finally {
+        database.enableLoadExtension?.(false);
+    }
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.433.0",
+  "version": "0.435.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",
@@ -177,7 +177,7 @@
     "@resvg/resvg-js": "2.6.2",
     "@valibot/to-json-schema": "1.8.0",
     "@wirecat/cli-core-onnx": "1.0.1",
-    "@wirecat/cli-core-sqlite": "1.0.1",
+    "@wirecat/cli-core-sqlite": "1.1.0",
     "@xmldom/xmldom": "0.9.12",
     "chardet": "2.2.0",
     "defuddle": "0.19.6",