@wirecat/cli-core 0.432.0 → 0.434.0

package/dist/store/sqlite/schema.js

@@ -1684,6 +1684,23 @@ export const mentionPending = sqliteTable("mention_pending", {
     sourceType: text("source_type").notNull(),
     sourceId: integer("source_id").notNull(),
 }, (table) => [primaryKey({ columns: [table.sourceType, table.sourceId] })]);
+/**
+ * One sqlite-vec `vec0` table per embedding model, `vector_index_<id>`, created at run time with its trigger since
+ * its width is the model's: a row per chunk with a vector of that model, rowid `chunks.id` (`vector-index.ts`).
+ */
+export const vectorIndexes = sqliteTable("vector_indexes", {
+    id: integer("id").primaryKey(),
+    model: text("model").notNull().unique(),
+    dims: integer("dims").notNull(),
+    /** 1 once it holds every vector; until then searches scan. */
+    ready: integer("ready").notNull(),
+    /** The last `content_hash` the fill of older vectors reached. */
+    filledThrough: text("filled_through").notNull(),
+});
+/** Always empty: each changed chunk's id passes through it so that every model's index re-derives its row. */
+export const vectorIndexRedo = sqliteTable("vector_index_redo", {
+    chunkId: integer("chunk_id").notNull(),
+});
 /** Settings of the store file itself, shared by every profile, tg and MAX — unlike a profile's config file. */
 export const storeSettings = sqliteTable("store_settings", {
     key: text("key").primaryKey(),

package/dist/store/sqlite/vector-index.d.ts

@@ -0,0 +1,67 @@
+import type { CacheDatabase } from "../driver.js";
+/**
+ * Every stored vector has length one, so the scan's dot product and the index's cosine rank alike. A model that
+ * already gives unit vectors (e5, the API models) changes by float rounding only. A zero vector (a text the model
+ * knows nothing of) stays zero: vec0 puts it first with no distance, so it is always among the candidates and scores
+ * 0 both ways.
+ */
+export declare const unitVector: (vector: Float32Array) => Float32Array;
+export interface VectorIndex {
+    table: string;
+    dims: number;
+    ready: boolean;
+}
+export declare const vectorIndexOf: (database: CacheDatabase, model: string) => VectorIndex | undefined;
+/**
+ * The model's index, made on its first vector. A model that already has vectors (an older store) is filled by
+ * `fillVectorIndexes` in short batches, and searched by scan until it is ready.
+ */
+export declare const ensureVectorIndex: (database: CacheDatabase, model: string, dims: number) => void;
+/**
+ * Indexes the vectors a store held before its index existed, in hash order from where the last batch stopped. Rows
+ * the triggers wrote meanwhile are skipped. Returns the models it filled.
+ */
+export declare const fillVectorIndexes: (database: CacheDatabase, { batch, notice }?: {
+    batch?: number;
+    notice?: (model: string) => void;
+}) => string[];
+export interface IndexFilter {
+    kind: string;
+    account: number;
+    chat?: number;
+    since?: number;
+    before?: number;
+}
+export interface Candidate {
+    id: number;
+    hash: string;
+    /** None for a zero vector. */
+    distance: number | null;
+}
+/**
+ * The best `limit` results by the index, or `undefined` when the caller must scan: no ready index for the model, or
+ * no exact answer within the most candidates vec0 returns. `survivors` applies the scan's own SQL to the candidates,
+ * in the scan's order (and checks each one's hash, so a reused chunk id is never taken for the old chunk), scores
+ * them, and names the chunks behind its best `limit` results. Those are exact once each is nearer than the last
+ * candidate by more than `NEAR_TIE`: every row nearer than that is among the candidates, and the rest cannot outrank
+ * them even where float32 and float64 disagree. A text many
+ * chunks share has one vector, so a tie that crosses the last candidate asks for more, as the scan's order decides it.
+ */
+export declare const searchIndex: <T>(database: CacheDatabase, model: string, filter: IndexFilter, query: Float32Array, limit: number, survivors: (candidates: Candidate[]) => {
+    results: T[];
+    best: number[];
+}) => T[] | undefined;
+/**
+ * Per model, `store check`'s safety net: the rows its index holds against the rows `source` derives now, compared
+ * whole (chunk, kind, account, chat, time, hash, vector) both ways. `missing` are rows the index lacks or holds
+ * wrong; `extra` rows it should not hold.
+ */
+export declare const vectorIndexCounts: (database: CacheDatabase) => {
+    model: string;
+    dims: number;
+    ready: boolean;
+    indexed: number;
+    expected: number;
+    missing: number;
+    extra: number;
+}[];

package/dist/store/sqlite/vector-index.js

@@ -0,0 +1,180 @@
+import { CliError } from "@wirecat/cli-base";
+import { writeBatch } from "./transaction.js";
+/**
+ * One sqlite-vec `vec0` table per embedding model, a row per chunk that has a vector of that model: rowid is
+ * `chunks.id`. The static triggers of store version 34 push every changed chunk id through `vector_index_redo`, and
+ * each model's own trigger re-derives that chunk's row, so every writer — the store's functions, the schema's own
+ * delete triggers, a raw `DELETE` — keeps the index in step inside its own transaction.
+ *
+ * `kind` and `account` are partition keys, `occurred_at` and `chat` metadata: the filters a KNN query applies before
+ * it ranks. `account`, `chat` and `occurred_at` are the columns each kind's scan filters on, so the index never
+ * leaves out a row the scan would keep; filters it cannot express are checked on the candidates after.
+ */
+const source = (model, dims) => `SELECT k.id, k.chunkable_type,
+    coalesce(CASE k.chunkable_type
+      WHEN 'image' THEN (SELECT ch.account_id FROM attachments a JOIN messages m ON m.id = a.attachable_id
+        JOIN chats ch ON ch.id = m.chat_id WHERE a.id = k.chunkable_id AND a.attachable_type = 'message')
+      WHEN 'conversation' THEN (SELECT ch.account_id FROM conversations c JOIN chats ch ON ch.id = c.chat_id
+        WHERE c.id = k.chunkable_id)
+      ELSE k.account_id END, 0),
+    coalesce(CASE k.chunkable_type
+      WHEN 'message' THEN (SELECT m.chat_id FROM messages m WHERE m.id = k.chunkable_id)
+      WHEN 'image' THEN (SELECT m.chat_id FROM attachments a JOIN messages m ON m.id = a.attachable_id
+        WHERE a.id = k.chunkable_id AND a.attachable_type = 'message')
+      WHEN 'conversation' THEN (SELECT c.chat_id FROM conversations c WHERE c.id = k.chunkable_id) END, 0),
+    coalesce(CASE k.chunkable_type
+      WHEN 'message' THEN (SELECT m.sent_at FROM messages m WHERE m.id = k.chunkable_id)
+      ELSE k.occurred_at END, 0),
+    v.vector, k.content_hash
+  FROM chunks k JOIN embeddings v ON v.model = ${literal(model)} AND v.content_hash = k.content_hash
+    AND length(v.vector) = ${dims * 4}`;
+/**
+ * Every stored vector has length one, so the scan's dot product and the index's cosine rank alike. A model that
+ * already gives unit vectors (e5, the API models) changes by float rounding only. A zero vector (a text the model
+ * knows nothing of) stays zero: vec0 puts it first with no distance, so it is always among the candidates and scores
+ * 0 both ways.
+ */
+export const unitVector = (vector) => {
+    let sum = 0;
+    for (const value of vector)
+        sum += value * value;
+    const length = Math.sqrt(sum);
+    if (!Number.isFinite(length))
+        throw new CliError("validation_error", "a vector must have finite values", { reason: "store.vector-invalid" });
+    return length === 0 ? vector : vector.map((value) => value / length);
+};
+const literal = (text) => `'${text.replaceAll("'", "''")}'`;
+const COLUMNS = "rowid, kind, account, chat, occurred_at, embedding, hash";
+const tableOf = (id) => `vector_index_${id}`;
+export const vectorIndexOf = (database, model) => {
+    const row = database.prepare("SELECT id, dims, ready FROM vector_indexes WHERE model = ?").get(model);
+    return row && { table: tableOf(Number(row.id)), dims: Number(row.dims), ready: row.ready === 1 };
+};
+/**
+ * The model's index, made on its first vector. A model that already has vectors (an older store) is filled by
+ * `fillVectorIndexes` in short batches, and searched by scan until it is ready.
+ */
+export const ensureVectorIndex = (database, model, dims) => {
+    if (vectorIndexOf(database, model))
+        return;
+    const held = database.prepare("SELECT 1 FROM embeddings WHERE model = ? LIMIT 1").get(model) !== undefined;
+    const id = Number(database
+        .prepare("INSERT INTO vector_indexes (model, dims, ready, filled_through) VALUES (?, ?, ?, '') RETURNING id")
+        .get(model, dims, held ? 0 : 1)?.id);
+    const table = tableOf(id);
+    database.exec(`CREATE VIRTUAL TABLE ${table} USING vec0(kind text partition key, account integer partition key,
+    chat integer, occurred_at integer, embedding float[${dims}] distance_metric=cosine, +hash text)`);
+    database.exec(`CREATE TRIGGER ${table}_redo AFTER INSERT ON vector_index_redo BEGIN
+    DELETE FROM ${table} WHERE rowid = new.chunk_id;
+    INSERT INTO ${table} (${COLUMNS}) ${source(model, dims)} WHERE k.id = new.chunk_id;
+  END`);
+};
+/** The models with vectors and no index, by a walk of the primary key rather than a scan of every vector. */
+const modelsWithoutIndex = (database) => database
+    .prepare(`WITH RECURSIVE m(model) AS (SELECT min(model) FROM embeddings
+         UNION ALL SELECT (SELECT min(model) FROM embeddings WHERE model > m.model) FROM m WHERE m.model IS NOT NULL)
+       SELECT m.model, (SELECT dims FROM embeddings WHERE model = m.model LIMIT 1) AS dims FROM m
+       WHERE m.model IS NOT NULL AND NOT EXISTS (SELECT 1 FROM vector_indexes i WHERE i.model = m.model)`)
+    .all();
+/** Hashes per batch: a batch is one short write transaction, so other processes wait less than their timeout. */
+const FILL_HASHES = 1_000;
+/**
+ * Indexes the vectors a store held before its index existed, in hash order from where the last batch stopped. Rows
+ * the triggers wrote meanwhile are skipped. Returns the models it filled.
+ */
+export const fillVectorIndexes = (database, { batch = FILL_HASHES, notice = () => { } } = {}) => {
+    for (const { model, dims } of modelsWithoutIndex(database))
+        writeBatch(database, () => ensureVectorIndex(database, model, Number(dims)));
+    const pending = database
+        .prepare("SELECT id, model, dims, filled_through FROM vector_indexes WHERE ready = 0 ORDER BY id")
+        .all();
+    for (const { id, model, dims, filled_through } of pending) {
+        notice(model);
+        const table = tableOf(id);
+        const step = database.prepare(`INSERT INTO ${table} (${COLUMNS}) ${source(model, dims)}
+        WHERE k.content_hash IN (SELECT value FROM json_each(?)) AND NOT EXISTS (SELECT 1 FROM ${table} WHERE rowid = k.id)`);
+        const hashes = database.prepare("SELECT content_hash AS hash FROM embeddings WHERE model = ? AND content_hash > ? ORDER BY content_hash LIMIT ?");
+        let after = filled_through;
+        for (;;) {
+            const done = writeBatch(database, () => {
+                const page = hashes.all(model, after, batch).map(({ hash }) => hash);
+                if (page.length > 0)
+                    step.run(JSON.stringify(page));
+                after = page.at(-1) ?? after;
+                const finished = page.length < batch;
+                // Another process may be filling too: neither moves the other back, nor un-readies a finished index.
+                database
+                    .prepare("UPDATE vector_indexes SET filled_through = ?, ready = ? WHERE id = ? AND ready = 0 AND filled_through <= ?")
+                    .run(after, finished ? 1 : 0, id, after);
+                return finished;
+            });
+            if (done)
+                break;
+        }
+    }
+    return pending.map(({ model }) => model);
+};
+/**
+ * vec0 measures in float32 and the scan in float64, so two results closer than this may swap. Far above float32's
+ * error at these lengths, far below any distance that matters for ranking.
+ */
+const NEAR_TIE = 1e-5;
+/** vec0 refuses a k above this. */
+const MAX_K = 4_096;
+const knn = (database, index, filter, query, k) => database
+    .prepare(`SELECT rowid AS id, hash, distance FROM ${index.table} WHERE embedding MATCH ? AND k = ? AND kind = ? AND account = ?
+        ${filter.chat === undefined ? "" : "AND chat = ?"}
+        ${filter.since === undefined ? "" : "AND occurred_at >= ?"}
+        ${filter.before === undefined ? "" : "AND occurred_at < ?"}`)
+    .all(new Uint8Array(query.buffer, query.byteOffset, query.byteLength), BigInt(k), filter.kind, 
+// node:sqlite binds a JS number as REAL, which vec0's integer columns refuse.
+BigInt(filter.account), ...[filter.chat, filter.since, filter.before].flatMap((value) => (value === undefined ? [] : [BigInt(value)])));
+/**
+ * The best `limit` results by the index, or `undefined` when the caller must scan: no ready index for the model, or
+ * no exact answer within the most candidates vec0 returns. `survivors` applies the scan's own SQL to the candidates,
+ * in the scan's order (and checks each one's hash, so a reused chunk id is never taken for the old chunk), scores
+ * them, and names the chunks behind its best `limit` results. Those are exact once each is nearer than the last
+ * candidate by more than `NEAR_TIE`: every row nearer than that is among the candidates, and the rest cannot outrank
+ * them even where float32 and float64 disagree. A text many
+ * chunks share has one vector, so a tie that crosses the last candidate asks for more, as the scan's order decides it.
+ */
+export const searchIndex = (database, model, filter, query, limit, survivors) => {
+    const index = vectorIndexOf(database, model);
+    if (!index?.ready || index.dims !== query.length)
+        return undefined;
+    let k = Math.min(MAX_K, Math.max(100, limit * 4));
+    for (;;) {
+        const candidates = knn(database, index, filter, query, k);
+        const { results, best } = survivors(candidates);
+        if (candidates.length < k)
+            return results;
+        const edge = candidates.at(-1)?.distance ?? null;
+        const distance = new Map(candidates.map((candidate) => [candidate.id, candidate.distance]));
+        // A last candidate with no distance is a zero vector: nothing is known to be nearer yet.
+        const nearer = (id) => edge !== null && (distance.get(id) ?? edge) < edge - NEAR_TIE;
+        if (best.length >= limit && best.every(nearer))
+            return results;
+        if (k === MAX_K)
+            return undefined;
+        k = Math.min(MAX_K, k * 4);
+    }
+};
+/**
+ * Per model, `store check`'s safety net: the rows its index holds against the rows `source` derives now, compared
+ * whole (chunk, kind, account, chat, time, hash, vector) both ways. `missing` are rows the index lacks or holds
+ * wrong; `extra` rows it should not hold.
+ */
+export const vectorIndexCounts = (database) => database.prepare("SELECT id, model, dims, ready FROM vector_indexes ORDER BY id").all().map(({ id, model, dims, ready }) => {
+    const held = `SELECT ${COLUMNS} FROM ${tableOf(id)}`;
+    const wanted = source(model, Number(dims));
+    const count = (sql) => Number(database.prepare(`SELECT count(*) AS n FROM (${sql})`).get()?.n ?? 0);
+    return {
+        model,
+        dims: Number(dims),
+        ready: ready === 1,
+        indexed: count(held),
+        expected: count(wanted),
+        missing: count(`${wanted} EXCEPT ${held}`),
+        extra: count(`${held} EXCEPT ${wanted}`),
+    };
+});

package/dist/store/sqlite/vectors.d.ts

@@ -36,8 +36,11 @@ export declare const conversationVectors: (context: StoreContext, conversationPk
     chunks: number;
     vectors: Float32Array[];
 };
-/** A vector already there for the same model and text is kept: the same text gives the same vector. */
-export declare const saveVectors: ({ orm, now }: StoreContext, model: string, dims: number, vectors: {
+/**
+ * A vector already there for the same model and text is kept: the same text gives the same vector. Each is stored at
+ * length one; a zero or non-finite vector is refused.
+ */
+export declare const saveVectors: ({ orm, now, database }: StoreContext, model: string, dims: number, vectors: {
     hash: string;
     vector: Float32Array;
 }[]) => void;
@@ -65,7 +68,7 @@ export declare const dot: (query: Float32Array, blob: Uint8Array) => number;
  * The conversations of the current builds in scope nearest to `query`, best first: each scored by its
  * best chunk. Vectors are unit length, so the dot product is the cosine (phase 5 E3).
  */
-export declare const nearestChunks: ({ orm }: StoreContext, accountPk: number, { chatKey, model, since, limit, query, exclude, conversations, scope, projectId, before, personId, }: {
+export declare const nearestChunks: (context: StoreContext, accountPk: number, filter: {
     chatKey?: number;
     model: string;
     since?: number;

package/dist/store/sqlite/vectors.js

@@ -1,5 +1,6 @@
 import { chunkHash, chunkTextOf } from "../../conversations/chunks.js";
 import { sql } from "./drizzle/core.js";
+import { ensureVectorIndex, searchIndex, unitVector } from "./vector-index.js";
 const currentChunks = (chatKey) => sql `SELECT k.content_hash, k.chunkable_id AS conversation_id, r.first_message_id,
     r.last_message_id, r.text_start, r.text_end FROM chunks k JOIN chunk_messages r ON r.chunk_id = k.id
     JOIN conversations c ON k.chunkable_type = 'conversation' AND c.id = k.chunkable_id
@@ -113,10 +114,17 @@ export const conversationVectors = (context, conversationPk, model) => {
         : []);
     return { chunks: rows.length, vectors };
 };
-/** A vector already there for the same model and text is kept: the same text gives the same vector. */
-export const saveVectors = ({ orm, now }, model, dims, vectors) => {
+/**
+ * A vector already there for the same model and text is kept: the same text gives the same vector. Each is stored at
+ * length one; a zero or non-finite vector is refused.
+ */
+export const saveVectors = ({ orm, now, database }, model, dims, vectors) => {
+    if (vectors.length === 0)
+        return;
+    ensureVectorIndex(database, model, dims);
     const at = now();
-    for (const { hash, vector } of vectors) {
+    for (const { hash, vector: given } of vectors) {
+        const vector = unitVector(given);
         orm.run(sql `INSERT INTO embeddings (model, content_hash, dims, vector, created_at, updated_at)
         VALUES (${model}, ${hash}, ${dims}, ${Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength)}, ${at}, ${at})
         ON CONFLICT DO NOTHING`);
@@ -157,14 +165,13 @@ export const dot = (query, blob) => {
  * The conversations of the current builds in scope nearest to `query`, best first: each scored by its
  * best chunk. Vectors are unit length, so the dot product is the cosine (phase 5 E3).
  */
-export const nearestChunks = ({ orm }, accountPk, { chatKey, model, since, limit, query, exclude, conversations, scope, projectId, before, personId, }) => {
-    const best = new Map();
-    let after = { conversation: 0, ordinal: -1 };
-    for (;;) {
-        const rows = orm.all(
-        // CROSS JOIN keeps the chunks first, so each page walks their key; led by the vectors, SQLite re-read and
-        // sorted every one of them per page — 1.3 s against 140 ms at 42k chunks (bench/embeddings/README.md).
-        sql `SELECT k.chunkable_id AS conversation, k.position AS ordinal, r.first_message_id AS first, r.last_message_id AS last,
+export const nearestChunks = (context, accountPk, filter) => {
+    const { orm } = context;
+    const { chatKey, model, since, limit, query, exclude, conversations, scope, projectId, before, personId } = filter;
+    const rows = (page) => orm.all(
+    // CROSS JOIN keeps the chunks first, so each page walks their key; led by the vectors, SQLite re-read and
+    // sorted every one of them per page — 1.3 s against 140 ms at 42k chunks (bench/embeddings/README.md).
+    sql `SELECT k.id, k.chunkable_id AS conversation, k.position AS ordinal, r.first_message_id AS first, r.last_message_id AS last,
           r.text_start AS start, r.text_end AS end, k.content_hash AS hash, v.vector FROM chunks k
         CROSS JOIN conversations c ON c.id = k.chunkable_id
         JOIN chunk_messages r ON r.chunk_id = k.id
@@ -180,29 +187,62 @@ export const nearestChunks = ({ orm }, accountPk, { chatKey, model, since, limit
           ${since === undefined ? sql `` : sql `AND k.occurred_at >= ${since}`}
           ${exclude === undefined ? sql `` : sql `AND c.id <> ${exclude}`}
           ${conversations === undefined ? sql `` : sql `AND c.id IN (SELECT value FROM json_each(${JSON.stringify(conversations)}))`}
-          AND (k.chunkable_id, k.position) > (${after.conversation}, ${after.ordinal})
-        ORDER BY k.chunkable_id, k.position LIMIT ${SCAN_PAGE}`);
-        for (const row of rows) {
-            const score = dot(query, row.vector);
-            const held = best.get(row.conversation);
-            if (!held || score > held.score) {
-                const range = rangeOf(row.start, row.end);
-                best.set(row.conversation, {
-                    conversationPk: row.conversation,
-                    firstMessagePk: row.first,
-                    lastMessagePk: row.last,
-                    ...(range ? { range } : {}),
-                    hash: row.hash,
-                    score,
-                });
-            }
-        }
-        const last = rows.at(-1);
-        if (!last || rows.length < SCAN_PAGE)
+          ${"ids" in page
+        ? sql `AND k.id IN (SELECT value FROM json_each(${JSON.stringify(page.ids)}))
+                  ORDER BY k.chunkable_id, k.position`
+        : sql `AND (k.chunkable_id, k.position) > (${page.after.conversation}, ${page.after.ordinal})
+                  ORDER BY k.chunkable_id, k.position LIMIT ${SCAN_PAGE}`}`);
+    const keepBest = (best, row) => {
+        const score = dot(query, row.vector);
+        const held = best.get(row.conversation);
+        if (held && score <= held.score)
+            return;
+        const range = rangeOf(row.start, row.end);
+        best.set(row.conversation, {
+            id: row.id,
+            conversationPk: row.conversation,
+            firstMessagePk: row.first,
+            lastMessagePk: row.last,
+            ...(range ? { range } : {}),
+            hash: row.hash,
+            score,
+        });
+    };
+    // Stable, so equal scores keep the scan's order, by conversation.
+    const ranked = (found) => found.sort((a, b) => b.score - a.score).slice(0, limit);
+    const plain = (found) => found.map(({ id: _, ...chunk }) => chunk);
+    // The index has no person, project, scope or conversation list; with one of those the scan stays.
+    const indexable = personId === undefined && projectId === undefined && scope === undefined && conversations === undefined;
+    const indexed = indexable &&
+        searchIndex(context.database, model, {
+            kind: "conversation",
+            account: accountPk,
+            ...(chatKey === undefined ? {} : { chat: chatKey }),
+            ...(since === undefined ? {} : { since }),
+            ...(before === undefined ? {} : { before }),
+        }, query, limit, (candidates) => {
+            const hashes = new Map(candidates.map(({ id, hash }) => [id, hash]));
+            const best = new Map();
+            for (const row of rows({ ids: [...hashes.keys()] }))
+                if (hashes.get(row.id) === row.hash)
+                    keepBest(best, row);
+            const results = ranked([...best.values()]);
+            return { results, best: results.map(({ id }) => id) };
+        });
+    if (indexed)
+        return plain(indexed);
+    const best = new Map();
+    let after = { conversation: 0, ordinal: -1 };
+    for (;;) {
+        const page = rows({ after });
+        for (const row of page)
+            keepBest(best, row);
+        const last = page.at(-1);
+        if (!last || page.length < SCAN_PAGE)
             break;
         after = { conversation: last.conversation, ordinal: last.ordinal };
     }
-    return [...best.values()].sort((a, b) => b.score - a.score).slice(0, limit);
+    return plain(ranked([...best.values()]));
 };
 const hasVectorOf = (models) => sql `EXISTS (SELECT 1 FROM chunks k
     JOIN conversations c ON k.chunkable_type = 'conversation' AND c.id = k.chunkable_id

package/dist/store/vector-extension.d.ts

@@ -0,0 +1,10 @@
+/**
+ * The sqlite-vec extension every store connection loads: the vector index's tables and triggers cannot be
+ * read or written without it. It ships in `@wirecat/cli-core-sqlite`, so a missing file is a broken install.
+ */
+export declare const vectorExtension: () => Promise<string>;
+/** Loads the extension into a connection that allows it, then closes the door for SQL's `load_extension()`. */
+export declare const loadVectorExtension: (database: {
+    loadExtension(path: string): void;
+    enableLoadExtension?(allow: boolean): void;
+}, path: string) => void;

package/dist/store/vector-extension.js

@@ -0,0 +1,53 @@
+import { readFileSync } from "node:fs";
+import { CliError } from "@wirecat/cli-base";
+const musl = () => {
+    if (process.platform !== "linux")
+        return false;
+    try {
+        return readFileSync("/proc/self/maps", "utf8").includes("ld-musl");
+    }
+    catch {
+        return false;
+    }
+};
+let found;
+/**
+ * The sqlite-vec extension every store connection loads: the vector index's tables and triggers cannot be
+ * read or written without it. It ships in `@wirecat/cli-core-sqlite`, so a missing file is a broken install.
+ */
+export const vectorExtension = () => {
+    found ??= locate();
+    return found;
+};
+const locate = async () => {
+    const onMusl = musl();
+    const target = `${process.platform}-${process.arch}${onMusl ? "-musl" : ""}`;
+    let path;
+    try {
+        const { extensionFor } = await import("@wirecat/cli-core-sqlite");
+        path = extensionFor({ musl: onMusl });
+    }
+    catch {
+        path = undefined;
+    }
+    if (path === undefined) {
+        found = undefined;
+        throw new CliError("configuration_error", `the message store needs the sqlite-vec extension, and @wirecat/cli-core-sqlite has none for ${target} — ` +
+            "reinstall the CLI so its dependencies install with it (an install that skips optional or native " +
+            "packages leaves it out)", { reason: "store.vector-extension-missing", target });
+    }
+    return path;
+};
+/** Loads the extension into a connection that allows it, then closes the door for SQL's `load_extension()`. */
+export const loadVectorExtension = (database, path) => {
+    try {
+        database.loadExtension(path);
+    }
+    catch (error) {
+        throw new CliError("configuration_error", `the sqlite-vec extension did not load on ${process.platform}-${process.arch} — reinstall the CLI; on macOS ` +
+            "under Bun, @wirecat/cli-core-sqlite's SQLite must be installed, since the system's cannot load extensions", { reason: "store.vector-extension-failed", cause: error instanceof Error ? error.message : String(error) });
+    }
+    finally {
+        database.enableLoadExtension?.(false);
+    }
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.432.0",
+  "version": "0.434.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",
@@ -16,7 +16,7 @@
   "engines": {
     "node": "^22.16.0 || >=24"
   },
-  "packageManager": "pnpm@11.20.0",
+  "packageManager": "pnpm@11.28.8",
   "files": [
     "dist",
     "parity.json",
@@ -170,17 +170,17 @@
     "help:examples": "HELP_EXAMPLES_REPORT=1 vitest run scripts/help-examples.test.ts --silent=false"
   },
   "dependencies": {
-    "@anthropic-ai/sdk": "0.131.0",
+    "@anthropic-ai/sdk": "1.3.0",
     "@bomb.sh/tab": "0.0.22",
     "@huggingface/tokenizers": "0.2.0",
-    "@modelcontextprotocol/server": "2.3.0",
+    "@modelcontextprotocol/server": "^2.4.0",
     "@resvg/resvg-js": "2.6.2",
     "@valibot/to-json-schema": "1.8.0",
     "@wirecat/cli-core-onnx": "1.0.1",
-    "@wirecat/cli-core-sqlite": "1.0.1",
+    "@wirecat/cli-core-sqlite": "1.1.0",
     "@xmldom/xmldom": "0.9.12",
     "chardet": "2.2.0",
-    "defuddle": "0.19.5",
+    "defuddle": "0.19.6",
     "dictionary-en": "4.0.0",
     "dictionary-es": "4.0.0",
     "dictionary-ru": "3.0.0",
@@ -188,9 +188,9 @@
     "fflate": "0.8.3",
     "iconv-lite": "0.7.3",
     "image-size": "2.0.4",
-    "linkedom": "0.18.13",
+    "linkedom": "0.18.14",
     "liquidjs": "10.30.0",
-    "lossless-json": "4.3.0",
+    "lossless-json": "4.3.3",
     "mammoth": "1.13.0",
     "mdast-util-from-markdown": "2.1.0",
     "mdast-util-gfm": "3.1.0",
@@ -198,40 +198,39 @@
     "ogg-opus-decoder": "1.7.5",
     "psl": "1.15.0",
     "qrcode-generator": "2.0.4",
-    "sherpa-onnx": "1.13.8",
+    "sherpa-onnx": "2.0.0",
     "string-width": "8.3.0",
-    "unpdf": "1.8.1",
+    "unpdf": "1.9.1",
     "valibot": "1.5.0",
     "word-extractor": "1.0.4",
     "wrap-ansi": "10.0.2",
     "yaml": "2.9.1"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.5.15",
-    "@cspell/dict-en-gb": "5.0.34",
+    "@biomejs/biome": "2.6.3",
+    "@cspell/dict-en-gb": "5.0.38",
     "@cspell/dict-es-es": "3.0.9",
-    "@cspell/dict-ru_ru": "2.3.3",
-    "@modelcontextprotocol/client": "2.3.0",
-    "@napi-rs/canvas": "1.0.10",
+    "@cspell/dict-ru_ru": "2.3.4",
+    "@modelcontextprotocol/client": "2.4.0",
+    "@napi-rs/canvas": "1.0.13",
     "@types/mdast": "4.0.4",
-    "@types/node": "^22.20.4",
+    "@types/node": "^22.20.5",
     "@types/pngjs": "6.0.5",
-    "@vitest/coverage-v8": "^5.0.2",
+    "@vitest/coverage-v8": "^5.0.5",
     "@wirecat/cli-base": "0.57.0",
     "@wirecat/cli-core-legacy-tags": "npm:@wirecat/cli-core@0.414.0",
     "commander": "15.0.0",
-    "cspell": "10.3.6",
-    "drizzle-kit": "1.0.0-rc.4",
-    "drizzle-orm": "1.0.0-rc.4",
+    "cspell": "10.4.0",
+    "drizzle-kit": "1.0.0-rc.5",
+    "drizzle-orm": "1.0.0-rc.5",
     "esbuild": "0.28.2",
-    "exceljs": "4.4.0",
     "jsqr": "1.4.0",
-    "lefthook": "2.1.16",
-    "libphonenumber-js": "1.13.14",
-    "markdownlint-cli2": "0.23.3",
+    "lefthook": "2.2.2",
+    "libphonenumber-js": "1.13.15",
+    "markdownlint-cli2": "0.24.0",
     "pngjs": "7.0.0",
     "typescript": "^7.0.2",
-    "vitest": "^5.0.1"
+    "vitest": "^5.0.5"
   },
   "publishConfig": {
     "access": "public"