@wirecat/cli-core 0.434.0 → 0.436.0

package/dist/store/sqlite/note-index.js

@@ -8,6 +8,7 @@ import { indexRow } from "./index-state.js";
 import { speech } from "./lucene-fields.js";
 import { savedStemmers } from "./stems.js";
 import { write } from "./transaction.js";
+import { dropUnusedVectors } from "./vector-index.js";
 export const CORPORA = {
     document: {
         type: "document",
@@ -114,8 +115,6 @@ const writer = (database, corpus, stemmer) => {
         stems: database.prepare(`INSERT INTO ${corpus.type}_stems (rowid, stems, scope) VALUES (?, ?, ?)`),
         chunk: database.prepare("INSERT INTO chunks (chunkable_type, chunkable_id, position, start_offset, end_offset, content_hash, scope, account_id, created_at, updated_at) " +
             "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"),
-        purge: database.prepare(`DELETE FROM embeddings WHERE content_hash = ?
-         AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ?)`),
         dequeue: database.prepare(`DELETE FROM ${corpus.type}_index_pending WHERE indexable_type = ? AND id = ?`),
     };
     return (id) => {
@@ -134,7 +133,6 @@ const writer = (database, corpus, stemmer) => {
         const stems = terms ? terms.stems : text ? stemmer.indexText(text) : "";
         if (stems !== "")
             statements.stems.run(id, stems, scope);
-        const after = new Set();
         if (text.trim() && corpus.pieces !== false) {
             const at = Date.now();
             const sections = row?.metadata == null ? [] : storedSections(String(row.metadata));
@@ -143,13 +141,10 @@ const writer = (database, corpus, stemmer) => {
                 : splitText(text, CHUNK_CHARS);
             ranges.forEach(({ start, end }, position) => {
                 const hash = chunkHash(text.slice(start, end));
-                after.add(hash);
                 statements.chunk.run(corpus.type, id, position, start, end, hash, scope, int(row?.account_id), at, at);
             });
         }
-        for (const hash of before)
-            if (!after.has(hash))
-                statements.purge.run(hash, hash);
+        dropUnusedVectors(database, before);
         if (!live && corpus.type === "document")
             dropDocumentPicture(database, id);
         statements.dequeue.run(queuedAs(corpus), id);

package/dist/store/sqlite/schema.d.ts

@@ -15535,7 +15535,7 @@ export declare const conversationState: import("drizzle-orm/sqlite-core").SQLite
     dialect: 'sqlite';
     isAlias: false;
 }>;
-/** One model's vector for one chunk text; no owner, so vectors outlive the builds and chunks that point at them. */
+/** One model's vector for one chunk text; no owner: chunks of any kind share it, and it goes once none holds the text. */
 export declare const embeddings: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
     name: "embeddings";
     schema: undefined;

package/dist/store/sqlite/schema.js

@@ -1556,7 +1556,7 @@ export const conversationState = sqliteTable("conversation_state", {
     /** The build readers see; a higher one is being written, or failed. */
     currentBuild: integer("current_build"),
 });
-/** One model's vector for one chunk text; no owner, so vectors outlive the builds and chunks that point at them. */
+/** One model's vector for one chunk text; no owner: chunks of any kind share it, and it goes once none holds the text. */
 export const embeddings = sqliteTable("embeddings", {
     /** `<provider>:<model>:<dims>` — vectors of different models never mix. */
     model: text("model").notNull(),

package/dist/store/sqlite/vector-index.d.ts

@@ -65,3 +65,12 @@ export declare const vectorIndexCounts: (database: CacheDatabase) => {
     missing: number;
     extra: number;
 }[];
+/**
+ * The one rule for a vector whose text may be gone: it goes once no chunk of any kind holds its hash, for every model.
+ * Call it with the hashes a write released, after that write's new chunks are in, so a text that comes back keeps its
+ * vector. The key is (model, content_hash): naming each model reaches the row by the key; the hash alone scans every
+ * vector.
+ */
+export declare const dropUnusedVectors: (database: CacheDatabase, hashes: Iterable<string>) => void;
+/** Every vector no chunk holds, left by a build before `dropUnusedVectors` covered every write; how many. */
+export declare const dropAllUnusedVectors: (database: CacheDatabase) => number;

package/dist/store/sqlite/vector-index.js

@@ -178,3 +178,28 @@ export const vectorIndexCounts = (database) => database.prepare("SELECT id, mode
         extra: count(`${held} EXCEPT ${wanted}`),
     };
 });
+/**
+ * The one rule for a vector whose text may be gone: it goes once no chunk of any kind holds its hash, for every model.
+ * Call it with the hashes a write released, after that write's new chunks are in, so a text that comes back keeps its
+ * vector. The key is (model, content_hash): naming each model reaches the row by the key; the hash alone scans every
+ * vector.
+ */
+export const dropUnusedVectors = (database, hashes) => {
+    const unique = new Set(hashes);
+    if (unique.size === 0)
+        return;
+    const models = database
+        .prepare(`WITH RECURSIVE m(model) AS (SELECT min(model) FROM embeddings
+         UNION ALL SELECT (SELECT min(model) FROM embeddings WHERE model > m.model) FROM m WHERE m.model IS NOT NULL)
+       SELECT model FROM m WHERE model IS NOT NULL`)
+        .all()
+        .map(({ model }) => String(model));
+    const drop = database.prepare("DELETE FROM embeddings WHERE model = ? AND content_hash = ? AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ?)");
+    for (const hash of unique)
+        for (const model of models)
+            drop.run(model, hash, hash);
+};
+/** Every vector no chunk holds, left by a build before `dropUnusedVectors` covered every write; how many. */
+export const dropAllUnusedVectors = (database) => database
+    .prepare("DELETE FROM embeddings WHERE NOT EXISTS (SELECT 1 FROM chunks k WHERE k.content_hash = embeddings.content_hash) RETURNING 1")
+    .all().length;

package/dist/store/sqlite/vectors.d.ts

@@ -50,8 +50,8 @@ export declare const vectorStatus: ({ orm }: StoreContext, chatKey: number, mode
     embedded: number;
 };
 /**
- * Drops the vectors of the chat's chunks, or only one model's — but not one another chat's chunk still
- * points at: the same text in two chats has one vector. Messages are never touched.
+ * Drops the vectors of the chat's conversation and message chunks, or only one model's — but not one a chunk outside
+ * the chat still points at: the same text in two chats has one vector. The chunks themselves stay, to be embedded again.
  */
 export declare const clearVectors: ({ orm }: StoreContext, chatKey: number, model: string | undefined) => number;
 export interface NearestChunk {

package/dist/store/sqlite/vectors.js

@@ -138,16 +138,19 @@ export const vectorStatus = ({ orm }, chatKey, model) => {
     return { chunks: Number(row?.chunks ?? 0), embedded: Number(row?.embedded ?? 0) };
 };
 /**
- * Drops the vectors of the chat's chunks, or only one model's — but not one another chat's chunk still
- * points at: the same text in two chats has one vector. Messages are never touched.
+ * Drops the vectors of the chat's conversation and message chunks, or only one model's — but not one a chunk outside
+ * the chat still points at: the same text in two chats has one vector. The chunks themselves stay, to be embedded again.
  */
-export const clearVectors = ({ orm }, chatKey, model) => orm.all(sql `DELETE FROM embeddings
+export const clearVectors = ({ orm }, chatKey, model) => orm.all(sql `WITH mine(id) AS (
+        SELECT k.id FROM chunks k JOIN conversations c ON k.chunkable_type = 'conversation' AND c.id = k.chunkable_id
+          WHERE c.chat_id = ${chatKey}
+        UNION ALL SELECT k.id FROM chunks k JOIN messages m ON k.chunkable_type = 'message' AND m.id = k.chunkable_id
+          WHERE m.chat_id = ${chatKey})
+      DELETE FROM embeddings
       WHERE ${model === undefined ? sql `1` : sql `model = ${model}`}
-        AND content_hash IN (SELECT k.content_hash FROM chunks k
-          JOIN conversations c ON k.chunkable_type = 'conversation' AND c.id = k.chunkable_id WHERE c.chat_id = ${chatKey})
-        AND content_hash NOT IN (SELECT k.content_hash FROM chunks k
-          LEFT JOIN conversations c ON k.chunkable_type = 'conversation' AND c.id = k.chunkable_id
-          WHERE c.id IS NULL OR c.chat_id <> ${chatKey})
+        AND content_hash IN (SELECT content_hash FROM chunks WHERE id IN (SELECT id FROM mine))
+        AND NOT EXISTS (SELECT 1 FROM chunks o WHERE o.content_hash = embeddings.content_hash
+          AND o.id NOT IN (SELECT id FROM mine))
       RETURNING 1 AS n`).length;
 /** Rows read per step of a search, so a chat of any size is scanned in bounded memory. */
 const SCAN_PAGE = 5_000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.434.0",
+  "version": "0.436.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",