@wirecat/cli-core 0.20.0 → 0.262.0

package/dist/store/sqlite/note-index.js

@@ -0,0 +1,178 @@
+import { CHUNK_CHARS, chunkHash, splitSections, splitText } from "../../conversations/chunks.js";
+import { analyzerIdentity, DEFAULT_STEMMERS } from "../../search/stem.js";
+import { normalize } from "../normalize.js";
+import { int } from "./events.js";
+import { inBatch, indexRow } from "./search-index.js";
+import { savedStemmers } from "./stems.js";
+export const CORPORA = {
+    document: {
+        type: "document",
+        index: "document_index",
+        table: "documents",
+        read: "SELECT d.title, coalesce(d.body, '') AS body, d.metadata, d.deleted_at IS NULL AS live, a.scope, d.account_id FROM documents d " +
+            "JOIN accounts a ON a.id = d.account_id WHERE d.id = ?",
+    },
+    note: {
+        type: "note",
+        index: "note_index",
+        table: "notes",
+        read: "SELECT title, body, deleted_at IS NULL AS live, 'personal' AS scope FROM notes WHERE id = ?",
+    },
+    memory: {
+        type: "memory",
+        index: "memory_index",
+        table: "memories",
+        read: "SELECT NULL AS title, body, status <> 'superseded' AS live, scope FROM memories WHERE id = ?",
+    },
+    email: {
+        type: "email",
+        index: "email_index",
+        table: "emails",
+        read: "SELECT e.subject AS title, coalesce(e.body_text, '') AS body, e.deleted_at IS NULL AS live, a.scope, e.account_id FROM emails e " +
+            "JOIN accounts a ON a.id = e.account_id WHERE e.id = ?",
+        pieces: false,
+    },
+};
+/** What a row is indexed as: its title, then its text. Chunk offsets point into this. */
+export const noteIndexText = (title, text) => (title ? `${title}\n\n${text}` : text);
+export const corpusIndexState = (database, corpus) => {
+    const row = indexRow(database, corpus.index);
+    const pending = Number(database.prepare(`SELECT count(*) AS n FROM ${corpus.type}_index_pending WHERE indexable_type = ?`).get(corpus.type)
+        ?.n);
+    const saved = savedStemmers(database);
+    const wanted = saved === null ? null : analyzerIdentity(saved ?? DEFAULT_STEMMERS);
+    const built = row.analyzer === null ? null : String(row.analyzer);
+    return { pending, built, wanted, ready: pending === 0 && wanted !== null && (built === wanted || built === null) };
+};
+/** Documents and notes together: what a notes search reads. */
+export const noteIndexState = (database) => {
+    const documents = corpusIndexState(database, CORPORA.document);
+    const notes = corpusIndexState(database, CORPORA.note);
+    return {
+        pending: documents.pending + notes.pending,
+        built: notes.built ?? documents.built,
+        wanted: notes.wanted,
+        ready: documents.ready && notes.ready,
+    };
+};
+const storedSections = (metadata) => {
+    try {
+        const { sections } = JSON.parse(metadata);
+        return (sections ?? []).filter((section) => Number.isInteger(section.start));
+    }
+    catch {
+        return [];
+    }
+};
+const writer = (database, corpus, stemmer) => {
+    const read = database.prepare(corpus.read);
+    const oldHashes = database.prepare("SELECT content_hash AS hash FROM chunks WHERE chunkable_type = ? AND chunkable_id = ?");
+    const statements = {
+        dropWords: database.prepare(`DELETE FROM ${corpus.type}_words WHERE rowid = ?`),
+        dropStems: database.prepare(`DELETE FROM ${corpus.type}_stems WHERE rowid = ?`),
+        dropChunks: database.prepare("DELETE FROM chunks WHERE chunkable_type = ? AND chunkable_id = ?"),
+        words: database.prepare(`INSERT INTO ${corpus.type}_words (rowid, normalized_text, scope) VALUES (?, ?, ?)`),
+        stems: database.prepare(`INSERT INTO ${corpus.type}_stems (rowid, stems, scope) VALUES (?, ?, ?)`),
+        chunk: database.prepare("INSERT INTO chunks (chunkable_type, chunkable_id, position, start_offset, end_offset, content_hash, scope, account_id, created_at, updated_at) " +
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"),
+        purge: database.prepare(`DELETE FROM embeddings WHERE content_hash = ?
+         AND NOT EXISTS (SELECT 1 FROM chunks WHERE content_hash = ?)`),
+        dequeue: database.prepare(`DELETE FROM ${corpus.type}_index_pending WHERE indexable_type = ? AND id = ?`),
+    };
+    return (id) => {
+        const row = read.get(id);
+        const before = oldHashes.all(corpus.type, id).map((hash) => String(hash.hash));
+        statements.dropWords.run(id);
+        statements.dropStems.run(id);
+        statements.dropChunks.run(corpus.type, id);
+        const live = row !== undefined && Number(row.live) === 1;
+        const text = live ? noteIndexText(row.title == null ? null : String(row.title), String(row.body)) : "";
+        const scope = live ? String(row.scope ?? "personal") : "";
+        const words = normalize(text);
+        if (words !== "")
+            statements.words.run(id, words, scope);
+        const stems = text ? stemmer.indexText(text) : "";
+        if (stems !== "")
+            statements.stems.run(id, stems, scope);
+        const after = new Set();
+        if (text.trim() && corpus.pieces !== false) {
+            const at = Date.now();
+            const sections = row?.metadata == null ? [] : storedSections(String(row.metadata));
+            const ranges = sections.length
+                ? splitSections(text, sections, CHUNK_CHARS, text.length - String(row?.body ?? "").length)
+                : splitText(text, CHUNK_CHARS);
+            ranges.forEach(({ start, end }, position) => {
+                const hash = chunkHash(text.slice(start, end));
+                after.add(hash);
+                statements.chunk.run(corpus.type, id, position, start, end, hash, scope, int(row?.account_id), at, at);
+            });
+        }
+        for (const hash of before)
+            if (!after.has(hash))
+                statements.purge.run(hash, hash);
+        statements.dequeue.run(corpus.type, id);
+    };
+};
+/**
+ * Indexes one corpus's rows written since the last drain, in short write transactions. These corpora are
+ * small, so a change of stemmer choices simply queues every row again instead of waiting for `store reindex`.
+ */
+export const drainCorpus = (database, corpus, stemmerFor, { batch = 200, until = () => false } = {}) => {
+    // Pieces written before they carried their account get it, once: purge and vector search filter by it.
+    if (corpus.type === "email" || corpus.type === "document")
+        database
+            .prepare(`UPDATE chunks SET account_id = (SELECT account_id FROM ${corpus.table} WHERE id = chunks.chunkable_id)
+          WHERE chunkable_type = ? AND account_id IS NULL`)
+            .run(corpus.type);
+    // The first schema seeded no row for mail; a data row, so no migration.
+    database
+        .prepare("INSERT OR IGNORE INTO search_index_state (name, watermark, filled_through, terms_through, normalizer_version) VALUES (?, 0, 0, 0, 1)")
+        .run(corpus.index);
+    const state = corpusIndexState(database, corpus);
+    if (state.wanted === null)
+        return 0;
+    const stemmer = stemmerFor(savedStemmers(database) ?? DEFAULT_STEMMERS);
+    const queue = `${corpus.type}_index_pending`;
+    if (state.built !== stemmer.identity) {
+        inBatch(database, () => {
+            if (state.built !== null)
+                database
+                    .prepare(`INSERT OR IGNORE INTO ${queue} (indexable_type, id) SELECT ?, id FROM ${corpus.table}`)
+                    .run(corpus.type);
+            database.prepare("UPDATE search_index_state SET analyzer = ? WHERE name = ?").run(stemmer.identity, corpus.index);
+        });
+    }
+    if (state.pending === 0 && state.built === stemmer.identity)
+        return 0;
+    const next = database.prepare(`SELECT id FROM ${queue} WHERE indexable_type = ? ORDER BY id LIMIT ?`);
+    const index = writer(database, corpus, stemmer);
+    let done = 0;
+    for (;;) {
+        if (until())
+            break;
+        const count = inBatch(database, () => {
+            const ids = next.all(corpus.type, batch).map((row) => Number(row.id));
+            for (const id of ids)
+                index(id);
+            return ids.length;
+        });
+        if (count === 0)
+            break;
+        done += count;
+    }
+    return done;
+};
+/** Documents, then notes: what a notes search reads. */
+export const drainNoteIndex = (database, stemmerFor, options = {}) => drainCorpus(database, CORPORA.document, stemmerFor, options) +
+    drainCorpus(database, CORPORA.note, stemmerFor, options);
+/** Queues every document, note and memory again — for `store reindex`. */
+export const resetNoteIndex = (database) => {
+    inBatch(database, () => {
+        for (const corpus of Object.values(CORPORA))
+            database
+                .prepare(`INSERT OR IGNORE INTO ${corpus.type}_index_pending (indexable_type, id) SELECT ?, id FROM ${corpus.table}`)
+                .run(corpus.type);
+    });
+    return true;
+};
+//# sourceMappingURL=note-index.js.map

package/dist/store/sqlite/note-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"note-index.js","sourceRoot":"","sources":["../../../src/store/sqlite/note-index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAA;AAChG,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAA+B,MAAM,sBAAsB,CAAA;AAEtG,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AA4B1C,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,QAAQ,EAAE;QACR,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,gBAAgB;QACvB,KAAK,EAAE,WAAW;QAClB,IAAI,EACF,iIAAiI;YACjI,uDAAuD;KAC1D;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,YAAY;QACnB,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,6FAA6F;KACpG;IACD,MAAM,EAAE;QACN,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,cAAc;QACrB,KAAK,EAAE,UAAU;QACjB,IAAI,EAAE,8FAA8F;KACrG;IACD,KAAK,EAAE;QACL,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,aAAa;QACpB,KAAK,EAAE,QAAQ;QACf,IAAI,EACF,kIAAkI;YAClI,uDAAuD;QACzD,MAAM,EAAE,KAAK;KACd;CACwC,CAAA;AAE3C,yFAAyF;AACzF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,KAAoB,EAAE,IAAY,EAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;AAEnH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAuB,EAAE,MAAc,EAAkB,EAAE;IAC1F,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAA4B,CAAA;IACvE,MAAM,OAAO,GAAG,MAAM,CACpB,QAAQ,CAAC,OAAO,CAAC,6BAA6B,MAAM,CAAC,IAAI,yCAAyC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAClH,EAAE,CAAC,CACN,CAAA;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,MAAM,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,IAAI,gBAAgB,CAAC,CAAA;IAClF,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IACjE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,KAAK,CAAC,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,IAAI,CAAC,EAAE,CAAA;AACpH,CAAC,CAAA;AAED,+DAA+D;AAC/D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,QAAuB,EAAkB,EAAE;IACxE,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC9D,MAAM,KAAK,GAAG,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;IACtD,OAAO;QACL,OAAO,EAAE,SAAS,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;QAC1C,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK;QACrC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK;KACtC,CAAA;AACH,CAAC,CAAA;AAED,MAAM,cAAc,GAAG,CAAC,QAAgB,EAAuB,EAAE;IAC/D,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAyC,CAAA;QACjF,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAgC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IAC5G,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAA;IACX,CAAC;AACH,CAAC,CAAA;AAED,MAAM,MAAM,GAAG,CAAC,QAAuB,EAAE,MAAc,EAAE,OAAgB,EAAE,EAAE;IAC3E,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAChC,uFAAuF,CACxF,CAAA;IACD,MAAM,UAAU,GAAG;QACjB,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,MAAM,CAAC,IAAI,wBAAwB,CAAC;QAC/E,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,MAAM,CAAC,IAAI,wBAAwB,CAAC;QAC/E,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,kEAAkE,CAAC;QAChG,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,MAAM,CAAC,IAAI,yDAAyD,CAAC;QAC5G,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,MAAM,CAAC,IAAI,+CAA+C,CAAC;QAClG,KAAK,EAAE,QAAQ,CAAC,OAAO,CACrB,iJAAiJ;YAC/I,uCAAuC,CAC1C;QACD,KAAK,EAAE,QAAQ,CAAC,OAAO,CACrB;sEACgE,CACjE;QACD,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,MAAM,CAAC,IAAI,oDAAoD,CAAC;KAC1G,CAAA;IACD,OAAO,CAAC,EAAU,EAAE,EAAE;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACxB,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QAC9E,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC5B,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC5B,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QAC1C,MAAM,IAAI,GAAG,GAAG,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACxD,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QACtG,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QACzD,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;QAC7B,IAAI,KAAK,KAAK,EAAE;YAAE,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QACjD,IAAI,KAAK,KAAK,EAAE;YAAE,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;QACxD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;QAC/B,IAAI,IAAI,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YAC3C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YACrB,MAAM,QAAQ,GAAG,GAAG,EAAE,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;YAClF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM;gBAC5B,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;gBAC1F,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,QAAQ,EAAE,EAAE;gBAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAA;gBAC9C,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBACf,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAA;YACxG,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM;YAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACjF,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IACzC,CAAC,CAAA;AACH,CAAC,CAAA;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,QAAuB,EACvB,MAAc,EACd,UAA2C,EAC3C,EAAE,KAAK,GAAG,GAAG,EAAE,KAAK,GAAG,GAAG,EAAE,CAAC,KAAK,EAAE,GAA8C,EAAE,EAC5E,EAAE;IACV,uGAAuG;IACvG,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU;QACvD,QAAQ;aACL,OAAO,CACN,0DAA0D,MAAM,CAAC,KAAK;0DACpB,CACnD;aACA,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACrB,wEAAwE;IACxE,QAAQ;SACL,OAAO,CACN,sIAAsI,CACvI;SACA,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACpB,MAAM,KAAK,GAAG,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAChD,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI;QAAE,OAAO,CAAC,CAAA;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAA;IACvE,MAAM,KAAK,GAAG,GAAG,MAAM,CAAC,IAAI,gBAAgB,CAAA;IAC5C,IAAI,KAAK,CAAC,KAAK,KAAK,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrC,OAAO,CAAC,QAAQ,EAAE,GAAG,EAAE;YACrB,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI;gBACtB,QAAQ;qBACL,OAAO,CAAC,yBAAyB,KAAK,2CAA2C,MAAM,CAAC,KAAK,EAAE,CAAC;qBAChG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACrB,QAAQ,CAAC,OAAO,CAAC,2DAA2D,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAA;QACnH,CAAC,CAAC,CAAA;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,KAAK,OAAO,CAAC,QAAQ;QAAE,OAAO,CAAC,CAAA;IACrE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,KAAK,+CAA+C,CAAC,CAAA;IACrG,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;IAC/C,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,SAAS,CAAC;QACR,IAAI,KAAK,EAAE;YAAE,MAAK;QAClB,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,EAAE;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;YACrE,KAAK,MAAM,EAAE,IAAI,GAAG;gBAAE,KAAK,CAAC,EAAE,CAAC,CAAA;YAC/B,OAAO,GAAG,CAAC,MAAM,CAAA;QACnB,CAAC,CAAC,CAAA;QACF,IAAI,KAAK,KAAK,CAAC;YAAE,MAAK;QACtB,IAAI,IAAI,KAAK,CAAA;IACf,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AAED,wDAAwD;AACxD,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,QAAuB,EACvB,UAA2C,EAC3C,OAAO,GAA8C,EAAE,EAC/C,EAAE,CACV,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC;IAC5D,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAA;AAE1D,0EAA0E;AAC1E,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,QAAuB,EAAW,EAAE;IACjE,OAAO,CAAC,QAAQ,EAAE,GAAG,EAAE;QACrB,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;YACzC,QAAQ;iBACL,OAAO,CACN,yBAAyB,MAAM,CAAC,IAAI,yDAAyD,MAAM,CAAC,KAAK,EAAE,CAC5G;iBACA,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACvB,CAAC,CAAC,CAAA;IACF,OAAO,IAAI,CAAA;AACb,CAAC,CAAA"}

package/dist/store/sqlite/note-search.d.ts

@@ -0,0 +1,77 @@
+import type { ResolvedNode } from "../../search/lucene/resolved.js";
+import type { Stemmer } from "../../search/stem.js";
+import { type NoteIndexState } from "./note-index.js";
+import { type Note } from "./notes.js";
+import type { StoreContext } from "./open.js";
+export interface NoteQuery {
+    root: ResolvedNode;
+    limit: number;
+    offset?: number;
+    folderIds?: string[];
+    /** With `folderIds`, the notes written in memo too, in the same ranked list. */
+    withInternal?: boolean;
+    source?: Note["source"];
+    /** Newest first instead of best first. */
+    newest?: boolean;
+    /** The store's stemmer, set when a leaf is stemmed. */
+    stemmer?: Stemmer;
+    signal?: AbortSignal;
+}
+export interface NoteHit {
+    ref: string;
+    note: Note;
+    /** bm25 of the words or stems; lower is better. `null` when ranked by time. */
+    relevance: number | null;
+    /** Whether the note holds the exact forms, when the search was stemmed. */
+    exact?: boolean;
+}
+/** The fields a note has. A message-only field (`from:`, `chat:`, …) is refused rather than matching nothing. */
+export declare const NOTE_FIELDS: readonly ["text", "exact", "body", "tag", "date", "in", "has", "title", "site", "lang", "author", "keyword"];
+/** Notes matching a parsed query, by the same language as messages: files and written notes, merged. */
+export declare const searchNotes: (context: StoreContext, query: NoteQuery) => {
+    items: NoteHit[];
+    hasMore: boolean;
+};
+export interface NoteChunkToEmbed {
+    hash: string;
+    text: string;
+}
+export interface NearestNote {
+    ref: string;
+    note: Note;
+    /** Cosine of the note's best chunk; higher is nearer. */
+    score: number;
+    /** The stretch of the indexed text (title, then text) that chunk holds. */
+    range: {
+        start: number;
+        end: number;
+    };
+}
+export interface NoteSearch {
+    /** Indexes what was written since, then searches; a note written a moment ago is found. */
+    search(query: NoteQuery): Promise<{
+        items: NoteHit[];
+        hasMore: boolean;
+    }>;
+    indexState(): Promise<NoteIndexState | undefined>;
+    /** Live notes' chunks with no vector of `model`, by hash from `after`, each with its text cut again. */
+    chunksToEmbed(model: string, options: {
+        after?: string;
+        limit: number;
+    }): Promise<NoteChunkToEmbed[]>;
+    /** Whether any live chunk in this scope has a vector of `model`: none means meaning search has nothing to rank. */
+    embedded(model: string, options: {
+        folderIds?: string[];
+        withInternal?: boolean;
+        source?: Note["source"];
+    }): Promise<boolean>;
+    /** Notes nearest in meaning to `query`, each scored by its best chunk. Vectors are unit length. */
+    nearest(model: string, query: Float32Array, options: {
+        limit: number;
+        folderIds?: string[];
+        withInternal?: boolean;
+        source?: Note["source"];
+    }): Promise<NearestNote[]>;
+}
+export declare const noteSearchOver: (context: StoreContext) => NoteSearch;
+//# sourceMappingURL=note-search.d.ts.map

package/dist/store/sqlite/note-search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"note-search.d.ts","sourceRoot":"","sources":["../../../src/store/sqlite/note-search.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAqB,MAAM,iCAAiC,CAAA;AAGtF,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAA;AAKnD,OAAO,EAAkB,KAAK,cAAc,EAAiC,MAAM,iBAAiB,CAAA;AACpG,OAAO,EAAc,KAAK,IAAI,EAAU,MAAM,YAAY,CAAA;AAC1D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAI7C,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,YAAY,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAA;IACpB,gFAAgF;IAChF,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,MAAM,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IACvB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,uDAAuD;IACvD,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,MAAM,CAAC,EAAE,WAAW,CAAA;CACrB;AAED,MAAM,WAAW,OAAO;IACtB,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,IAAI,CAAA;IACV,+EAA+E;IAC/E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,2EAA2E;IAC3E,KAAK,CAAC,EAAE,OAAO,CAAA;CAChB;AAED,iHAAiH;AACjH,eAAO,MAAM,WAAW,YACtB,MAAM,EACN,OAAO,EACP,MAAM,EACN,KAAK,EACL,MAAM,EACN,IAAI,EACJ,KAAK,EACL,OAAO,EACP,MAAM,EACN,MAAM,EACN,QAAQ,EACR,SAAS,CACD,CAAA;AAmEV,wGAAwG;AACxG,eAAO,MAAM,WAAW,YAAa,YAAY,SAAS,SAAS,KAAG;IAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAmCzG,CAAA;AA8OD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,IAAI,CAAA;IACV,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAA;IACb,2EAA2E;IAC3E,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;CACtC;AAED,MAAM,WAAW,UAAU;IACzB,2FAA2F;IAC3F,MAAM,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACzE,UAAU,IAAI,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAA;IACjD,wGAAwG;IACxG,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAA;IACrG,mHAAmH;IACnH,QAAQ,CACN,KAAK,EAAE,MAAM,EACb,OAAO,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,YAAY,CAAC,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;KAAE,GACjF,OAAO,CAAC,OAAO,CAAC,CAAA;IACnB,mGAAmG;IACnG,OAAO,CACL,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,YAAY,EACnB,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,YAAY,CAAC,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;KAAE,GAChG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAA;CAC1B;AAUD,eAAO,MAAM,cAAc,YAAa,YAAY,KAAG,UAgGtD,CAAA"}