@wirecat/cli-core 0.488.0 → 0.490.0

package/dist/store/sqlite/manifest.js

@@ -53,6 +53,7 @@ export const MANIFEST = [
     { name: "20261029224106_version-38-folded-names", version: 38, minCompatible: 34 },
     { name: "20261029224107_version-38-folded-triggers", version: 38, minCompatible: 34 },
     { name: "20261030114008_version-39-email-locations", version: 39, minCompatible: 34 },
+    { name: "20261030225520_version-40-note-trigrams", version: 40, minCompatible: 34 },
 ];
 export const generatedMigrations = (manifest = MANIFEST, generated = GENERATED) => {
     const migrations = [];

package/dist/store/sqlite/migrations.generated.js

@@ -837,5 +837,15 @@ export const GENERATED = [
             "CREATE TABLE `email_locations` (\n\t`mailbox_id` integer NOT NULL,\n\t`uidvalidity` text NOT NULL,\n\t`uid` integer NOT NULL,\n\t`email_id` integer NOT NULL,\n\tCONSTRAINT `email_locations_pk` PRIMARY KEY(`mailbox_id`, `uidvalidity`, `uid`),\n\tCONSTRAINT `fk_email_locations_mailbox_id_mailboxes_id_fk` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes`(`id`),\n\tCONSTRAINT `fk_email_locations_email_id_emails_id_fk` FOREIGN KEY (`email_id`) REFERENCES `emails`(`id`)\n);",
             "CREATE INDEX `email_locations_by_email_id` ON `email_locations` (`email_id`);"
         ]
+    },
+    {
+        "name": "20261030225520_version-40-note-trigrams",
+        "statements": [
+            "CREATE VIRTUAL TABLE note_trigrams USING fts5(text, content = '', contentless_delete = 1, tokenize = 'trigram');",
+            "CREATE VIRTUAL TABLE document_trigrams USING fts5(text, content = '', contentless_delete = 1, tokenize = 'trigram');",
+            "CREATE TRIGGER note_trigrams_bd BEFORE DELETE ON notes BEGIN\n  DELETE FROM note_trigrams WHERE rowid = old.id;\nEND;",
+            "CREATE TRIGGER document_trigrams_bd BEFORE DELETE ON documents BEGIN\n  DELETE FROM document_trigrams WHERE rowid = old.id;\nEND;",
+            "-- The trigrams hold JS-normalized text, which SQL cannot produce: fillTrigrams fills the rows stored before this\n-- version, up to `watermark`, and a search reads the rest of them by their text meanwhile.\nINSERT INTO search_index_state (name, watermark, filled_through, terms_through, normalizer_version)\n  SELECT 'note_trigrams', coalesce(max(id), 0), 0, 0, 1 FROM notes\n  UNION ALL SELECT 'document_trigrams', coalesce(max(id), 0), 0, 0, 1 FROM documents;"
+        ]
     }
 ];

package/dist/store/sqlite/note-index.d.ts

@@ -1,5 +1,6 @@
 import { type Stemmer, type Stemmers } from "../../search/stem.js";
 import type { CacheDatabase } from "../driver.js";
+import type { Where } from "./query-filter.js";
 export interface NoteIndexState {
     /** Rows written since they were indexed. */
     pending: number;
@@ -102,6 +103,20 @@ export declare const CORPORA: {
 };
 /** What a row is indexed as: its title, then its text. Chunk offsets point into this. */
 export declare const noteIndexText: (title: string | null, text: string) => string;
+/**
+ * `column` (a note's or document's id) is a row whose title or text holds `search` anywhere, in any letter case;
+ * no condition for an empty `search`. A row the trigrams do not hold yet — waiting in its queue, or stored before
+ * version 40 and not yet reached by `fillTrigrams` — is matched by its text as SQLite's `lower()` reads it.
+ */
+export declare const holdingText: (type: "note" | "document", column: string, search: string) => Where | undefined;
+/**
+ * Writes the trigrams of the notes and documents stored before version 40, in short write transactions, up to the
+ * highest id the migration found; later rows get theirs from the drain. Stopping between batches loses nothing.
+ */
+export declare const fillTrigrams: (database: CacheDatabase, { batch, until }?: {
+    batch?: number;
+    until?: () => boolean;
+}) => number;
 export declare const corpusIndexState: (database: CacheDatabase, corpus: Corpus) => NoteIndexState;
 /** Documents and notes together: what a notes search reads. */
 export declare const noteIndexState: (database: CacheDatabase) => NoteIndexState;
@@ -113,7 +128,7 @@ export declare const drainCorpus: (database: CacheDatabase, corpus: Corpus, stem
     batch?: number;
     until?: () => boolean;
 }) => number;
-/** Documents, then notes: what a notes search reads. */
+/** Documents, then notes: what a notes search reads; then the trigrams still to fill. */
 export declare const drainNoteIndex: (database: CacheDatabase, stemmerFor: (stemmers: Stemmers) => Stemmer, options?: {
     batch?: number;
     until?: () => boolean;

package/dist/store/sqlite/note-index.js

@@ -1,3 +1,4 @@
+import { CliError } from "@wirecat/cli-base";
 import { CHUNK_CHARS, chunkHash, splitSections, splitText } from "../../conversations/chunks.js";
 import { analyzerIdentity, DEFAULT_STEMMERS } from "../../search/stem.js";
 import { taskStemsOf, taskWordsOf } from "../../search/task-similarity.js";
@@ -103,6 +104,62 @@ export const CORPORA = {
 /** What a row is indexed as: its title, then its text. Chunk offsets point into this. */
 export const noteIndexText = (title, text) => (title ? `${title}\n\n${text}` : text);
 const queuedAs = (corpus) => corpus.queued ?? corpus.type;
+const hasTrigrams = (corpus) => corpus.type === "note" || corpus.type === "document";
+/**
+ * `column` (a note's or document's id) is a row whose title or text holds `search` anywhere, in any letter case;
+ * no condition for an empty `search`. A row the trigrams do not hold yet — waiting in its queue, or stored before
+ * version 40 and not yet reached by `fillTrigrams` — is matched by its text as SQLite's `lower()` reads it.
+ */
+export const holdingText = (type, column, search) => {
+    const text = normalize(search);
+    if (text === "")
+        return undefined;
+    // A trigram index answers a shorter query with nothing, silently.
+    if ([...text].length < 3)
+        throw new CliError("validation_error", "search needs at least three characters");
+    const queued = `SELECT id FROM ${type}_index_pending WHERE indexable_type = '${type}'`;
+    const state = (field) => `(SELECT ${field} FROM search_index_state WHERE name = '${type}_trigrams')`;
+    return {
+        sql: `${column} IN (SELECT rowid FROM ${type}_trigrams WHERE ${type}_trigrams MATCH ? AND rowid NOT IN (${queued}) ` +
+            `UNION ALL SELECT r.id FROM ${type}s r WHERE (r.id IN (${queued}) OR (r.id > ${state("filled_through")} ` +
+            `AND r.id <= ${state("watermark")})) AND instr(lower(coalesce(r.body, '')), lower(?)) > 0)`,
+        params: [`"${text.replaceAll('"', '""')}"`, search],
+    };
+};
+/**
+ * Writes the trigrams of the notes and documents stored before version 40, in short write transactions, up to the
+ * highest id the migration found; later rows get theirs from the drain. Stopping between batches loses nothing.
+ */
+export const fillTrigrams = (database, { batch = 200, until = () => false } = {}) => {
+    let filled = 0;
+    for (const type of ["document", "note"]) {
+        const name = `${type}_trigrams`;
+        const state = database.prepare("SELECT watermark, filled_through FROM search_index_state WHERE name = ?");
+        const rows = database.prepare(`SELECT id, title, coalesce(body, '') AS body FROM ${type}s WHERE id > ? AND id <= ? AND deleted_at IS NULL ` +
+            "ORDER BY id LIMIT ?");
+        const drop = database.prepare(`DELETE FROM ${name} WHERE rowid = ?`);
+        const add = database.prepare(`INSERT INTO ${name} (rowid, text) VALUES (?, ?)`);
+        const reached = database.prepare("UPDATE search_index_state SET filled_through = ? WHERE name = ?");
+        for (;;) {
+            const at = state.get(name);
+            const mark = Number(at?.watermark ?? 0);
+            if (Number(at?.filled_through ?? mark) >= mark || until())
+                break;
+            filled += write(database, () => {
+                const page = rows.all(Number(at?.filled_through), mark, batch);
+                for (const row of page) {
+                    const text = normalize(noteIndexText(row.title == null ? null : String(row.title), String(row.body)));
+                    drop.run(Number(row.id));
+                    if (text !== "")
+                        add.run(Number(row.id), text);
+                }
+                reached.run(page.length < batch ? mark : Number(page.at(-1)?.id), name);
+                return page.length;
+            });
+        }
+    }
+    return filled;
+};
 export const corpusIndexState = (database, corpus) => {
     const row = indexRow(database, corpus.index);
     const pending = Number(database
@@ -139,9 +196,13 @@ const writer = (database, corpus, stemmer) => {
     const statements = {
         dropWords: database.prepare(`DELETE FROM ${corpus.type}_words WHERE rowid = ?`),
         dropStems: database.prepare(`DELETE FROM ${corpus.type}_stems WHERE rowid = ?`),
+        dropTrigrams: hasTrigrams(corpus) ? database.prepare(`DELETE FROM ${corpus.type}_trigrams WHERE rowid = ?`) : null,
         dropChunks: database.prepare("DELETE FROM chunks WHERE chunkable_type = ? AND chunkable_id = ?"),
         words: database.prepare(`INSERT INTO ${corpus.type}_words (rowid, normalized_text, scope) VALUES (?, ?, ?)`),
         stems: database.prepare(`INSERT INTO ${corpus.type}_stems (rowid, stems, scope) VALUES (?, ?, ?)`),
+        trigrams: hasTrigrams(corpus)
+            ? database.prepare(`INSERT INTO ${corpus.type}_trigrams (rowid, text) VALUES (?, ?)`)
+            : null,
         chunk: database.prepare("INSERT INTO chunks (chunkable_type, chunkable_id, position, start_offset, end_offset, content_hash, scope, account_id, created_at, updated_at) " +
             "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"),
         dequeue: database.prepare(`DELETE FROM ${corpus.type}_index_pending WHERE indexable_type = ? AND id = ?`),
@@ -151,14 +212,17 @@ const writer = (database, corpus, stemmer) => {
         const before = oldHashes.all(corpus.type, id).map((hash) => String(hash.hash));
         statements.dropWords.run(id);
         statements.dropStems.run(id);
+        statements.dropTrigrams?.run(id);
         statements.dropChunks.run(corpus.type, id);
         const live = row !== undefined && Number(row.live) === 1;
         const text = live ? noteIndexText(row.title == null ? null : String(row.title), String(row.body)) : "";
         const scope = live ? String(row.scope ?? "personal") : "";
         const terms = corpus.terms && text ? corpus.terms(text, stemmer) : undefined;
         const words = terms ? terms.words : normalize(text);
-        if (words !== "")
+        if (words !== "") {
             statements.words.run(id, words, scope);
+            statements.trigrams?.run(id, words);
+        }
         const stems = terms ? terms.stems : text ? stemmer.indexText(text) : "";
         if (stems !== "")
             statements.stems.run(id, stems, scope);
@@ -228,9 +292,10 @@ export const drainCorpus = (database, corpus, stemmerFor, { batch = 200, until =
     }
     return done;
 };
-/** Documents, then notes: what a notes search reads. */
+/** Documents, then notes: what a notes search reads; then the trigrams still to fill. */
 export const drainNoteIndex = (database, stemmerFor, options = {}) => drainCorpus(database, CORPORA.document, stemmerFor, options) +
-    drainCorpus(database, CORPORA.note, stemmerFor, options);
+    drainCorpus(database, CORPORA.note, stemmerFor, options) +
+    fillTrigrams(database, options);
 /** Queues every document, note and memory again — for `store rebuild`. */
 export const resetNoteIndex = (database) => {
     write(database, () => {

package/dist/store/sqlite/notes.js

@@ -12,6 +12,7 @@ import { PREFERRED_FILE, syncFile } from "./files.js";
 import { importedDocumentWhere } from "./imported-documents.js";
 import { LINK_KINDS, LINK_ORIGINS, linkKind } from "./link-kinds.js";
 import { reviewLink } from "./link-reviews.js";
+import { holdingText } from "./note-index.js";
 import { addTags, ensureTag, labelsOf } from "./tags.js";
 import { canonicalThingReference, referenceOfThing, storedThing, thingOf } from "./things.js";
 import { write } from "./transaction.js";
@@ -525,7 +526,8 @@ export const notesStoreOver = (context) => {
             if (options.about !== undefined && !about)
                 return { items: [], hasMore: false };
             const folder = options.folderId === undefined ? null : Number(folderRow(options.folderId).id);
-            const search = options.search ?? null;
+            const documentText = options.search === undefined ? undefined : holdingText("document", "d.id", options.search);
+            const noteText = options.search === undefined ? undefined : holdingText("note", "n.id", options.search);
             const aboutType = about?.type ?? null;
             const aboutId = about?.id ?? null;
             const files = options.source === "internal"
@@ -541,16 +543,16 @@ export const notesStoreOver = (context) => {
                         "WHERE f.deleted_at IS NULL AND d.deleted_at IS NULL AND f.account_id = ? AND ? IS NOT NULL ") +
                     "AND coalesce(json_extract(d.metadata, '$.catalog.active'), 1)=1 " +
                     `AND (? IS NULL OR EXISTS (SELECT 1 FROM links l WHERE l.from_type = 'document' AND l.from_id = d.id AND l.kind = ${linkKind("about")} AND l.to_type = ? AND l.to_id = ?)) ` +
-                    "AND (? IS NULL OR instr(lower(coalesce(d.body, '')), lower(?)) > 0) ORDER BY d.created_at DESC, d.id DESC LIMIT ?")
-                    .all(folder, folder, aboutType, aboutType, aboutId, search, search, limit + offset + 1)
+                    `${documentText ? `AND ${documentText.sql} ` : ""}ORDER BY d.created_at DESC, d.id DESC LIMIT ?`)
+                    .all(folder, folder, aboutType, aboutType, aboutId, ...(documentText?.params ?? []), limit + offset + 1)
                     .map(documentOf);
             const written = options.source === "file" || folder !== null
                 ? []
                 : database
                     .prepare("SELECT n.* FROM notes n WHERE n.deleted_at IS NULL " +
                     `AND (? IS NULL OR (n.notable_type = ? AND n.notable_id = ?) OR EXISTS (SELECT 1 FROM links l WHERE l.from_type = 'note' AND l.from_id = n.id AND l.kind = ${linkKind("about")} AND l.to_type = ? AND l.to_id = ?)) ` +
-                    "AND (? IS NULL OR instr(lower(n.body), lower(?)) > 0) ORDER BY n.created_at DESC, n.id DESC LIMIT ?")
-                    .all(aboutType, aboutType, aboutId, aboutType, aboutId, search, search, limit + offset + 1)
+                    `${noteText ? `AND ${noteText.sql} ` : ""}ORDER BY n.created_at DESC, n.id DESC LIMIT ?`)
+                    .all(aboutType, aboutType, aboutId, aboutType, aboutId, ...(noteText?.params ?? []), limit + offset + 1)
                     .map(noteOf);
             const all = [...files, ...written].sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.ref.localeCompare(a.ref));
             return { items: all.slice(offset, offset + limit), hasMore: all.length > offset + limit };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.488.0",
+  "version": "0.490.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",