@wirecat/cli-core 0.489.0 → 0.490.0

package/CHANGELOG.md

@@ -5,6 +5,22 @@ Notable changes to `@wirecat/cli-core` (`@wirecat/cli-messaging` up to 0.261.0,
 minor version may break callers, and says how under "Changed — may break callers". `pnpm docs:check` checks the
 shape of this file.
 
+## Unreleased
+
+## 0.490.0 — 31.10.2026
+
+### Changed — may break callers
+
+- **Notes, documents, knowledge annotations and contacts `--notes` are found by any part of a word, in any letter
+  case (store version 40).** `notes` with a search, `knowledge.annotations` with a search and `contacts` with
+  `notes` used `instr(lower(...))`, which folds only Latin letters and reads every row: a mixed-case Cyrillic word
+  was missed, and a search took 1.5–4 s on a store five times today's size. They now match `note_trigrams` and
+  `document_trigrams`, trigram indexes over the normalized title and text (`ё` as `е`, accents dropped) that the
+  notes drain keeps, and answer in milliseconds. A query of one or two characters is refused with "search needs at
+  least three characters", as message search is; an empty one still filters nothing. The title is searched too,
+  not only the text. The notes and documents stored before version 40 get their trigrams in batches from the
+  notes drain (`search notes`, `store migrate`); until then they are matched as before.
+
 ## 0.489.0 — 31.10.2026
 
 ### Added

package/dist/store/sqlite/identities.d.ts

@@ -28,7 +28,7 @@ export declare const people: ({ orm, database }: StoreContext, provider: Provide
 export interface ContactFilter {
     /** Three letters or more of the name, the @username or the local alias. */
     query?: string;
-    /** Text in one of the owner's private notes about them; case folds for Latin letters only, as SQLite's lower(). */
+    /** Three characters or more of one of the owner's private notes about them, anywhere, in any letter case. */
     notes?: string;
 }
 export declare const contacts: ({ orm }: StoreContext, accountKey: number, key: AccountKey, { order, limit, offset, ...filter }: {

package/dist/store/sqlite/identities.js

@@ -2,7 +2,9 @@ import { CliError } from "@wirecat/cli-base";
 import { fold } from "../normalize.js";
 import { and, asc, desc, eq, inArray, isNotNull, isNull, ne, sql } from "./drizzle/core.js";
 import { folded } from "./folded.js";
+import { holdingText } from "./note-index.js";
 import { resolvePersonLinks } from "./person-resolution.js";
+import { bound } from "./query-filter.js";
 import { accountIdentities, accounts, aliases, chatMembers, chats, identities, identityLinkEvents, identityLinks, identityRevisions, messages, persons, } from "./schema.js";
 import { toIso } from "./values.js";
 const flag = (value) => (value === undefined || value === null ? null : Number(value));
@@ -209,13 +211,14 @@ const contactsWhere = (accountKey, key, { query, notes }) => {
     if (query !== undefined && query.trim().length < 3) {
         throw new CliError("validation_error", `a contact search takes at least 3 characters, got "${query}"`);
     }
+    const noted = notes === undefined ? undefined : holdingText("note", "n.id", notes);
     return and(eq(accountIdentities.accountId, accountKey), ne(identities.externalId, key.account), sql `EXISTS (SELECT 1 FROM ${chatMembers} JOIN ${chats} ON ${chats.id} = ${chatMembers.chatId}
                 WHERE ${chatMembers.identityId} = ${identities.id} AND ${chats.accountId} = ${accountIdentities.accountId}
                   AND ${chats.kind} = 'dialog')`, query === undefined
         ? undefined
         : sql `(${identities.id} IN (SELECT rowid FROM identities_fts WHERE identities_fts MATCH ${`"${query.trim().replaceAll('"', '""')}"`}) OR EXISTS (SELECT 1 FROM aliases ca WHERE ca.aliasable_type='identity' AND ca.display=1 AND ca.account_id=${accountKey} AND ca.aliasable_id=${identities.id} AND instr(ca.name_folded, ${fold(query.trim())}) > 0))`, notes === undefined
         ? undefined
-        : sql `EXISTS (SELECT 1 FROM notes n WHERE n.notable_type='identity' AND n.notable_id=${identities.id} AND n.deleted_at IS NULL AND instr(lower(n.body), lower(${notes})) > 0)`);
+        : sql `EXISTS (SELECT 1 FROM notes n WHERE n.notable_type='identity' AND n.notable_id=${identities.id} AND n.deleted_at IS NULL${noted ? sql ` AND ${bound(noted)}` : sql ``})`);
 };
 export const contacts = ({ orm }, accountKey, key, { order, limit, offset = 0, ...filter }) => {
     const byName = [

package/dist/store/sqlite/knowledge.js

@@ -10,6 +10,7 @@ import { ulid } from "../ulid.js";
 import { ownerPerson } from "./actors.js";
 import { linkKind, RELATION_KINDS } from "./link-kinds.js";
 import { meetingProjectsStoreOver } from "./meeting-projects.js";
+import { holdingText } from "./note-index.js";
 import { insertLink, linkOf } from "./notes.js";
 import { queryFilter } from "./query-filter.js";
 import { addTags as addThingTags, canonicalTagNamed, createTag, labelsOf, liveTagging, removeTags as removeThingTags, setMainTopic, } from "./tags.js";
@@ -314,10 +315,11 @@ export const knowledgeStoreOver = (context) => {
             const thing = options.target ? resolve(key, options.target).thing : undefined;
             if (options.target && !thing)
                 return { items: [], hasMore: false };
+            const text = options.search === undefined ? undefined : holdingText("note", "id", options.search);
             const rows = database
                 .prepare("SELECT * FROM notes WHERE deleted_at IS NULL AND (? IS NULL OR (notable_type = ? AND notable_id = ?)) " +
-                "AND (? IS NULL OR instr(lower(body), lower(?)) > 0) ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?")
-                .all(thing?.type ?? null, thing?.type ?? null, thing?.id ?? null, options.search ?? null, options.search ?? null, limit + 1, offset);
+                `${text ? `AND ${text.sql} ` : ""}ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?`)
+                .all(thing?.type ?? null, thing?.type ?? null, thing?.id ?? null, ...(text?.params ?? []), limit + 1, offset);
             const items = rows.slice(0, limit).flatMap((row) => annotationOf(row) ?? []);
             return { items, hasMore: rows.length > limit };
         },

package/dist/store/sqlite/manifest.js

@@ -53,6 +53,7 @@ export const MANIFEST = [
     { name: "20261029224106_version-38-folded-names", version: 38, minCompatible: 34 },
     { name: "20261029224107_version-38-folded-triggers", version: 38, minCompatible: 34 },
     { name: "20261030114008_version-39-email-locations", version: 39, minCompatible: 34 },
+    { name: "20261030225520_version-40-note-trigrams", version: 40, minCompatible: 34 },
 ];
 export const generatedMigrations = (manifest = MANIFEST, generated = GENERATED) => {
     const migrations = [];

package/dist/store/sqlite/migrations.generated.js

@@ -837,5 +837,15 @@ export const GENERATED = [
             "CREATE TABLE `email_locations` (\n\t`mailbox_id` integer NOT NULL,\n\t`uidvalidity` text NOT NULL,\n\t`uid` integer NOT NULL,\n\t`email_id` integer NOT NULL,\n\tCONSTRAINT `email_locations_pk` PRIMARY KEY(`mailbox_id`, `uidvalidity`, `uid`),\n\tCONSTRAINT `fk_email_locations_mailbox_id_mailboxes_id_fk` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes`(`id`),\n\tCONSTRAINT `fk_email_locations_email_id_emails_id_fk` FOREIGN KEY (`email_id`) REFERENCES `emails`(`id`)\n);",
             "CREATE INDEX `email_locations_by_email_id` ON `email_locations` (`email_id`);"
         ]
+    },
+    {
+        "name": "20261030225520_version-40-note-trigrams",
+        "statements": [
+            "CREATE VIRTUAL TABLE note_trigrams USING fts5(text, content = '', contentless_delete = 1, tokenize = 'trigram');",
+            "CREATE VIRTUAL TABLE document_trigrams USING fts5(text, content = '', contentless_delete = 1, tokenize = 'trigram');",
+            "CREATE TRIGGER note_trigrams_bd BEFORE DELETE ON notes BEGIN\n  DELETE FROM note_trigrams WHERE rowid = old.id;\nEND;",
+            "CREATE TRIGGER document_trigrams_bd BEFORE DELETE ON documents BEGIN\n  DELETE FROM document_trigrams WHERE rowid = old.id;\nEND;",
+            "-- The trigrams hold JS-normalized text, which SQL cannot produce: fillTrigrams fills the rows stored before this\n-- version, up to `watermark`, and a search reads the rest of them by their text meanwhile.\nINSERT INTO search_index_state (name, watermark, filled_through, terms_through, normalizer_version)\n  SELECT 'note_trigrams', coalesce(max(id), 0), 0, 0, 1 FROM notes\n  UNION ALL SELECT 'document_trigrams', coalesce(max(id), 0), 0, 0, 1 FROM documents;"
+        ]
     }
 ];

package/dist/store/sqlite/note-index.d.ts

@@ -1,5 +1,6 @@
 import { type Stemmer, type Stemmers } from "../../search/stem.js";
 import type { CacheDatabase } from "../driver.js";
+import type { Where } from "./query-filter.js";
 export interface NoteIndexState {
     /** Rows written since they were indexed. */
     pending: number;
@@ -102,6 +103,20 @@ export declare const CORPORA: {
 };
 /** What a row is indexed as: its title, then its text. Chunk offsets point into this. */
 export declare const noteIndexText: (title: string | null, text: string) => string;
+/**
+ * `column` (a note's or document's id) is a row whose title or text holds `search` anywhere, in any letter case;
+ * no condition for an empty `search`. A row the trigrams do not hold yet — waiting in its queue, or stored before
+ * version 40 and not yet reached by `fillTrigrams` — is matched by its text as SQLite's `lower()` reads it.
+ */
+export declare const holdingText: (type: "note" | "document", column: string, search: string) => Where | undefined;
+/**
+ * Writes the trigrams of the notes and documents stored before version 40, in short write transactions, up to the
+ * highest id the migration found; later rows get theirs from the drain. Stopping between batches loses nothing.
+ */
+export declare const fillTrigrams: (database: CacheDatabase, { batch, until }?: {
+    batch?: number;
+    until?: () => boolean;
+}) => number;
 export declare const corpusIndexState: (database: CacheDatabase, corpus: Corpus) => NoteIndexState;
 /** Documents and notes together: what a notes search reads. */
 export declare const noteIndexState: (database: CacheDatabase) => NoteIndexState;
@@ -113,7 +128,7 @@ export declare const drainCorpus: (database: CacheDatabase, corpus: Corpus, stem
     batch?: number;
     until?: () => boolean;
 }) => number;
-/** Documents, then notes: what a notes search reads. */
+/** Documents, then notes: what a notes search reads; then the trigrams still to fill. */
 export declare const drainNoteIndex: (database: CacheDatabase, stemmerFor: (stemmers: Stemmers) => Stemmer, options?: {
     batch?: number;
     until?: () => boolean;

package/dist/store/sqlite/note-index.js

@@ -1,3 +1,4 @@
+import { CliError } from "@wirecat/cli-base";
 import { CHUNK_CHARS, chunkHash, splitSections, splitText } from "../../conversations/chunks.js";
 import { analyzerIdentity, DEFAULT_STEMMERS } from "../../search/stem.js";
 import { taskStemsOf, taskWordsOf } from "../../search/task-similarity.js";
@@ -103,6 +104,62 @@ export const CORPORA = {
 /** What a row is indexed as: its title, then its text. Chunk offsets point into this. */
 export const noteIndexText = (title, text) => (title ? `${title}\n\n${text}` : text);
 const queuedAs = (corpus) => corpus.queued ?? corpus.type;
+const hasTrigrams = (corpus) => corpus.type === "note" || corpus.type === "document";
+/**
+ * `column` (a note's or document's id) is a row whose title or text holds `search` anywhere, in any letter case;
+ * no condition for an empty `search`. A row the trigrams do not hold yet — waiting in its queue, or stored before
+ * version 40 and not yet reached by `fillTrigrams` — is matched by its text as SQLite's `lower()` reads it.
+ */
+export const holdingText = (type, column, search) => {
+    const text = normalize(search);
+    if (text === "")
+        return undefined;
+    // A trigram index answers a shorter query with nothing, silently.
+    if ([...text].length < 3)
+        throw new CliError("validation_error", "search needs at least three characters");
+    const queued = `SELECT id FROM ${type}_index_pending WHERE indexable_type = '${type}'`;
+    const state = (field) => `(SELECT ${field} FROM search_index_state WHERE name = '${type}_trigrams')`;
+    return {
+        sql: `${column} IN (SELECT rowid FROM ${type}_trigrams WHERE ${type}_trigrams MATCH ? AND rowid NOT IN (${queued}) ` +
+            `UNION ALL SELECT r.id FROM ${type}s r WHERE (r.id IN (${queued}) OR (r.id > ${state("filled_through")} ` +
+            `AND r.id <= ${state("watermark")})) AND instr(lower(coalesce(r.body, '')), lower(?)) > 0)`,
+        params: [`"${text.replaceAll('"', '""')}"`, search],
+    };
+};
+/**
+ * Writes the trigrams of the notes and documents stored before version 40, in short write transactions, up to the
+ * highest id the migration found; later rows get theirs from the drain. Stopping between batches loses nothing.
+ */
+export const fillTrigrams = (database, { batch = 200, until = () => false } = {}) => {
+    let filled = 0;
+    for (const type of ["document", "note"]) {
+        const name = `${type}_trigrams`;
+        const state = database.prepare("SELECT watermark, filled_through FROM search_index_state WHERE name = ?");
+        const rows = database.prepare(`SELECT id, title, coalesce(body, '') AS body FROM ${type}s WHERE id > ? AND id <= ? AND deleted_at IS NULL ` +
+            "ORDER BY id LIMIT ?");
+        const drop = database.prepare(`DELETE FROM ${name} WHERE rowid = ?`);
+        const add = database.prepare(`INSERT INTO ${name} (rowid, text) VALUES (?, ?)`);
+        const reached = database.prepare("UPDATE search_index_state SET filled_through = ? WHERE name = ?");
+        for (;;) {
+            const at = state.get(name);
+            const mark = Number(at?.watermark ?? 0);
+            if (Number(at?.filled_through ?? mark) >= mark || until())
+                break;
+            filled += write(database, () => {
+                const page = rows.all(Number(at?.filled_through), mark, batch);
+                for (const row of page) {
+                    const text = normalize(noteIndexText(row.title == null ? null : String(row.title), String(row.body)));
+                    drop.run(Number(row.id));
+                    if (text !== "")
+                        add.run(Number(row.id), text);
+                }
+                reached.run(page.length < batch ? mark : Number(page.at(-1)?.id), name);
+                return page.length;
+            });
+        }
+    }
+    return filled;
+};
 export const corpusIndexState = (database, corpus) => {
     const row = indexRow(database, corpus.index);
     const pending = Number(database
@@ -139,9 +196,13 @@ const writer = (database, corpus, stemmer) => {
     const statements = {
         dropWords: database.prepare(`DELETE FROM ${corpus.type}_words WHERE rowid = ?`),
         dropStems: database.prepare(`DELETE FROM ${corpus.type}_stems WHERE rowid = ?`),
+        dropTrigrams: hasTrigrams(corpus) ? database.prepare(`DELETE FROM ${corpus.type}_trigrams WHERE rowid = ?`) : null,
         dropChunks: database.prepare("DELETE FROM chunks WHERE chunkable_type = ? AND chunkable_id = ?"),
         words: database.prepare(`INSERT INTO ${corpus.type}_words (rowid, normalized_text, scope) VALUES (?, ?, ?)`),
         stems: database.prepare(`INSERT INTO ${corpus.type}_stems (rowid, stems, scope) VALUES (?, ?, ?)`),
+        trigrams: hasTrigrams(corpus)
+            ? database.prepare(`INSERT INTO ${corpus.type}_trigrams (rowid, text) VALUES (?, ?)`)
+            : null,
         chunk: database.prepare("INSERT INTO chunks (chunkable_type, chunkable_id, position, start_offset, end_offset, content_hash, scope, account_id, created_at, updated_at) " +
             "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"),
         dequeue: database.prepare(`DELETE FROM ${corpus.type}_index_pending WHERE indexable_type = ? AND id = ?`),
@@ -151,14 +212,17 @@ const writer = (database, corpus, stemmer) => {
         const before = oldHashes.all(corpus.type, id).map((hash) => String(hash.hash));
         statements.dropWords.run(id);
         statements.dropStems.run(id);
+        statements.dropTrigrams?.run(id);
         statements.dropChunks.run(corpus.type, id);
         const live = row !== undefined && Number(row.live) === 1;
         const text = live ? noteIndexText(row.title == null ? null : String(row.title), String(row.body)) : "";
         const scope = live ? String(row.scope ?? "personal") : "";
         const terms = corpus.terms && text ? corpus.terms(text, stemmer) : undefined;
         const words = terms ? terms.words : normalize(text);
-        if (words !== "")
+        if (words !== "") {
             statements.words.run(id, words, scope);
+            statements.trigrams?.run(id, words);
+        }
         const stems = terms ? terms.stems : text ? stemmer.indexText(text) : "";
         if (stems !== "")
             statements.stems.run(id, stems, scope);
@@ -228,9 +292,10 @@ export const drainCorpus = (database, corpus, stemmerFor, { batch = 200, until =
     }
     return done;
 };
-/** Documents, then notes: what a notes search reads. */
+/** Documents, then notes: what a notes search reads; then the trigrams still to fill. */
 export const drainNoteIndex = (database, stemmerFor, options = {}) => drainCorpus(database, CORPORA.document, stemmerFor, options) +
-    drainCorpus(database, CORPORA.note, stemmerFor, options);
+    drainCorpus(database, CORPORA.note, stemmerFor, options) +
+    fillTrigrams(database, options);
 /** Queues every document, note and memory again — for `store rebuild`. */
 export const resetNoteIndex = (database) => {
     write(database, () => {

package/dist/store/sqlite/notes.js

@@ -12,6 +12,7 @@ import { PREFERRED_FILE, syncFile } from "./files.js";
 import { importedDocumentWhere } from "./imported-documents.js";
 import { LINK_KINDS, LINK_ORIGINS, linkKind } from "./link-kinds.js";
 import { reviewLink } from "./link-reviews.js";
+import { holdingText } from "./note-index.js";
 import { addTags, ensureTag, labelsOf } from "./tags.js";
 import { canonicalThingReference, referenceOfThing, storedThing, thingOf } from "./things.js";
 import { write } from "./transaction.js";
@@ -525,7 +526,8 @@ export const notesStoreOver = (context) => {
             if (options.about !== undefined && !about)
                 return { items: [], hasMore: false };
             const folder = options.folderId === undefined ? null : Number(folderRow(options.folderId).id);
-            const search = options.search ?? null;
+            const documentText = options.search === undefined ? undefined : holdingText("document", "d.id", options.search);
+            const noteText = options.search === undefined ? undefined : holdingText("note", "n.id", options.search);
             const aboutType = about?.type ?? null;
             const aboutId = about?.id ?? null;
             const files = options.source === "internal"
@@ -541,16 +543,16 @@ export const notesStoreOver = (context) => {
                         "WHERE f.deleted_at IS NULL AND d.deleted_at IS NULL AND f.account_id = ? AND ? IS NOT NULL ") +
                     "AND coalesce(json_extract(d.metadata, '$.catalog.active'), 1)=1 " +
                     `AND (? IS NULL OR EXISTS (SELECT 1 FROM links l WHERE l.from_type = 'document' AND l.from_id = d.id AND l.kind = ${linkKind("about")} AND l.to_type = ? AND l.to_id = ?)) ` +
-                    "AND (? IS NULL OR instr(lower(coalesce(d.body, '')), lower(?)) > 0) ORDER BY d.created_at DESC, d.id DESC LIMIT ?")
-                    .all(folder, folder, aboutType, aboutType, aboutId, search, search, limit + offset + 1)
+                    `${documentText ? `AND ${documentText.sql} ` : ""}ORDER BY d.created_at DESC, d.id DESC LIMIT ?`)
+                    .all(folder, folder, aboutType, aboutType, aboutId, ...(documentText?.params ?? []), limit + offset + 1)
                     .map(documentOf);
             const written = options.source === "file" || folder !== null
                 ? []
                 : database
                     .prepare("SELECT n.* FROM notes n WHERE n.deleted_at IS NULL " +
                     `AND (? IS NULL OR (n.notable_type = ? AND n.notable_id = ?) OR EXISTS (SELECT 1 FROM links l WHERE l.from_type = 'note' AND l.from_id = n.id AND l.kind = ${linkKind("about")} AND l.to_type = ? AND l.to_id = ?)) ` +
-                    "AND (? IS NULL OR instr(lower(n.body), lower(?)) > 0) ORDER BY n.created_at DESC, n.id DESC LIMIT ?")
-                    .all(aboutType, aboutType, aboutId, aboutType, aboutId, search, search, limit + offset + 1)
+                    `${noteText ? `AND ${noteText.sql} ` : ""}ORDER BY n.created_at DESC, n.id DESC LIMIT ?`)
+                    .all(aboutType, aboutType, aboutId, aboutType, aboutId, ...(noteText?.params ?? []), limit + offset + 1)
                     .map(noteOf);
             const all = [...files, ...written].sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.ref.localeCompare(a.ref));
             return { items: all.slice(offset, offset + limit), hasMore: all.length > offset + limit };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.489.0",
+  "version": "0.490.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",