@wirecat/cli-core 0.395.0 → 0.396.0

package/dist/search/lucene/registry.js

@@ -294,11 +294,11 @@ export const QUERY_FIELDS = [
     {
         name: "transcript",
         example: "transcript:invoice",
-        index: "attachment_words (voice transcripts)",
+        index: "attachment_words ∪ transcript_stems; garbled spellings from transcript_words_vocab",
         aliases: [],
         type: "txt",
         operators: terms,
-        normalization: "NFKD/marks/NFC/lowercase v1; what a voice message said, as transcribed",
+        normalization: "any form of the word (Snowball), widened to spellings speech recognition garbles; what a voice message said",
         support: "A2",
         validate: filled,
     },

package/dist/services/messages-search.js

@@ -331,7 +331,9 @@ export const searchLucene = async (store, account, request, messenger = {}) => {
     // Coverage counts messenger chats fetched from a server; mail is imported whole, so for mail it says nothing.
     const { completeness, coverage } = request.kind === "mail" ? { completeness: [], coverage: undefined } : await coverageOf(store, prepared, messenger);
     const items = await Promise.all(found.items.map(async (item) => {
-        const hit = withExcerpt(item, request.text ?? "");
+        // A garbled spelling is not among the typed words, so the excerpt looks for it too.
+        const garbled = item.corrections?.flatMap(({ to }) => to) ?? [];
+        const hit = withExcerpt(item, [request.text ?? "", ...garbled].join(" "));
         if (!request.context)
             return hit;
         const locator = parseLocator(hit.locator);
@@ -347,7 +349,7 @@ export const searchLucene = async (store, account, request, messenger = {}) => {
     return {
         ...found,
         items,
-        corrections: [],
+        corrections: correctionsOf(found.items),
         wordsReady: prepared.wordsReady,
         stemsReady: prepared.stemsReady,
         completeness,
@@ -359,6 +361,15 @@ export const searchLucene = async (store, account, request, messenger = {}) => {
             : {}),
     };
 };
+/** The garbled spellings voice-note hits were found through, one entry per query word. */
+const correctionsOf = (items) => {
+    const found = new Map();
+    for (const { corrections = [] } of items)
+        for (const { from, to } of corrections)
+            for (const word of to)
+                found.set(from, (found.get(from) ?? new Set()).add(word));
+    return [...found].map(([from, to]) => ({ from, to: [...to] }));
+};
 const QUERY_PAGE_LIMIT = 1000;
 const oldestFetch = (chats) => chats.length === 0 || chats.some(({ fetchedAt }) => fetchedAt === null)
     ? null

package/dist/services/messages.d.ts

@@ -81,6 +81,11 @@ export type FoundMessage = StoredHit & {
         parent?: string;
     };
     match?: "words" | "beginnings" | "corrected";
+    /** The garbled spellings a `corrected` hit's transcript holds, per query word. */
+    corrections?: {
+        from: string;
+        to: string[];
+    }[];
     /** Where the words were: the text, a voice note's transcript, a file's name or a file's text. */
     matchedIn?: MatchPlace;
     score?: number | null;

package/dist/store/search-index.d.ts

@@ -24,7 +24,10 @@ export interface SearchIndexStore {
     }): Promise<SearchIndexFill>;
     /** How far the stems are built, and by which stemmer choices; `undefined` on a file before they existed. */
     stemsState(): Promise<StemsState | undefined>;
-    /** Builds the stems towards "ready" in short batches until done or `until` says stop; never rebuilds. */
+    /**
+     * Builds the stems towards "ready" in short batches until done or `until` says stop; never rebuilds. Then the
+     * voice-note transcripts written since, which are few, so they keep up with any stemmer choice.
+     */
     fillStems(options?: {
         until?: () => boolean;
     }): Promise<StemsFill>;

package/dist/store/search-index.js

@@ -1,3 +1,4 @@
+import { CORPORA, corpusIndexState, drainCorpus } from "./sqlite/note-index.js";
 import * as searchIndex from "./sqlite/search-index.js";
 import * as stems from "./sqlite/stems.js";
 import { fillStems } from "./sqlite/stems-fill.js";
@@ -5,13 +6,21 @@ import { write } from "./sqlite/transaction.js";
 import * as words from "./sqlite/words.js";
 export const searchIndexStoreOver = (context) => {
     const { database } = context;
+    const stemmerFor = stems.stemmerCache();
     return {
         knownTerms: async (terms) => words.knownTerms(context, terms),
         termCandidates: async (trigrams, lengths) => words.termCandidates(context, trigrams, lengths),
         searchIndexState: async () => searchIndex.searchIndexState(database),
         fillSearchIndex: async ({ until } = {}) => searchIndex.fillSearchIndex(database, { now: context.now, ...(until ? { until } : {}) }),
         stemsState: async () => stems.stemsState(database),
-        fillStems: async ({ until } = {}) => fillStems(database, { now: context.now, ...(until ? { until } : {}) }),
+        fillStems: async ({ until } = {}) => {
+            const filled = fillStems(database, { now: context.now, ...(until ? { until } : {}) });
+            // Read first: every search asks, and an idle drain must not take the write lock.
+            const transcripts = corpusIndexState(database, CORPORA.transcript);
+            if (transcripts.wanted !== null && (transcripts.pending > 0 || transcripts.built !== transcripts.wanted))
+                drainCorpus(database, CORPORA.transcript, stemmerFor, until ? { until } : {});
+            return filled;
+        },
         stemmers: async () => stems.savedStemmers(database),
         saveStemmers: async (stemmers) => write(database, () => stems.saveStoreStemmers(database, stemmers, context.now())),
     };

package/dist/store/sqlite/hit-places.d.ts

@@ -0,0 +1,12 @@
+import type { ResolvedPredicate } from "../../search/lucene/resolved.js";
+import type { CacheDatabase } from "../driver.js";
+import { type Corpus, type Leaf } from "./lucene-fields.js";
+import type { hitsByPk } from "./search.js";
+import { type TranscriptTerms } from "./transcript-query.js";
+type Hits = ReturnType<typeof hitsByPk>;
+/**
+ * Which place of each hit holds the words, first match wins in the order of `MatchPlace`; a hit its transcript
+ * holds only through a spelling speech recognition garbled is `corrected`.
+ */
+export declare const placeHits: (database: CacheDatabase, corpus: Corpus, leaves: Leaf[], transcripts: Map<ResolvedPredicate, TranscriptTerms>, found: Hits, pks: number[]) => Hits;
+export {};

package/dist/store/sqlite/hit-places.js

@@ -0,0 +1,76 @@
+import { normalize } from "../normalize.js";
+import { ftsMatch, SPEECH } from "./lucene-fields.js";
+import { spokenTranscripts } from "./transcript-query.js";
+/**
+ * Which place of each hit holds the words, first match wins in the order of `MatchPlace`; a hit its transcript
+ * holds only through a spelling speech recognition garbled is `corrected`.
+ */
+export const placeHits = (database, corpus, leaves, transcripts, found, pks) => {
+    const placeLeaves = leaves.filter(({ node }) => (node.operator === "term" || node.operator === "phrase") &&
+        (node.implicit || node.field === "content" || node.field === "transcript"));
+    if (!placeLeaves.length || !pks.length || found.length !== pks.length)
+        return found;
+    const keys = JSON.stringify(pks);
+    const wordy = placeLeaves.filter(({ node }) => /[\p{L}\p{N}]/u.test(normalize(node.value)));
+    if (!wordy.length)
+        return found;
+    const match = wordy.map(({ node }) => ftsMatch(node)).join(" OR ");
+    const implicit = placeLeaves.some(({ node }) => node.implicit);
+    const inText = new Set(implicit
+        ? database
+            .prepare(`SELECT rowid AS id FROM ${corpus.words} WHERE ${corpus.words} MATCH ? AND rowid IN (SELECT value FROM json_each(?))${placeLeaves.some(({ fragment }) => fragment.stems)
+            ? ` UNION SELECT rowid FROM ${corpus.stems} WHERE ${corpus.stems} MATCH ? AND rowid IN (SELECT value FROM json_each(?))`
+            : ""}`)
+            .all(`normalized_text : (${match})`, keys, ...(placeLeaves.some(({ fragment }) => fragment.stems)
+            ? [
+                `stems : (${placeLeaves.flatMap(({ fragment }) => (fragment.stems ? [`(${fragment.stems})`] : [])).join(" OR ")})`,
+                keys,
+            ]
+            : []))
+            .map(({ id }) => Number(id))
+        : []);
+    const files = database
+        .prepare(`SELECT att.attachable_id AS id, att.text AS text, ${SPEECH} AS speech FROM attachments att WHERE att.attachable_type='${corpus.item}' AND att.attachable_id IN (SELECT value FROM json_each(?)) AND att.id IN (SELECT rowid FROM attachment_words WHERE attachment_words MATCH ?) ORDER BY att.position`)
+        .all(keys, match);
+    const fileFields = new Set(placeLeaves.flatMap(({ node }) => (node.implicit ? ["transcript", "content"] : [node.field])));
+    const spoken = spokenTranscripts(database, corpus.item, wordy.flatMap(({ node }) => transcripts.get(node) ?? []), keys);
+    const allNames = implicit
+        ? database
+            .prepare(`SELECT att.attachable_id AS id, att.name AS name FROM attachments att WHERE att.attachable_type='${corpus.item}' AND att.attachable_id IN (SELECT value FROM json_each(?)) AND att.name IS NOT NULL ORDER BY att.position`)
+            .all(keys)
+        : [];
+    return found.map((hit, index) => {
+        const pk = pks[index];
+        const transcript = spoken.get(pk);
+        const file = files.find((row) => Number(row.id) === pk && Number(row.speech) !== 1);
+        const name = allNames.find((row) => {
+            if (Number(row.id) !== pk)
+                return false;
+            const words = normalize(String(row.name)).split(/[^\p{L}\p{N}]+/u);
+            return wordy.some(({ node }) => {
+                const phrase = normalize(node.value).match(/[\p{L}\p{N}]+/gu) ?? [];
+                if (node.slop !== undefined)
+                    return phrase.every((word) => words.includes(word));
+                return words.some((_, at) => phrase.every((word, offset) => words[at + offset] === word));
+            });
+        });
+        const place = inText.has(pk)
+            ? ["text", hit.text]
+            : transcript && fileFields.has("transcript")
+                ? ["transcript", String(transcript.text)]
+                : name
+                    ? ["filename", String(name.name)]
+                    : file && fileFields.has("content")
+                        ? ["file", String(file.text)]
+                        : undefined;
+        if (!place)
+            return hit;
+        const fixes = place[0] === "transcript" ? (transcript?.corrections ?? []) : [];
+        return {
+            ...hit,
+            matchedIn: place[0],
+            matchedText: place[1],
+            ...(fixes.length ? { match: "corrected", corrections: fixes } : {}),
+        };
+    });
+};

package/dist/store/sqlite/lucene-fields.d.ts

@@ -1,5 +1,6 @@
 import { type Automaton, type MatchBudget } from "../../search/lucene/automaton.js";
 import type { QueryExecution, ResolvedPredicate } from "../../search/lucene/resolved.js";
+import { type Stemmer } from "../../search/stem.js";
 import type { CacheDatabase, SqlValue } from "../driver.js";
 export interface Fragment {
     sql: string;
@@ -57,6 +58,11 @@ export interface FieldScope {
         sql: string;
         params: SqlValue[];
     };
+    /** The items whose voice-note transcripts a leaf matches: typed, in another form, or garbled. */
+    transcripts: (node: ResolvedPredicate) => {
+        sql: string;
+        params: SqlValue[];
+    };
     everywhere: (node: ResolvedPredicate, fragment: Fragment) => Fragment;
     /** Counts the patterns' expanded words against the query's budget. */
     expand: (count: number) => number;
@@ -67,6 +73,8 @@ type Compiled = {
     test?: Leaf["test"];
 } | Leaf;
 type FieldCompiler = (node: ResolvedPredicate, scope: FieldScope) => Compiled;
+/** A leaf as an FTS5 match over a stem index: phrases of stems, or with `~N` their `NEAR`. */
+export declare const stemMatch: (stemmer: Stemmer, node: ResolvedPredicate) => string | undefined;
 /** One compiler per field of `QUERY_FIELDS`, by its name: a leaf compiles by a lookup, never a branch. */
 export declare const FIELD_COMPILERS: Readonly<Record<string, FieldCompiler>>;
 export {};

package/dist/store/sqlite/lucene-fields.js

@@ -72,10 +72,12 @@ const mentioned = ({ mail }, where, params) => mail
     : bound(`m.id IN (SELECT mn.source_id FROM mentions mn WHERE mn.source_type='message' AND ${where} ` +
         "UNION SELECT a.attachable_id FROM mentions mn JOIN attachments a ON a.id=mn.source_id " +
         `AND a.attachable_type='message' WHERE mn.source_type='attachment' AND ${where})`, ...params, ...params);
-const stemsOf = ({ execution: { stemmer } }, node) => {
+const stemsOf = ({ execution: { stemmer } }, node) => stemmer ? stemMatch(stemmer, node) : undefined;
+/** A leaf as an FTS5 match over a stem index: phrases of stems, or with `~N` their `NEAR`. */
+export const stemMatch = (stemmer, node) => {
     if (node.slop === undefined)
-        return stemmer?.phrases(node.value);
-    const sequences = (stemmer?.stemSequences(node.value) ?? []).filter((sequence) => sequence.length);
+        return stemmer.phrases(node.value);
+    const sequences = stemmer.stemSequences(node.value).filter((sequence) => sequence.length);
     return sequences.length
         ? sequences
             .map((sequence) => (sequence.length > 1 ? nearMatch(sequence, node.slop) : quoted(sequence.join(""))))
@@ -256,7 +258,7 @@ const size = (node, { corpus }) => {
 const attachmentText = (which) => (node, scope) => {
     const text = normalize(node.value);
     // Never `fts`: the ranking ANDs every required word into message_words, where a file's words are not.
-    const set = scope.attachmentWords(ftsMatch(node), which);
+    const set = which === "transcript" ? scope.transcripts(node) : scope.attachmentWords(ftsMatch(node), which);
     return always(/[\p{L}\p{N}]/u.test(text) ? bound(`m.id IN (${set.sql})`, ...set.params) : bound("0"));
 };
 const date = (node) => {

package/dist/store/sqlite/lucene.js

@@ -2,11 +2,12 @@ import { CliError } from "@wirecat/cli-base";
 import { compileAutomaton } from "../../search/lucene/automaton.js";
 import { isStemmed } from "../../search/lucene/resolved.js";
 import { exhausted, QUERY_LIMITS, searchError } from "../../search/lucene/types.js";
-import { normalize } from "../normalize.js";
 import { emailHitsByPk } from "./emails.js";
+import { placeHits } from "./hit-places.js";
 import { combine, FIELD_COMPILERS, ftsMatch, SPEECH, } from "./lucene-fields.js";
 import { hitsByPk } from "./search.js";
 import { read } from "./transaction.js";
+import { transcriptItems, transcriptTerms } from "./transcript-query.js";
 export { prefixOf } from "./lucene-fields.js";
 const MESSAGES = {
     rows: "messages",
@@ -82,13 +83,19 @@ const compileQuery = (context, execution, boundedAttachments = false) => {
         sql: `SELECT att.attachable_id FROM attachments att WHERE att.attachable_type='${corpus.item}'${which === "any" ? "" : ` AND ${which === "file" ? "NOT " : ""}${SPEECH}`} AND att.id IN (SELECT rowid FROM attachment_words WHERE attachment_words MATCH ?)`,
         params: [match],
     });
+    const transcripts = transcriptTerms(database, execution.root, corpus.item);
+    const said = (node, part) => transcriptItems(corpus.item, transcripts.get(node) ?? { match: ftsMatch(node), widened: [], from: node.value }, part);
     /** A bare word also matches a transcript, a file's text and a word of a file's name. */
     const everywhere = (node, fragment) => {
         if (!node.implicit || fragment.sql === "0")
             return fragment;
         const match = ftsMatch(node);
-        const files = attachmentWords(match, "any");
-        const elsewhere = { sql: `${files.sql} UNION ${namedSql}`, params: [...files.params, match] };
+        const files = attachmentWords(match, "file");
+        const spoken = said(node);
+        const elsewhere = {
+            sql: `${files.sql} UNION ${spoken.sql} UNION ${namedSql}`,
+            params: [...files.params, ...spoken.params, match],
+        };
         return {
             ...fragment,
             sql: `(${fragment.sql} OR m.id IN (${elsewhere.sql}))`,
@@ -107,6 +114,7 @@ const compileQuery = (context, execution, boundedAttachments = false) => {
         patternOf,
         expanded,
         attachmentWords,
+        transcripts: (node) => said(node),
         everywhere,
         expand: (count) => {
             expansions += count;
@@ -284,14 +292,21 @@ const compileQuery = (context, execution, boundedAttachments = false) => {
         ranked.sql.push(`f(id, rank) AS MATERIALIZED (SELECT rowid, bm25(${rankIndex.table}, 1.0, 0.0) FROM ${rankIndex.table} WHERE ${rankIndex.table} MATCH ?)`);
         ranked.params.push(`${rankIndex.column} : (${ranking})`);
     }
-    // Among hits the message text does not hold: a transcript, then a file's name, then a file's text. bm25 scores
-    // of different indexes do not compare, and a long spreadsheet must not crowd out what was said.
-    const places = [...fragments.values()].flatMap(({ node, fragment }) => fragment.elsewhere && fragment.fts ? [ftsMatch(node)] : []);
+    // Among hits the message text does not hold: a transcript as typed, in another form, then a file's name, a file's
+    // text, and last a garbled spelling. bm25 scores of different indexes do not compare, and a long spreadsheet must
+    // not crowd out what was said.
+    const placeNodes = [...fragments.values()].flatMap(({ node, fragment }) => fragment.elsewhere && fragment.fts ? [node] : []);
+    const places = placeNodes.map(ftsMatch);
     if (places.length && !execution.newest) {
-        ranked.sql.push(`s(id, tier) AS MATERIALIZED (SELECT att.attachable_id, min(CASE WHEN ${SPEECH} THEN 1 ELSE 3 END) FROM attachments att WHERE att.attachable_type='${corpus.item}' AND att.id IN (SELECT rowid FROM attachment_words WHERE attachment_words MATCH ?) GROUP BY att.attachable_id UNION ALL SELECT attachable_id, 2 FROM (${namedSql}))`);
-        ranked.params.push(places.join(" OR "), places.join(" OR "));
+        const spoken = (part, tier) => placeNodes.map((node) => {
+            const one = said(node, part);
+            return { sql: `SELECT attachable_id, ${tier} FROM (${one.sql})`, params: one.params };
+        });
+        const parts = [...spoken("forms", 2), ...spoken("widened", 5)];
+        ranked.sql.push(`s(id, tier) AS MATERIALIZED (SELECT att.attachable_id, min(CASE WHEN ${SPEECH} THEN 1 ELSE 4 END) FROM attachments att WHERE att.attachable_type='${corpus.item}' AND att.id IN (SELECT rowid FROM attachment_words WHERE attachment_words MATCH ?) GROUP BY att.attachable_id UNION ALL SELECT attachable_id, 3 FROM (${namedSql})${parts.map(({ sql }) => ` UNION ALL ${sql}`).join("")})`);
+        ranked.params.push(places.join(" OR "), places.join(" OR "), ...parts.flatMap(({ params }) => params));
     }
-    const placeTier = places.length && !execution.newest ? "coalesce((SELECT min(tier) FROM s WHERE s.id=m.id), 4)," : "";
+    const placeTier = places.length && !execution.newest ? "coalesce((SELECT min(tier) FROM s WHERE s.id=m.id), 6)," : "";
     if (exactTier) {
         ranked.sql.push(`x(id) AS MATERIALIZED (SELECT rowid FROM ${corpus.words} WHERE ${corpus.words} MATCH ?)`);
         ranked.params.push(`normalized_text : (${exactTier})`);
@@ -336,67 +351,7 @@ const compileQuery = (context, execution, boundedAttachments = false) => {
             count: Number(row.count),
         }));
     };
-    const placeLeaves = [...fragments.values()].filter(({ node }) => (node.operator === "term" || node.operator === "phrase") &&
-        (node.implicit || node.field === "content" || node.field === "transcript"));
-    /** Which place of each hit holds the words, first match wins in the order of `MatchPlace`. */
-    const placed = (found, pks) => {
-        if (!placeLeaves.length || !pks.length || found.length !== pks.length)
-            return found;
-        const keys = JSON.stringify(pks);
-        const wordy = placeLeaves.filter(({ node }) => /[\p{L}\p{N}]/u.test(normalize(node.value)));
-        if (!wordy.length)
-            return found;
-        const match = wordy.map(({ node }) => ftsMatch(node)).join(" OR ");
-        const implicit = placeLeaves.some(({ node }) => node.implicit);
-        const inText = new Set(implicit
-            ? database
-                .prepare(`SELECT rowid AS id FROM ${corpus.words} WHERE ${corpus.words} MATCH ? AND rowid IN (SELECT value FROM json_each(?))${placeLeaves.some(({ fragment }) => fragment.stems)
-                ? ` UNION SELECT rowid FROM ${corpus.stems} WHERE ${corpus.stems} MATCH ? AND rowid IN (SELECT value FROM json_each(?))`
-                : ""}`)
-                .all(`normalized_text : (${match})`, keys, ...(placeLeaves.some(({ fragment }) => fragment.stems)
-                ? [
-                    `stems : (${placeLeaves.flatMap(({ fragment }) => (fragment.stems ? [`(${fragment.stems})`] : [])).join(" OR ")})`,
-                    keys,
-                ]
-                : []))
-                .map(({ id }) => Number(id))
-            : []);
-        const files = database
-            .prepare(`SELECT att.attachable_id AS id, att.text AS text, ${SPEECH} AS speech FROM attachments att WHERE att.attachable_type='${corpus.item}' AND att.attachable_id IN (SELECT value FROM json_each(?)) AND att.id IN (SELECT rowid FROM attachment_words WHERE attachment_words MATCH ?) ORDER BY att.position`)
-            .all(keys, match);
-        const fileFields = new Set(placeLeaves.flatMap(({ node }) => (node.implicit ? ["transcript", "content"] : [node.field])));
-        const allNames = implicit
-            ? database
-                .prepare(`SELECT att.attachable_id AS id, att.name AS name FROM attachments att WHERE att.attachable_type='${corpus.item}' AND att.attachable_id IN (SELECT value FROM json_each(?)) AND att.name IS NOT NULL ORDER BY att.position`)
-                .all(keys)
-            : [];
-        return found.map((hit, index) => {
-            const pk = pks[index];
-            const transcript = files.find((row) => Number(row.id) === pk && Number(row.speech) === 1);
-            const file = files.find((row) => Number(row.id) === pk && Number(row.speech) !== 1);
-            const name = allNames.find((row) => {
-                if (Number(row.id) !== pk)
-                    return false;
-                const words = normalize(String(row.name)).split(/[^\p{L}\p{N}]+/u);
-                return wordy.some(({ node }) => {
-                    const phrase = normalize(node.value).match(/[\p{L}\p{N}]+/gu) ?? [];
-                    if (node.slop !== undefined)
-                        return phrase.every((word) => words.includes(word));
-                    return words.some((_, at) => phrase.every((word, offset) => words[at + offset] === word));
-                });
-            });
-            const place = inText.has(pk)
-                ? ["text", hit.text]
-                : transcript && fileFields.has("transcript")
-                    ? ["transcript", String(transcript.text)]
-                    : name
-                        ? ["filename", String(name.name)]
-                        : file && fileFields.has("content")
-                            ? ["file", String(file.text)]
-                            : undefined;
-            return place ? { ...hit, matchedIn: place[0], matchedText: place[1] } : hit;
-        });
-    };
+    const placed = (found, pks) => placeHits(database, corpus, [...fragments.values()], transcripts, found, pks);
     const hits = (pks) => placed(mail ? emailHitsByPk(database, pks) : hitsByPk(context, pks), pks);
     return {
         corpus,

package/dist/store/sqlite/manifest.js

@@ -34,6 +34,8 @@ export const MANIFEST = [
     { name: "20261021214318_version-26-message-reactions", version: 26, minCompatible: 1 },
     { name: "20261021221808_version-27-message-delivery", version: 27, minCompatible: 1 },
     { name: "20261021224625_version-28-chat-state", version: 28, minCompatible: 1 },
+    { name: "20261023105057_version-29-transcript-words", version: 29, minCompatible: 1 },
+    { name: "20261023105101_version-29-transcript-words-triggers", version: 29, minCompatible: 1 },
 ];
 export const generatedMigrations = (manifest = MANIFEST, generated = GENERATED) => {
     const migrations = [];

package/dist/store/sqlite/migrations.generated.js

@@ -577,5 +577,24 @@ export const GENERATED = [
             "ALTER TABLE `chats` ADD `marked_unread` integer;",
             "ALTER TABLE `chats` ADD `unread_counted_at` integer;"
         ]
+    },
+    {
+        "name": "20261023105057_version-29-transcript-words",
+        "statements": [
+            "CREATE TABLE `transcript_index_pending` (\n\t`id` integer NOT NULL,\n\t`indexable_type` text NOT NULL,\n\tCONSTRAINT `transcript_index_pending_pk` PRIMARY KEY(`indexable_type`, `id`)\n);"
+        ]
+    },
+    {
+        "name": "20261023105101_version-29-transcript-words-triggers",
+        "statements": [
+            "-- Words and stems of a voice note's transcript, rowid = attachment id. The words are the vocabulary garbled\n-- spellings are read from; the typed form is still found in attachment_words. Stemming needs JS, so the\n-- triggers only queue the attachment and the next search indexes it, as tasks do.\nCREATE VIRTUAL TABLE transcript_words USING fts5(\n  normalized_text, scope,\n  content = '', contentless_delete = 1,\n  tokenize = 'unicode61 remove_diacritics 2', prefix = '3');",
+            "CREATE VIRTUAL TABLE transcript_words_vocab USING fts5vocab(transcript_words, 'col');",
+            "CREATE VIRTUAL TABLE transcript_stems USING fts5(\n  stems, scope,\n  content = '', contentless_delete = 1,\n  tokenize = 'unicode61 remove_diacritics 2');",
+            "-- A transcript is the text of a message's voice attachment, whoever wrote it, as `speech()` in lucene-fields.ts.\nCREATE TRIGGER transcript_index_ai AFTER INSERT ON attachments\n  WHEN new.attachable_type = 'message' AND (coalesce(new.extractor, '') LIKE 'speech:%'\n    OR new.kind IN ('voice', 'video_note') OR coalesce(new.mime, '') = 'audio/ogg') BEGIN\n  INSERT OR IGNORE INTO transcript_index_pending (indexable_type, id) VALUES ('attachment', new.id);\nEND;",
+            "-- Queued whether it became a transcript or stopped being one: the drain drops what is no longer live.\nCREATE TRIGGER transcript_index_au AFTER UPDATE OF text, kind, mime, extractor, attachable_type ON attachments\n  WHEN (old.text IS NOT new.text OR old.kind IS NOT new.kind OR old.mime IS NOT new.mime\n    OR old.extractor IS NOT new.extractor OR old.attachable_type IS NOT new.attachable_type)\n    AND ((new.attachable_type = 'message' AND (coalesce(new.extractor, '') LIKE 'speech:%'\n      OR new.kind IN ('voice', 'video_note') OR coalesce(new.mime, '') = 'audio/ogg'))\n    OR (old.attachable_type = 'message' AND (coalesce(old.extractor, '') LIKE 'speech:%'\n      OR old.kind IN ('voice', 'video_note') OR coalesce(old.mime, '') = 'audio/ogg'))) BEGIN\n  INSERT OR IGNORE INTO transcript_index_pending (indexable_type, id) VALUES ('attachment', new.id);\nEND;",
+            "CREATE TRIGGER transcript_index_ad AFTER DELETE ON attachments BEGIN\n  DELETE FROM transcript_words WHERE rowid = old.id;\n  DELETE FROM transcript_stems WHERE rowid = old.id;\n  DELETE FROM transcript_index_pending WHERE indexable_type = 'attachment' AND id = old.id;\nEND;",
+            "INSERT OR IGNORE INTO search_index_state (name, watermark, filled_through, terms_through, normalizer_version, built_at, analyzer)\n  VALUES ('transcript_index', 0, 0, 0, 1, NULL, NULL);",
+            "INSERT OR IGNORE INTO transcript_index_pending (indexable_type, id)\n  SELECT 'attachment', id FROM attachments WHERE attachable_type = 'message' AND coalesce(text, '') <> ''\n    AND (coalesce(extractor, '') LIKE 'speech:%' OR kind IN ('voice', 'video_note') OR coalesce(mime, '') = 'audio/ogg');"
+        ]
     }
 ];

package/dist/store/sqlite/note-index.d.ts

@@ -15,12 +15,14 @@ export interface NoteIndexState {
  */
 export interface Corpus {
     /** The singular table name: `indexable_type` in the queue, `chunkable_type` in chunks. */
-    type: "document" | "note" | "memory" | "email" | "task";
+    type: "document" | "note" | "memory" | "email" | "task" | "transcript";
     /** Its row in `search_index_state`. */
     index: string;
     table: string;
     /** Title and body as `title`/`body`, whether it is live, and its scope, for one row by `id`. */
     read: string;
+    /** `indexable_type` in the queue when it is not `type`: the table a queued id points into. */
+    queued?: "attachment";
     /** False when the corpus cuts its own pieces for search by meaning, as mail does by thread. */
     pieces?: false;
     /** The words and stems rows, when the corpus cleans its text its own way; `normalize` and `indexText` otherwise. */
@@ -66,6 +68,14 @@ export declare const CORPORA: {
             stems: string;
         };
     };
+    readonly transcript: {
+        readonly type: "transcript";
+        readonly queued: "attachment";
+        readonly index: "transcript_index";
+        readonly table: `(SELECT a.id FROM attachments a WHERE a.attachable_type = 'message' AND ${string})`;
+        readonly read: string;
+        readonly pieces: false;
+    };
 };
 /** What a row is indexed as: its title, then its text. Chunk offsets point into this. */
 export declare const noteIndexText: (title: string | null, text: string) => string;

package/dist/store/sqlite/note-index.js

@@ -5,6 +5,7 @@ import { normalize } from "../normalize.js";
 import { dropDocumentPicture } from "./document-pictures.js";
 import { int } from "./events.js";
 import { indexRow } from "./index-state.js";
+import { speech } from "./lucene-fields.js";
 import { savedStemmers } from "./stems.js";
 import { write } from "./transaction.js";
 export const CORPORA = {
@@ -45,13 +46,24 @@ export const CORPORA = {
         pieces: false,
         terms: (text, stemmer) => ({ words: taskWordsOf(text), stems: [...taskStemsOf(text, stemmer)].join(" ") }),
     },
+    transcript: {
+        type: "transcript",
+        queued: "attachment",
+        index: "transcript_index",
+        table: `(SELECT a.id FROM attachments a WHERE a.attachable_type = 'message' AND ${speech("a")})`,
+        read: `SELECT NULL AS title, coalesce(a.text, '') AS body, a.attachable_type = 'message' AND ${speech("a")} AS live, ` +
+            "'personal' AS scope FROM attachments a WHERE a.id = ?",
+        pieces: false,
+    },
 };
 /** What a row is indexed as: its title, then its text. Chunk offsets point into this. */
 export const noteIndexText = (title, text) => (title ? `${title}\n\n${text}` : text);
+const queuedAs = (corpus) => corpus.queued ?? corpus.type;
 export const corpusIndexState = (database, corpus) => {
     const row = indexRow(database, corpus.index);
-    const pending = Number(database.prepare(`SELECT count(*) AS n FROM ${corpus.type}_index_pending WHERE indexable_type = ?`).get(corpus.type)
-        ?.n);
+    const pending = Number(database
+        .prepare(`SELECT count(*) AS n FROM ${corpus.type}_index_pending WHERE indexable_type = ?`)
+        .get(queuedAs(corpus))?.n);
     const saved = savedStemmers(database);
     const wanted = saved === null ? null : analyzerIdentity(saved ?? DEFAULT_STEMMERS);
     const built = row.analyzer === null ? null : String(row.analyzer);
@@ -126,7 +138,7 @@ const writer = (database, corpus, stemmer) => {
                 statements.purge.run(hash, hash);
         if (!live && corpus.type === "document")
             dropDocumentPicture(database, id);
-        statements.dequeue.run(corpus.type, id);
+        statements.dequeue.run(queuedAs(corpus), id);
     };
 };
 /**
@@ -154,7 +166,7 @@ export const drainCorpus = (database, corpus, stemmerFor, { batch = 200, until =
             if (state.built !== null)
                 database
                     .prepare(`INSERT OR IGNORE INTO ${queue} (indexable_type, id) SELECT ?, id FROM ${corpus.table}`)
-                    .run(corpus.type);
+                    .run(queuedAs(corpus));
             database.prepare("UPDATE search_index_state SET analyzer = ? WHERE name = ?").run(stemmer.identity, corpus.index);
         });
     }
@@ -167,7 +179,7 @@ export const drainCorpus = (database, corpus, stemmerFor, { batch = 200, until =
         if (until())
             break;
         const count = write(database, () => {
-            const ids = next.all(corpus.type, batch).map((row) => Number(row.id));
+            const ids = next.all(queuedAs(corpus), batch).map((row) => Number(row.id));
             for (const id of ids)
                 index(id);
             return ids.length;
@@ -187,7 +199,7 @@ export const resetNoteIndex = (database) => {
         for (const corpus of Object.values(CORPORA))
             database
                 .prepare(`INSERT OR IGNORE INTO ${corpus.type}_index_pending (indexable_type, id) SELECT ?, id FROM ${corpus.table}`)
-                .run(corpus.type);
+                .run(queuedAs(corpus));
     });
     return true;
 };

package/dist/store/sqlite/schema.d.ts

@@ -6497,6 +6497,45 @@ export declare const attachments: import("drizzle-orm/sqlite-core").SQLiteTableW
     };
     dialect: 'sqlite';
 }>;
+export declare const transcriptIndexPending: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
+    name: "transcript_index_pending";
+    schema: undefined;
+    columns: {
+        id: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "transcript_index_pending";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+        indexableType: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "transcript_index_pending";
+            dataType: "string";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+    };
+    dialect: 'sqlite';
+}>;
 export { documentAnnotationTransfers } from "./schema/document-transfers.js";
 export declare const documents: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
     name: "documents";

package/dist/store/sqlite/schema.js

@@ -715,6 +715,10 @@ export const attachments = sqliteTable("attachments", {
         .on(table.id)
         .where(sql `local_path IS NOT NULL AND local_path_folded IS NULL`),
 ]);
+export const transcriptIndexPending = sqliteTable("transcript_index_pending", {
+    id: integer("id").notNull(),
+    indexableType: text("indexable_type").notNull(),
+}, (table) => [primaryKey({ columns: [table.indexableType, table.id] })]);
 export { documentAnnotationTransfers } from "./schema/document-transfers.js";
 export const documents = sqliteTable("documents", {
     id: integer("id").primaryKey(),

package/dist/store/sqlite/transcript-query.d.ts

@@ -0,0 +1,40 @@
+import type { ResolvedNode, ResolvedPredicate } from "../../search/lucene/resolved.js";
+import type { CacheDatabase, SqlValue } from "../driver.js";
+/** How one leaf reaches voice-note transcripts: its typed form, other forms by stem, and garbled spellings. */
+export interface TranscriptTerms {
+    /** The leaf's words as typed, matched in `attachment_words`. */
+    match: string;
+    /** Any form of them, matched in `transcript_stems`; none under `exact:`. */
+    stems?: string;
+    /** Stored spellings speech recognition may have made of a single word; matched in `attachment_words`. */
+    widened: string[];
+    from: string;
+}
+/**
+ * Each leaf that reads transcripts, with what it matches there. A transcript always widens, not only when nothing
+ * matched as typed text does: a correct and a garbled spelling sit in one chat. Only a single word widens —
+ * FTS5 `NEAR` and phrases take no alternatives — and never under `exact:`, as meetings do.
+ */
+export declare const transcriptTerms: (database: CacheDatabase, root: ResolvedNode, item: string) => Map<ResolvedPredicate, TranscriptTerms>;
+export type TranscriptPart = "all" | "typed" | "forms" | "widened";
+/** The attachment ids a leaf's transcripts match, as a condition on `att.id`; `part` keeps one of them. */
+export declare const transcriptCondition: (terms: TranscriptTerms, part?: TranscriptPart) => {
+    sql: string;
+    params: SqlValue[];
+};
+/** The items whose transcripts a leaf matches. */
+export declare const transcriptItems: (item: string, terms: TranscriptTerms, part?: TranscriptPart) => {
+    sql: string;
+    params: SqlValue[];
+};
+/**
+ * Of the items `keys` names, the transcript the leaves match — one they match as typed or in another form first —
+ * and, when only a spelling recognition garbled matched it, which: as meetings say it.
+ */
+export declare const spokenTranscripts: (database: CacheDatabase, item: string, speaking: TranscriptTerms[], keys: string) => Map<number, {
+    text: string;
+    corrections: {
+        from: string;
+        to: string[];
+    }[];
+}>;