@wirecat/cli-core 0.379.0 → 0.380.0

package/dist/meetings/search-store.d.ts

@@ -1,3 +1,4 @@
+import type { QueryAst } from "@wirecat/cli-base/query";
 import type { SearchHit } from "./store.js";
 export interface MeetingSearchOptions {
     accountId: number;
@@ -37,7 +38,8 @@ export interface MeetingSearchPage {
 }
 /** Optional bounded read capability; paging reaches inside a single large transcript. */
 export interface MeetingSearchStore {
-    searchHits(query: string, options: MeetingSearchOptions): Promise<MeetingSearchPage>;
+    /** A string is what a person typed, parsed as a query; text that must not become syntax comes as `meetingWordsQuery`. */
+    searchHits(query: string | QueryAst, options: MeetingSearchOptions): Promise<MeetingSearchPage>;
 }
 export interface MeetingSearchIndexInput {
     accountId: number;

package/dist/meetings/store.d.ts

@@ -82,6 +82,13 @@ export interface SearchHit {
     id: number;
     text: string;
     startMs: number | null;
+    /** `corrected` when a query word matched only as a stored word recognition may have garbled. */
+    match?: "words" | "corrected";
+    /** For a `corrected` hit, the stored words each such query word matched in this passage. */
+    corrections?: {
+        from: string;
+        to: string[];
+    }[];
 }
 export interface ContractCase {
     name: string;

package/dist/meetings/transcript-text.d.ts

@@ -0,0 +1,20 @@
+import type { TranscriptLine } from "./model.js";
+export declare class TranscriptTextError extends Error {
+    name: string;
+}
+/** What the header of a text transcript says, as written; the caller reads the date in its zone. */
+export interface TranscriptTextHeader {
+    title: string | null;
+    date: string | null;
+    meetingId: string | null;
+    speakers: string[];
+}
+/**
+ * A transcript in the text form zoom-searcher wrote: a header (`Встреча:`, `Дата:`, `Zoom ID:`, `Говорят:`),
+ * a rule, then `[hh:mm:ss] Speaker: text` blocks apart by a blank line. A block may run over several lines.
+ * A line ends where the next begins: the file keeps start times only.
+ */
+export declare const parseTranscriptText: (input: string) => {
+    header: TranscriptTextHeader;
+    lines: TranscriptLine[];
+};

package/dist/meetings/transcript-text.js

@@ -0,0 +1,59 @@
+export class TranscriptTextError extends Error {
+    name = "TranscriptTextError";
+}
+const HEADER = {
+    "Встреча: ": "title",
+    "Дата: ": "date",
+    "Zoom ID: ": "meetingId",
+    "Говорят: ": "speakers",
+};
+const LINE = /^\[(\d{2,}):([0-5]\d):([0-5]\d)\]\s*(?:([^:\n]{1,60}):\s*)?([\s\S]*)$/;
+// Early files had no blank line after the rule, so the first line of speech is glued to it.
+const RULE = /^[—–-]{10,}[^\n]*(?:\n|$)/;
+const EMPTY = new Set(["", "—", "-"]);
+/**
+ * A transcript in the text form zoom-searcher wrote: a header (`Встреча:`, `Дата:`, `Zoom ID:`, `Говорят:`),
+ * a rule, then `[hh:mm:ss] Speaker: text` blocks apart by a blank line. A block may run over several lines.
+ * A line ends where the next begins: the file keeps start times only.
+ */
+export const parseTranscriptText = (input) => {
+    const text = input.replace(/^/, "").replace(/\r\n?/g, "\n");
+    const header = { title: null, date: null, meetingId: null, speakers: [] };
+    let sawHeader = false;
+    for (const line of text.split("\n")) {
+        if (!line.trim())
+            break;
+        for (const [prefix, key] of Object.entries(HEADER)) {
+            if (!line.startsWith(prefix))
+                continue;
+            sawHeader = true;
+            const value = line.slice(prefix.length).trim();
+            if (key === "speakers")
+                header.speakers = value
+                    .split(",")
+                    .map((name) => name.trim())
+                    .filter((name) => !EMPTY.has(name));
+            else
+                header[key] = EMPTY.has(value) ? null : value;
+        }
+    }
+    if (!sawHeader)
+        throw new TranscriptTextError('a text transcript starts with a header such as "Встреча: …"');
+    const found = [];
+    for (const block of text.split(/\n\s*\n/)) {
+        const match = LINE.exec(block.trim().replace(RULE, ""));
+        if (!match)
+            continue;
+        const [, hours = "0", minutes = "0", seconds = "0", speaker, said = ""] = match;
+        found.push({
+            startMs: ((Number(hours) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1000,
+            speaker: speaker?.trim() || null,
+            text: said.trim(),
+        });
+    }
+    const lines = found.map((line, at) => ({
+        ...line,
+        endMs: Math.max(line.startMs, found[at + 1]?.startMs ?? line.startMs),
+    }));
+    return { header, lines };
+};

package/dist/render/markdown.js

@@ -8,11 +8,11 @@ import { singleLine, visibleControls } from "@wirecat/cli-base";
  * as often as in an editor, and a byte that rewrites the screen is nobody's text.
  */
 export const toMarkdown = (title, messages, timeZone) => {
-    const zone = timeZone ? { timeZone } : {};
+    const zone = { timeZone: timeZone ?? Intl.DateTimeFormat().resolvedOptions().timeZone };
     const day = new Intl.DateTimeFormat("sv-SE", { ...zone, year: "numeric", month: "2-digit", day: "2-digit" });
     const clock = new Intl.DateTimeFormat("en-GB", { ...zone, hour: "2-digit", minute: "2-digit", hourCycle: "h23" });
     const byId = new Map(messages.map((message) => [message.id, message]));
-    const lines = [`# ${literal(singleLine(title))}`];
+    const lines = [`# ${literal(singleLine(title))}`, "", `Times are in ${zone.timeZone}.`];
     let today = "";
     for (const message of messages) {
         const when = new Date(message.timestamp);

package/dist/search/garbled.d.ts

@@ -0,0 +1,29 @@
+/** A query word as speech search widens it: its folded form and the stems the store's stemmers give it. */
+export interface GarbledProbe {
+    literal: string;
+    stems: string[];
+}
+/** ztx's floor: a shorter word has too many neighbours to widen safely. */
+export declare const SHORTEST_GARBLED = 4;
+export declare const isNumberWord: (word: string) => boolean;
+/** What the index already finds for the word: its beginning or one of its stems. */
+export declare const plainMatch: (probe: GarbledProbe, word: string, stems: string[]) => boolean;
+/**
+ * How far a stored word is from a query word, or `null` when it is not a candidate. `0` is another form the
+ * stemmer cut differently: a stem one side extends counts only when the other side's word spells the extension
+ * out — «циана» (stem `циа`) reaches «цианом» (`циан`), «устав» (`уста`) does not reach «установить» (`установ`).
+ * `1` is a spelling recognition may have garbled, compared by stem: «рилторов» is three edits from «риелтор»
+ * as a word but one as a stem.
+ */
+export declare const garbledDistance: (probe: GarbledProbe, word: string, stems: string[]) => number | null;
+/**
+ * The stored words each probe widens to, nearest first and then the most used, from one pass over the
+ * vocabulary, whose terms are folded as the index folds them. `stem` is the store's stemmer, so a stored
+ * word's stem is the one its index holds. A spelling that `known` finds in a dictionary is a different real
+ * word, not a garbled one — «договор» is one stem edit from «поговорить» — so only unknown spellings widen;
+ * on real transcripts that kept 517 of 10,271 widened passages.
+ */
+export declare const widenWords: (probes: GarbledProbe[], vocabulary: Iterable<{
+    term: string;
+    docs: number;
+}>, stem: (word: string) => string[], known: (word: string) => boolean) => string[][];

package/dist/search/garbled.js

@@ -0,0 +1,125 @@
+import { editDistance } from "./correct.js";
+import { createStemmer } from "./stem.js";
+/** ztx's floor: a shorter word has too many neighbours to widen safely. */
+export const SHORTEST_GARBLED = 4;
+const MOST_WIDENED = 64;
+/** The index folds ё but keeps э, and recognition writes «риэлтор» as often as «риелтор». */
+const plain = (word) => word.replaceAll("э", "е");
+const russian = createStemmer({ cyrillic: "russian", latin: "none" }, { cacheLimit: 10_000 });
+/**
+ * Recognition swaps one number for another («восемь» and «шесть» for one spoken word), and «десять» is one edit
+ * from «девять»; a widened number would be a wrong fact, so number words only ever match their own forms.
+ */
+const NUMBER_STEMS = new Set(`ноль нуль один одна одно одного одной одному одним одном одну два две двух двум двумя три трех трем тремя
+  четыре четырех четырем четырьмя пять пяти пятью шесть шести шестью семь семи семью восемь восьми восемью
+  девять девяти девятью десять десяти десятью одиннадцать двенадцать тринадцать четырнадцать пятнадцать
+  шестнадцать семнадцать восемнадцать девятнадцать двадцать двадцати тридцать тридцати сорок сорока пятьдесят
+  пятидесяти шестьдесят шестидесяти семьдесят семидесяти восемьдесят восьмидесяти девяносто девяноста сто ста
+  двести двухсот триста трехсот четыреста четырехсот пятьсот пятисот шестьсот семьсот восемьсот девятьсот
+  тысяча тысячи тысяч миллион миллиона миллионов миллиард первый второй третий четвертый пятый шестой седьмой
+  восьмой девятый десятый сотый тысячный половина треть четверть`
+    .split(/\s+/)
+    .map((word) => russian.stemToken(word)));
+export const isNumberWord = (word) => /\d/.test(word) || NUMBER_STEMS.has(russian.stemToken(word));
+/**
+ * One edit between stems, tighter than typed-text correction (`maxEdits`), because a transcript always widens
+ * instead of only when nothing matched: two edits turn «проверка» into «провела», «работа» into «заботы».
+ * A stem shorter than this keeps its first letter; a longer one may gain one, as «дриелтора» does.
+ */
+const LOOSE_START = 7;
+/** What the index already finds for the word: its beginning or one of its stems. */
+export const plainMatch = (probe, word, stems) => word.startsWith(probe.literal) || stems.some((stem) => probe.stems.includes(stem));
+/**
+ * How far a stored word is from a query word, or `null` when it is not a candidate. `0` is another form the
+ * stemmer cut differently: a stem one side extends counts only when the other side's word spells the extension
+ * out — «циана» (stem `циа`) reaches «цианом» (`циан`), «устав» (`уста`) does not reach «установить» (`установ`).
+ * `1` is a spelling recognition may have garbled, compared by stem: «рилторов» is three edits from «риелтор»
+ * as a word but one as a stem.
+ */
+export const garbledDistance = (probe, word, stems) => {
+    if (probe.literal.length < SHORTEST_GARBLED || plainMatch(probe, word, stems))
+        return null;
+    if (isNumberWord(probe.literal) || isNumberWord(word))
+        return null;
+    const literal = plain(probe.literal);
+    const stored = plain(word);
+    let best = null;
+    for (const queryStem of probe.stems.map(plain))
+        for (const wordStem of stems.map(plain)) {
+            if (Math.min(queryStem.length, wordStem.length) < 3)
+                continue;
+            if ((wordStem.startsWith(queryStem) && literal.startsWith(wordStem)) ||
+                (queryStem.startsWith(wordStem) && stored.startsWith(queryStem)))
+                return 0;
+            if (queryStem.length < SHORTEST_GARBLED)
+                continue;
+            if (queryStem.length < LOOSE_START && queryStem[0] !== wordStem[0])
+                continue;
+            const distance = editDistance(queryStem, wordStem, 1);
+            if (distance <= 1)
+                best = distance;
+        }
+    return best;
+};
+/** `editDistance` from `stem` to the nearest beginning of `word`, or `max + 1` once every beginning is further. */
+const prefixDistance = (stem, word, max) => {
+    let before = new Int32Array(word.length + 1);
+    let previous = new Int32Array(word.length + 1);
+    let current = new Int32Array(word.length + 1);
+    for (let i = 1; i <= stem.length; i++) {
+        current[0] = i;
+        let lowest = i;
+        for (let j = 1; j <= word.length; j++) {
+            let value = Math.min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (stem[i - 1] === word[j - 1] ? 0 : 1));
+            if (i > 1 && j > 1 && stem[i - 1] === word[j - 2] && stem[i - 2] === word[j - 1])
+                value = Math.min(value, before[j - 2] + 1);
+            current[j] = value;
+            lowest = Math.min(lowest, value);
+        }
+        if (lowest > max)
+            return max + 1;
+        [before, previous, current] = [previous, current, before];
+    }
+    return previous.reduce((lowest, value) => Math.min(lowest, value), max + 1);
+};
+/**
+ * Whether a stored word can pass `garbledDistance`, checked without stemming it: a stem begins its word, so a
+ * near stem means a near beginning. Stemming every stored word in a fresh process costs half a second.
+ */
+const worthStemming = (probe, word) => {
+    const stored = plain(word);
+    return probe.stems.some((raw) => {
+        const stem = plain(raw);
+        if (stem.length >= 3 && stored.startsWith(stem))
+            return true;
+        return stem.length >= SHORTEST_GARBLED && prefixDistance(stem, stored, 1) <= 1;
+    });
+};
+/**
+ * The stored words each probe widens to, nearest first and then the most used, from one pass over the
+ * vocabulary, whose terms are folded as the index folds them. `stem` is the store's stemmer, so a stored
+ * word's stem is the one its index holds. A spelling that `known` finds in a dictionary is a different real
+ * word, not a garbled one — «договор» is one stem edit from «поговорить» — so only unknown spellings widen;
+ * on real transcripts that kept 517 of 10,271 widened passages.
+ */
+export const widenWords = (probes, vocabulary, stem, known) => {
+    const found = probes.map(() => []);
+    const live = probes.map((probe) => probe.literal.length >= SHORTEST_GARBLED);
+    if (!live.some(Boolean))
+        return probes.map(() => []);
+    for (const { term: word, docs } of vocabulary) {
+        let stems;
+        probes.forEach((probe, at) => {
+            if (!live[at] || !worthStemming(probe, word))
+                return;
+            stems ??= stem(word);
+            const distance = garbledDistance(probe, word, stems);
+            if (distance === 0 || (distance === 1 && !known(word)))
+                found[at]?.push({ term: word, docs, distance });
+        });
+    }
+    return found.map((words) => words
+        .sort((a, b) => a.distance - b.distance || b.docs - a.docs || a.term.localeCompare(b.term))
+        .slice(0, MOST_WIDENED)
+        .map((word) => word.term));
+};

package/dist/search/russian-dictionary.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Whether a word the index stores is a real Russian word. The index folds й into и and ё into е, so each is
+ * tried back one letter at a time.
+ */
+export declare const knownRussianWord: (word: string) => boolean;

package/dist/search/russian-dictionary.js

@@ -0,0 +1,66 @@
+import { readFileSync } from "node:fs";
+let loaded;
+/**
+ * Hunspell's own check, an ending at a time, instead of nspell's: nspell expands the 146,269 entries into
+ * 1.44 million forms (1.4 s, 172 MB) where this loads in about 0.15 s and agrees on every word it knows.
+ */
+const dictionary = () => {
+    if (loaded)
+        return loaded;
+    const file = (name) => readFileSync(new URL(name, import.meta.resolve("dictionary-ru")), "utf8");
+    const stems = new Map();
+    for (const line of file("index.dic").split("\n").slice(1)) {
+        const [entry, flags = ""] = line.trim().split("/");
+        if (!entry)
+            continue;
+        // Names and abbreviations are capitalised in the dictionary; a transcript's word index is lower case.
+        const word = entry.toLowerCase();
+        stems.set(word, (stems.get(word) ?? "") + flags);
+    }
+    const suffixes = new Map();
+    let longest = 0;
+    for (const line of file("index.aff").split("\n")) {
+        const [kind, flag, strip, append, condition] = line.trim().split(/\s+/);
+        if (kind !== "SFX" || flag === undefined || strip === undefined || append === undefined || !condition)
+            continue;
+        const ending = append === "0" ? "" : append;
+        const rules = suffixes.get(ending) ?? [];
+        rules.push({
+            flag,
+            strip: strip === "0" ? "" : strip,
+            condition: new RegExp(`${condition === "." ? "" : condition}$`),
+        });
+        suffixes.set(ending, rules);
+        longest = Math.max(longest, ending.length);
+    }
+    loaded = { stems, suffixes, longest };
+    return loaded;
+};
+const spelled = (word) => {
+    const { stems, suffixes, longest } = dictionary();
+    if (stems.has(word))
+        return true;
+    for (let cut = 0; cut <= Math.min(word.length - 1, longest); cut++)
+        for (const rule of suffixes.get(word.slice(word.length - cut)) ?? []) {
+            const stem = word.slice(0, word.length - cut) + rule.strip;
+            if (stems.get(stem)?.includes(rule.flag) && rule.condition.test(stem))
+                return true;
+        }
+    return false;
+};
+/**
+ * Whether a word the index stores is a real Russian word. The index folds й into и and ё into е, so each is
+ * tried back one letter at a time.
+ */
+export const knownRussianWord = (word) => {
+    if (spelled(word))
+        return true;
+    for (const [folded, original] of [
+        ["и", "й"],
+        ["е", "ё"],
+    ])
+        for (let at = word.indexOf(folded); at >= 0; at = word.indexOf(folded, at + 1))
+            if (spelled(word.slice(0, at) + original + word.slice(at + 1)))
+                return true;
+    return false;
+};

package/dist/services/meeting-project-discovery.js

@@ -1,7 +1,7 @@
 import { createHash } from "node:crypto";
 import { CliError } from "@wirecat/cli-base";
 import { normalizeVocabularyName } from "../domain/vocabulary.js";
-import { canonicalMeetingReference, parseMeetingReference, validateMeetingHitSearch, } from "../meetings/index.js";
+import { canonicalMeetingReference, meetingWordsQuery, parseMeetingReference, validateMeetingHitSearch, } from "../meetings/index.js";
 const scopeKeys = [
     "accountId",
     "meetingId",
@@ -134,7 +134,7 @@ export const readMeetingProjectDiscovery = async (ports, projectName, original)
     const previous = cursorOf(cursor, key, terms.length);
     const termIndex = previous?.term ?? 0;
     const term = terms[termIndex];
-    const result = await ports.search.searchHits(term.name, {
+    const result = await ports.search.searchHits(meetingWordsQuery(term.name), {
         ...input,
         ...(previous?.after ? { after: previous.after } : {}),
     });

package/dist/services/moment.d.ts

@@ -12,3 +12,24 @@ export declare const momentOf: (reference: string, flag?: string, now?: number,
  * is not a time; a caller that clears a date checks for it first.
  */
 export declare const aheadOf: (reference: string, flag: string, now?: number, at?: MomentZone) => number;
+export interface DateWindow {
+    since?: number;
+    until?: number;
+    /** What the answer says about it: the zone a value without one was read in, and the bounds in UTC. */
+    meta: {
+        timezone?: string;
+        window?: {
+            since?: string;
+            until?: string;
+        };
+    };
+}
+/** `--since-time` and `--until-time` together, as the filter and the `meta` an answer reports it by. */
+export declare const dateWindow: ({ sinceTime, untilTime, zone, now, names, }: {
+    sinceTime?: string;
+    untilTime?: string;
+    zone?: string;
+    now?: number;
+    /** How the caller spells the two, for the error: `since_time` over MCP. */
+    names?: [string, string];
+}) => DateWindow;

package/dist/services/moment.js

@@ -1,4 +1,5 @@
 import { CliError } from "@wirecat/cli-base";
+import { timezoneOf } from "@wirecat/cli-base/query";
 import { calendarMoment } from "../domain/moment.js";
 const AGO = /^(\d+)(m|h|d)$/;
 const AGO_MS = { m: 60_000, h: 3_600_000, d: 86_400_000 };
@@ -43,3 +44,21 @@ export const aheadOf = (reference, flag, now = Date.now(), at = {}) => {
     }
     return time;
 };
+/** `--since-time` and `--until-time` together, as the filter and the `meta` an answer reports it by. */
+export const dateWindow = ({ sinceTime, untilTime, zone, now = Date.now(), names = ["--since-time", "--until-time"], }) => {
+    const since = sinceTime === undefined ? undefined : momentOf(sinceTime, names[0], now, { zone });
+    const until = untilTime === undefined ? undefined : momentOf(untilTime, names[1], now, { zone, end: true });
+    const readInZone = [sinceTime, untilTime].some((value) => value !== undefined && DATE.test(value.trim()) && !/(?:Z|[+-]\d{2}:?\d{2})$/i.test(value.trim()));
+    const window = {
+        ...(since === undefined ? {} : { since: new Date(since).toISOString() }),
+        ...(until === undefined ? {} : { until: new Date(until).toISOString() }),
+    };
+    return {
+        ...(since === undefined ? {} : { since }),
+        ...(until === undefined ? {} : { until }),
+        meta: {
+            ...(readInZone ? { timezone: timezoneOf(zone) } : {}),
+            ...(since === undefined && until === undefined ? {} : { window }),
+        },
+    };
+};

package/dist/services/retention.js

@@ -13,16 +13,19 @@ export const observationDuration = (text, flag) => {
 export const retentionService = (deps) => ({
     report: async (chat, query = {}) => {
         const cutoff = Date.now();
+        const zone = timezoneOf(query.timezone);
         const options = {
-            since: query.sinceTime === undefined ? cutoff - 90 * 86_400_000 : momentOf(query.sinceTime, "--since-time"),
-            until: query.untilTime === undefined ? cutoff : momentOf(query.untilTime, "--until-time"),
+            since: query.sinceTime === undefined
+                ? cutoff - 90 * 86_400_000
+                : momentOf(query.sinceTime, "--since-time", cutoff, { zone }),
+            until: query.untilTime === undefined ? cutoff : momentOf(query.untilTime, "--until-time", cutoff, { zone, end: true }),
             cutoff,
             checkpoints: (query.checkpoints ?? "1d,7d,30d")
                 .split(",")
                 .map((one) => observationDuration(one.trim(), "--checkpoints")),
             within: observationDuration(query.within ?? "7d", "--within"),
             by: query.by ?? "week",
-            timezone: timezoneOf(query.timezone),
+            timezone: zone,
             limit: query.limit ?? 20,
         };
         const store = await deps.store(), account = await deps.account();

package/dist/store/sqlite/meeting-hit-query.d.ts

@@ -0,0 +1,60 @@
+import type { QueryAst } from "@wirecat/cli-base/query";
+import type { Stemmer } from "../../search/stem.js";
+import type { CacheDatabase, SqlValue } from "../driver.js";
+interface Word {
+    raw: string;
+    literal: string;
+    /** Matched as a whole word, not as a beginning: `exact:` and number words. */
+    whole: boolean;
+    stems: string[];
+    widened: string[];
+}
+type Leaf = {
+    kind: "words";
+    exact: boolean;
+    words: Word[];
+} | {
+    kind: "phrase";
+    exact: boolean;
+    words: Word[];
+}
+/** `"a b"~N`: every word within `lines` transcript lines of a line that holds one of them. */
+ | {
+    kind: "near";
+    exact: boolean;
+    words: Word[];
+    lines: number;
+} | {
+    kind: "speaker";
+    participants: number[];
+    names: string[];
+};
+export type Resolved = Leaf | {
+    kind: "boolean";
+    must: Resolved[];
+    should: Resolved[];
+    not: Resolved[];
+};
+interface State {
+    stemmer: Stemmer | null;
+    through: number;
+}
+/** The parsed query, its names resolved and its words stemmed and widened, before any text is read. */
+export declare const resolveMeetingQuery: (database: CacheDatabase, accountId: number, ast: QueryAst, state: State) => Resolved;
+/**
+ * A `(rid, tier)` select: tier 0 the typed form, 1 another form the stems found, 2 a garbled spelling. A passage
+ * all of a query's required parts reach takes the worst tier among them; one any optional part reaches, the best.
+ */
+export declare const meetingQuerySql: (tree: Resolved, state: State) => {
+    sql: string;
+    params: SqlValue[];
+};
+/**
+ * The tree checked again against a passage's current text — an index lags an edit — and the garbled spellings
+ * that let it match. A speaker cannot be read from the text, so it passes here.
+ */
+export declare const matchMeetingText: (tree: Resolved, text: string, stemmer: Stemmer | null) => {
+    from: string;
+    to: string[];
+}[] | null;
+export {};