@wirecat/cli-core 0.390.0 → 0.392.0

package/dist/http/public-url.js

@@ -0,0 +1,32 @@
+import { CliError } from "@wirecat/cli-base";
+const EXAMPLE = "--public-url https://<name>.ts.net";
+/**
+ * The tunnel's address a public listener is reached at, checked one way for every command that opens
+ * one. `missing` says what needs it; `reaches` who calls through it, for the https refusal.
+ */
+export const publicUrlOf = (given, { missing, reaches }) => {
+    if (!given)
+        throw new CliError("configuration_error", `${missing} needs the tunnel's address: ${EXAMPLE}`);
+    let url;
+    try {
+        url = new URL(given);
+    }
+    catch {
+        throw new CliError("validation_error", `--public-url is not an address: ${EXAMPLE}`);
+    }
+    const local = url.protocol === "http:" && ["localhost", "127.0.0.1"].includes(url.hostname);
+    if (url.protocol !== "https:" && !local)
+        throw new CliError("validation_error", `--public-url must be https — ${reaches} through it`);
+    if (url.pathname !== "/" || url.search || url.hash)
+        throw new CliError("validation_error", `--public-url is the tunnel's address only, without a path: ${EXAMPLE}`);
+    return url;
+};
+/** A local port from `--port`, or the command's own default. */
+export const portOf = (given, fallback) => {
+    if (given === undefined)
+        return fallback;
+    const port = Number(given);
+    if (!Number.isInteger(port) || port < 1 || port > 65535)
+        throw new CliError("validation_error", "--port takes 1–65535");
+    return port;
+};

package/dist/mcp/http/serve.d.ts

@@ -1,5 +1,5 @@
 import { type McpServer } from "@modelcontextprotocol/server";
-import { type Listening } from "./node.js";
+import { type Listening } from "../../http/listen.js";
 export declare const MCP_PATH = "/mcp";
 export interface HttpOptions {
     /** The tunnel's address, as the browser apps reach it — `https://name.ts.net`. */

package/dist/mcp/http/serve.js

@@ -1,6 +1,6 @@
 import { randomUUID } from "node:crypto";
 import { createMcpHandler, getOAuthProtectedResourceMetadataUrl, hostHeaderValidationResponse, isLegacyRequest, oauthMetadataResponse, requireBearerAuth, WebStandardStreamableHTTPServerTransport, } from "@modelcontextprotocol/server";
-import { listen } from "./node.js";
+import { listen } from "../../http/listen.js";
 import { ownerLogin } from "./oauth.js";
 export const MCP_PATH = "/mcp";
 const LOCAL = ["localhost", "127.0.0.1"];

package/dist/meetings/index.d.ts

@@ -9,6 +9,7 @@ export { type Finding, type FindingsInput, type FindingsReceipt, type FindingsRu
 export { MEETING_HIT_QUERY, meetingWordsQuery, parseMeetingQuery } from "./hit-query.js";
 export { type ImportFile, importFiles } from "./import.js";
 export { MEETING_ISSUES, MEETING_QUERY, type MeetingIssue, type MeetingItem, type MeetingOwner, meeting, meetingIssuesOf, meetingItem, } from "./item.js";
+export { CHAT_MESSAGES_READ, type MeetingChatLine, type MeetingChatLinesInput, type MeetingChatLinesStore, } from "./links.js";
 export { createMeetingContext, MEETING_FACT_TYPES, type MeetingContextAction, type MeetingContextChange, type MeetingContextOperation, type MeetingContextPorts, type MeetingContextReceipt, validateMeetingContextChange, } from "./meeting-context.js";
 export { createMeetingManagement, dispatchMeetingMutation, type ManagedMeeting, type ManagedMeetingChange, type MeetingManagement, type MeetingManagementPorts, type MeetingMutationOptions, type MeetingMutationProposal, type MeetingMutationReceipt, type MeetingOperationOptions, type MeetingProviderContext, validateMeetingChange, } from "./meeting-management.js";
 export type { Attachment, ChatLine, Event, EventSeries, JsonValue, Meeting, MeetingSeries, Metadata, Occurrence, Participant, RecordTimes, SourceChatLine, SourceMeeting, Summary, SummaryContent, Transcript, TranscriptLine, TranscriptRow, } from "./model.js";
@@ -32,6 +33,7 @@ export { parseRecordingSyncCheckpoint, type RecordingSyncCheckpoint, type Record
 export { type RecordingPartial, RecordingTransferError, type RecordingTransferResponse, type RecordingTransferSource, type RecordingTransferStorage, type RecordingTransferWriter, strongRecordingValidator, transferRecording, } from "./recording-transfer.js";
 export { canonicalMeetingReference, formatMeetingReference, type MeetingReference, parseMeetingReference, } from "./reference.js";
 export { type MeetingRelatedHistoryEntry, type MeetingRelatedHistoryRequest, type MeetingRelatedHistoryStore, validateMeetingRelatedHistory, } from "./related-history.js";
+export { type MeetingReview, type MeetingReviewInput, type MeetingReviewItem, type MeetingReviewSpeaker, type MeetingReviewStore, REVIEW_LIMIT, } from "./review.js";
 export { SAVE_HELP, SAVE_INSTRUCTION } from "./save-hints.js";
 export type { MeetingHitSearchRequest, MeetingSearchIndexInput, MeetingSearchIndexReceipt, MeetingSearchIndexStore, MeetingSearchOptions, MeetingSearchPage, MeetingSearchStore, } from "./search-store.js";
 export { validateMeetingHitSearch } from "./search-validation.js";

package/dist/meetings/index.js

@@ -7,6 +7,7 @@ export { findingFingerprint, validateFindings, } from "./findings-rules.js";
 export { MEETING_HIT_QUERY, meetingWordsQuery, parseMeetingQuery } from "./hit-query.js";
 export { importFiles } from "./import.js";
 export { MEETING_ISSUES, MEETING_QUERY, meeting, meetingIssuesOf, meetingItem, } from "./item.js";
+export { CHAT_MESSAGES_READ, } from "./links.js";
 export { createMeetingContext, MEETING_FACT_TYPES, validateMeetingContextChange, } from "./meeting-context.js";
 export { createMeetingManagement, dispatchMeetingMutation, validateMeetingChange, } from "./meeting-management.js";
 export { normalizeMeetingParticipantQuery, validateMeetingParticipantSearch, } from "./participant-search.js";
@@ -23,6 +24,7 @@ export { parseRecordingSyncCheckpoint, RecordingSyncError, recordingAssetFingerp
 export { RecordingTransferError, strongRecordingValidator, transferRecording, } from "./recording-transfer.js";
 export { canonicalMeetingReference, formatMeetingReference, parseMeetingReference, } from "./reference.js";
 export { validateMeetingRelatedHistory, } from "./related-history.js";
+export { REVIEW_LIMIT, } from "./review.js";
 export { SAVE_HELP, SAVE_INSTRUCTION } from "./save-hints.js";
 export { validateMeetingHitSearch } from "./search-validation.js";
 export { validateMeetingSeriesHistory, } from "./series-history.js";

package/dist/meetings/links.d.ts

@@ -0,0 +1,18 @@
+import type { MeetingStatsSelection } from "./stats.js";
+export interface MeetingChatLinesInput extends MeetingStatsSelection {
+    /** Most chat messages to read (default and at most 100000); more is an error, never a silent cut. */
+    maxMessages?: number;
+}
+/** One stored chat message with the meeting it was sent in. */
+export interface MeetingChatLine {
+    meetingId: number;
+    accountId: number;
+    title: string | null;
+    sentAt: number;
+    sender: string | null;
+    text: string;
+}
+export interface MeetingChatLinesStore {
+    meetingChatLines(input: MeetingChatLinesInput): Promise<MeetingChatLine[]>;
+}
+export declare const CHAT_MESSAGES_READ = 100000;

package/dist/meetings/links.js

@@ -0,0 +1 @@
+export const CHAT_MESSAGES_READ = 100_000;

package/dist/meetings/review.d.ts

@@ -0,0 +1,45 @@
+import type { MeetingStatsSelection } from "./stats.js";
+export interface MeetingReviewInput extends MeetingStatsSelection {
+    /** Most meetings, newest first (default 50, at most 200). */
+    limit?: number;
+}
+export interface MeetingReviewSpeaker {
+    name: string | null;
+    talkMs: number;
+}
+/** One meeting as a review shows it: what the store holds about it, never transcript text. */
+export interface MeetingReviewItem {
+    meetingId: number;
+    accountId: number;
+    title: string | null;
+    startedAt: number | null;
+    durationMs: number | null;
+    participantsCount: number | null;
+    projects: string[];
+    /** `null`: no current transcript. */
+    transcript: {
+        lines: number;
+        words: number;
+    } | null;
+    /** The three who talked longest, from the transcript. */
+    speakers: MeetingReviewSpeaker[];
+    /** The newest stored summary; `null` when the meeting has none. */
+    summary: {
+        source: string;
+        title: string | null;
+        overview: string | null;
+        nextSteps: string[];
+        url: string | null;
+    } | null;
+}
+export interface MeetingReview {
+    items: MeetingReviewItem[];
+    hasMore: boolean;
+}
+export interface MeetingReviewStore {
+    meetingReview(input: MeetingReviewInput): Promise<MeetingReview>;
+}
+export declare const REVIEW_LIMIT: {
+    readonly default: 50;
+    readonly most: 200;
+};

package/dist/meetings/review.js

@@ -0,0 +1 @@
+export const REVIEW_LIMIT = { default: 50, most: 200 };

package/dist/search/lucene/registry.d.ts

@@ -70,7 +70,11 @@ export declare const QUERY_OPERATORS: readonly [{
     readonly example: "text:/pass(port)?/";
     readonly semantics: "Полный term; bounded subset Lucene RegExp; буквы text приводятся как в индексе";
 }, {
-    readonly name: "fuzzy / proximity";
+    readonly name: "proximity";
+    readonly example: '"invoice paid"~3';
+    readonly semantics: "Слова одного сообщения, между ними не больше N других слов, в любом порядке; N до 50";
+}, {
+    readonly name: "fuzzy";
     readonly example: "invoice~1";
     readonly semantics: "unsupported_operator; формы слова ловит prefix `word*`";
 }, {

package/dist/search/lucene/registry.js

@@ -1,7 +1,7 @@
 import { queryAstSchema, validateQuery, walkQuery, } from "@wirecat/cli-base/query";
 import * as v from "valibot";
 import { tagOf } from "../../domain/tags.js";
-import { QUERY_LIMITS, SEARCH_QUERY, searchError } from "./types.js";
+import { MOST_WORDS_APART, QUERY_LIMITS, SEARCH_QUERY, searchError } from "./types.js";
 export const FIELD_VERSION = 2;
 const UNITS = {
     "": 1,
@@ -398,6 +398,7 @@ export const QUERY_FIELDS = [
     },
 ];
 const MESSAGES = { entity: "message", backend: "sql", fields: QUERY_FIELDS };
+const NEAR_FIELDS = new Set(["text", "exact", "transcript", "content"]);
 /** Field, operator and value of every term, against the message fields. */
 export const validateFields = (ast) => {
     for (const node of walkQuery(ast.root)) {
@@ -406,6 +407,10 @@ export const validateFields = (ast) => {
             const plain = ["term", "phrase"].includes(node.operator) && /^[\w:.+-]+$/u.test(node.value);
             searchError("legacy_date_field", node.span, plain ? `use date${node.field === "after" ? ">=" : "<"}${node.value}` : "use date>=YYYY-MM-DD or date:[a TO b]");
         }
+        if (node.slop !== undefined && !NEAR_FIELDS.has(node.field))
+            searchError("query.unsupported-operator", node.span, `"a b"~N takes words: ${[...NEAR_FIELDS].join(", ")}`);
+        if (node.slop !== undefined && node.slop > MOST_WORDS_APART)
+            searchError("query.invalid-number", node.span, `words at most ${MOST_WORDS_APART} apart: "a b"~${MOST_WORDS_APART}`);
         // The message compilers read `field:*` as a pattern, never as "has a value".
         if (field && node.operator === "wildcard" && !field.operators.includes("wildcard"))
             searchError("query.unsupported-operator", node.span, `${field.name} takes ${field.operators.join(", ")}`);
@@ -499,9 +504,10 @@ export const QUERY_OPERATORS = [
         semantics: "Полный term; bounded subset Lucene RegExp; буквы text приводятся как в индексе",
     },
     {
-        name: "fuzzy / proximity",
-        example: "invoice~1",
-        semantics: "unsupported_operator; формы слова ловит prefix `word*`",
+        name: "proximity",
+        example: '"invoice paid"~3',
+        semantics: "Слова одного сообщения, между ними не больше N других слов, в любом порядке; N до 50",
     },
+    { name: "fuzzy", example: "invoice~1", semantics: "unsupported_operator; формы слова ловит prefix `word*`" },
     { name: "boost / minimum / intervals", example: "invoice^2", semantics: "query.unsupported-operator" },
 ];

package/dist/search/lucene/types.d.ts

@@ -13,5 +13,13 @@ export declare const QUERY_LIMITS: {
     readonly work: 10000000;
 };
 export declare const SEARCH_QUERY: QueryContext;
+/** How message search parses typed text: `"a b"~N` is words at most N apart; notes still refuse `~`. */
+export declare const MESSAGE_PARSING: {
+    command?: string;
+    guide?: string;
+    proximity: boolean;
+};
+/** Further apart than this, two words are rarely about one thing, even in a long message. */
+export declare const MOST_WORDS_APART = 50;
 export declare function searchError(reason: string, span: Span, alternative?: string): never;
 export declare function exhausted(budget: string): never;

package/dist/search/lucene/types.js

@@ -15,6 +15,10 @@ export const SEARCH_QUERY = {
     command: "search",
     guide: "https://github.com/WireCatLabs/cli-core/blob/main/docs/search/query-language.md",
 };
+/** How message search parses typed text: `"a b"~N` is words at most N apart; notes still refuse `~`. */
+export const MESSAGE_PARSING = { ...SEARCH_QUERY, proximity: true };
+/** Further apart than this, two words are rarely about one thing, even in a long message. */
+export const MOST_WORDS_APART = 50;
 export function searchError(reason, span, alternative) {
     queryError(reason, span, alternative, SEARCH_QUERY);
 }

package/dist/search/question-plan.js

@@ -1,6 +1,6 @@
 import { implicitTextTerms, parseLucene, walkQuery } from "@wirecat/cli-base/query";
 import { normalize } from "../store/normalize.js";
-import { SEARCH_QUERY } from "./lucene/types.js";
+import { MESSAGE_PARSING } from "./lucene/types.js";
 export const words = (text) => normalize(text).match(/[\p{L}\p{N}]+/gu) ?? [];
 const scaffolding = new Set(words("what which how many much who where when is are was were do does did can could should we the a an for of to under from in on at has have сколько какой какая какие какое кто кому кем где когда как что во по на в к с у через ли должен должна нужно идёт идет можно может могут"));
 const questionWord = /\b(?:what|which|how|who|where|when|can|could|should)\b|(?:^|\s)(?:сколько|какой|какая|какие|какое|кто|кому|кем|где|когда|как|что|можно|может|могут|ли)(?:\s|$)/iu;
@@ -27,7 +27,7 @@ export function planQuestion(text, strict = false) {
             removed: [],
             aliases: [],
         };
-    const inputAst = parseLucene(text, SEARCH_QUERY);
+    const inputAst = parseLucene(text, MESSAGE_PARSING);
     const rawTextual = walkQuery(inputAst.root).filter((node) => node.field === "text" || node.field === "exact");
     const body = rawTextual.map((node) => node.value).join(" ");
     const prefix = words(body).slice(0, 2);
@@ -66,7 +66,7 @@ export function planQuestion(text, strict = false) {
         if (last?.operator === "wildcard" && text[mark] === "?")
             cleaned = text.slice(0, mark) + text.slice(mark + 1);
     }
-    const ast = cleaned === text ? inputAst : parseLucene(cleaned, SEARCH_QUERY);
+    const ast = cleaned === text ? inputAst : parseLucene(cleaned, MESSAGE_PARSING);
     const predicates = walkQuery(ast.root);
     const textual = predicates.filter((p) => p.field === "text" || p.field === "exact");
     const fields = predicates.filter((p) => p.field !== "text" && p.field !== "exact");

package/dist/services/excerpt.js

@@ -3,7 +3,10 @@ import { fold } from "../store/normalize.js";
 export const queryWords = (query) => query
     .toLowerCase()
     .split(/\s+/)
-    .map((word) => word.replace(/^[\p{L}_]+:/u, "").replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ""))
+    .map((word) => word
+    .replace(/^[\p{L}_]+:/u, "")
+    .replace(/~\d+$/u, "")
+    .replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ""))
     .filter((word) => word.length > 2 && !["and", "or", "not"].includes(word));
 /**
  * About `size` characters of `text` on one line, around the first word of `query` it holds, or its start when it

package/dist/services/index.d.ts

@@ -65,8 +65,10 @@ export type { InboxService } from "./inbox.js";
 export { inboxService } from "./inbox.js";
 export { readMail } from "./mail-search.js";
 export { type MeetingHitSearchRequest, refreshMeetingIndex, searchMeetingHits } from "./meeting-hit-search.js";
+export { type MeetingLink, type MeetingLinksRequest, readMeetingLinks } from "./meeting-links.js";
 export { createMeetingProjectCandidateReader } from "./meeting-project-candidates.js";
 export { readMeetingProjectDiscovery } from "./meeting-project-discovery.js";
+export { type MeetingReviewRequest, readMeetingReview } from "./meeting-review.js";
 export { type MeetingStatsRequest, readMeetingStats } from "./meeting-stats.js";
 export { chatIdIn, storedChatId } from "./message-resolution.js";
 export type { AroundWindow, FoundMessage, ListWindow, MessagesService, MessageTarget, Pinned, Reacted, SearchFound, SearchQuery, SendRequest, Starred, } from "./messages.js";

package/dist/services/index.js

@@ -46,8 +46,10 @@ export { foldersService } from "./folders.js";
 export { inboxService } from "./inbox.js";
 export { readMail } from "./mail-search.js";
 export { refreshMeetingIndex, searchMeetingHits } from "./meeting-hit-search.js";
+export { readMeetingLinks } from "./meeting-links.js";
 export { createMeetingProjectCandidateReader } from "./meeting-project-candidates.js";
 export { readMeetingProjectDiscovery } from "./meeting-project-discovery.js";
+export { readMeetingReview } from "./meeting-review.js";
 export { readMeetingStats } from "./meeting-stats.js";
 export { chatIdIn, storedChatId } from "./message-resolution.js";
 export { DELETE_AT_ONCE, messagesService } from "./messages.js";

package/dist/services/meeting-links.d.ts

@@ -0,0 +1,33 @@
+import type { MeetingChatLinesInput, MeetingHitSearchRequest } from "../meetings/index.js";
+import type { MessageStore } from "../store/store.js";
+export type MeetingLinksRequest = Omit<MeetingHitSearchRequest, "limit" | "after" | "maxReadBytes"> & Pick<MeetingChatLinesInput, "maxMessages">;
+/** One link as the meetings' chat holds it: where and how often it was shared. */
+export interface MeetingLink {
+    url: string;
+    domain: string | null;
+    /** The Google Docs, Sheets, Slides, Forms or Drive file it names. */
+    documentId: string | null;
+    count: number;
+    firstAt: number;
+    lastAt: number;
+    senders: string[];
+    meetings: {
+        meetingId: number;
+        accountId: number;
+        title: string | null;
+    }[];
+}
+/** Links shared in the chat of the selected meetings, most shared first; found when read, not stored. */
+export declare const readMeetingLinks: (store: Pick<MessageStore, "meetings"> & {
+    knowledge?: MessageStore["knowledge"];
+}, request: MeetingLinksRequest) => Promise<{
+    items: MeetingLink[];
+    messages: number;
+    meetings: number;
+    selection: {
+        kind: import("../store/index.js").VocabularyKind;
+        id: number;
+        name: string;
+        by: "explicit-link";
+    }[];
+}>;

package/dist/services/meeting-links.js

@@ -0,0 +1,41 @@
+import { googleDocumentId } from "../domain/google-documents.js";
+import { findMentions } from "../domain/mentions.js";
+import { resolveMeetingSelection } from "./meeting-hit-search.js";
+/** Links shared in the chat of the selected meetings, most shared first; found when read, not stored. */
+export const readMeetingLinks = async (store, request) => {
+    const { maxMessages, ...selecting } = request;
+    const { input, selection } = await resolveMeetingSelection(store, selecting);
+    const lines = await store.meetings.meetingChatLines({
+        ...input,
+        ...(maxMessages === undefined ? {} : { maxMessages }),
+    });
+    const links = new Map();
+    for (const line of lines)
+        for (const found of findMentions(line.text)) {
+            if (found.kind !== "url")
+                continue;
+            const link = links.get(found.value) ?? {
+                url: found.value,
+                domain: found.domain,
+                documentId: googleDocumentId(found.value),
+                count: 0,
+                firstAt: line.sentAt,
+                lastAt: line.sentAt,
+                senders: [],
+                meetings: [],
+            };
+            link.count += 1;
+            link.lastAt = line.sentAt;
+            if (line.sender !== null && !link.senders.includes(line.sender))
+                link.senders.push(line.sender);
+            if (!link.meetings.some((one) => one.meetingId === line.meetingId))
+                link.meetings.push({ meetingId: line.meetingId, accountId: line.accountId, title: line.title });
+            links.set(found.value, link);
+        }
+    return {
+        items: [...links.values()].sort((a, b) => b.count - a.count || b.lastAt - a.lastAt),
+        messages: lines.length,
+        meetings: new Set(lines.map((line) => line.meetingId)).size,
+        selection,
+    };
+};

package/dist/services/meeting-review.d.ts

@@ -0,0 +1,12 @@
+import type { MeetingHitSearchRequest, MeetingReview } from "../meetings/index.js";
+import type { MessageStore } from "../store/store.js";
+import { resolveMeetingSelection } from "./meeting-hit-search.js";
+export type MeetingReviewRequest = Omit<MeetingHitSearchRequest, "limit" | "after" | "maxReadBytes"> & {
+    limit?: number;
+};
+/** The newest meetings of the selection with their summaries, talk and projects; names resolved as search does. */
+export declare const readMeetingReview: (store: Pick<MessageStore, "meetings"> & {
+    knowledge?: MessageStore["knowledge"];
+}, request: MeetingReviewRequest) => Promise<MeetingReview & {
+    selection: Awaited<ReturnType<typeof resolveMeetingSelection>>["selection"];
+}>;

package/dist/services/meeting-review.js

@@ -0,0 +1,8 @@
+import { resolveMeetingSelection } from "./meeting-hit-search.js";
+/** The newest meetings of the selection with their summaries, talk and projects; names resolved as search does. */
+export const readMeetingReview = async (store, request) => {
+    const { limit, ...selecting } = request;
+    const { input, selection } = await resolveMeetingSelection(store, selecting);
+    const review = await store.meetings.meetingReview({ ...input, ...(limit === undefined ? {} : { limit }) });
+    return { ...review, selection };
+};

package/dist/services/messages-search.js

@@ -14,7 +14,7 @@ import { chatAmong, senderAmong } from "./message-resolution.js";
 import { accountsOfKind, listedAccounts, MAIL } from "./search-kind.js";
 const MAIL_FIELDS = new Set(["to", "cc", "bcc", "subject", "mailbox", "label", "is"]);
 import { dateRange, parseLucene, timezoneOf, walkQuery } from "@wirecat/cli-base/query";
-import { SEARCH_QUERY, searchError } from "../search/lucene/types.js";
+import { MESSAGE_PARSING, searchError } from "../search/lucene/types.js";
 const positiveSources = (node) => node.kind === "predicate"
     ? node.field === "in"
         ? [node.value.toLowerCase()]
@@ -47,7 +47,7 @@ export const prepareLucene = async (store, account, request, messenger) => {
                     span: { start: 0, end: 0 },
                 },
             }
-            : validateFields(parseLucene(request.text, { ...SEARCH_QUERY, defaultField: request.exact ? "exact" : "text" }));
+            : validateFields(parseLucene(request.text, { ...MESSAGE_PARSING, defaultField: request.exact ? "exact" : "text" }));
     const timezone = timezoneOf(request.timezone);
     const stored = await store.accounts.accounts();
     // A fresh store has no row yet for the account it runs as; "every account" still means that one.

package/dist/services/searches.js

@@ -1,7 +1,7 @@
 import { CliError, singleLine } from "@wirecat/cli-base";
 import { parseLucene } from "@wirecat/cli-base/query";
 import { validateAst, validateFields } from "../search/lucene/registry.js";
-import { SEARCH_QUERY } from "../search/lucene/types.js";
+import { MESSAGE_PARSING } from "../search/lucene/types.js";
 import { isAdminSelection, readAdminSelection } from "./admin-statistics.js";
 import { searchRecordOf } from "./search-record.js";
 /** A saved search or a run as every searches command answers it. */
@@ -31,7 +31,7 @@ const checked = (params) => {
     if (params.ast !== undefined)
         validateAst(params.ast);
     else if (params.text !== undefined)
-        validateFields(parseLucene(params.text, { ...SEARCH_QUERY, defaultField: params.exact ? "exact" : "text" }));
+        validateFields(parseLucene(params.text, { ...MESSAGE_PARSING, defaultField: params.exact ? "exact" : "text" }));
     return params;
 };
 const named = (search, check) => {
@@ -52,7 +52,7 @@ const both = (stored, words) => ({
         kind: "boolean",
         clauses: [
             { occur: "must", node: stored.root },
-            { occur: "must", node: parseLucene(words, SEARCH_QUERY).root },
+            { occur: "must", node: parseLucene(words, MESSAGE_PARSING).root },
         ],
         span: { start: 0, end: 0 },
     },

package/dist/store/sqlite/lucene-fields.d.ts

@@ -29,9 +29,16 @@ export interface Corpus {
     item: string;
     thread: string;
 }
-/** A voice note's transcript is its attachment's text, written by speech recognition (`speech:<model>`). */
-export declare const SPEECH = "coalesce(att.extractor,'') LIKE 'speech:%'";
+/**
+ * A voice note's transcript is its attachment's text, whoever wrote it: speech recognition (`speech:<model>`) or an
+ * agent, which writes the text of a voice attachment as `agent`. The voice attachment is the one
+ * `intoVoiceAttachment` picks.
+ */
+export declare const speech: (alias: string) => string;
+export declare const SPEECH: string;
 export declare const quoted: (value: string) => string;
+/** A leaf's words as an FTS5 match: a phrase, or with `~N` its words at most N apart, in any order. */
+export declare const ftsMatch: (node: ResolvedPredicate) => string;
 export declare const combine: (parts: Fragment[], operator: "AND" | "OR") => Fragment;
 export declare const prefixOf: (pattern: string) => string;
 export declare const wordMatch: (corpus: Corpus, match: string) => Fragment;
@@ -50,7 +57,7 @@ export interface FieldScope {
         sql: string;
         params: SqlValue[];
     };
-    everywhere: (node: ResolvedPredicate, text: string, fragment: Fragment) => Fragment;
+    everywhere: (node: ResolvedPredicate, fragment: Fragment) => Fragment;
     /** Counts the patterns' expanded words against the query's budget. */
     expand: (count: number) => number;
 }

package/dist/store/sqlite/lucene-fields.js

@@ -8,12 +8,25 @@ import { parseBytes } from "../../search/lucene/registry.js";
 import { isStemmed } from "../../search/lucene/resolved.js";
 import { exhausted, QUERY_LIMITS, searchError } from "../../search/lucene/types.js";
 import { inSource } from "../../search/query.js";
+import { STEM_SEPARATOR } from "../../search/stem.js";
 import { normalize } from "../normalize.js";
 import { linkMatch } from "./mention-links.js";
 import { valueMatch } from "./phone-countries.js";
-/** A voice note's transcript is its attachment's text, written by speech recognition (`speech:<model>`). */
-export const SPEECH = "coalesce(att.extractor,'') LIKE 'speech:%'";
+/**
+ * A voice note's transcript is its attachment's text, whoever wrote it: speech recognition (`speech:<model>`) or an
+ * agent, which writes the text of a voice attachment as `agent`. The voice attachment is the one
+ * `intoVoiceAttachment` picks.
+ */
+export const speech = (alias) => `(coalesce(${alias}.extractor,'') LIKE 'speech:%' OR ${alias}.kind IN ('voice','video_note') OR coalesce(${alias}.mime,'')='audio/ogg')`;
+export const SPEECH = speech("att");
 export const quoted = (value) => `"${value.replaceAll('"', '""')}"`;
+/** A leaf's words as an FTS5 match: a phrase, or with `~N` its words at most N apart, in any order. */
+export const ftsMatch = (node) => {
+    const text = normalize(node.value);
+    const words = node.slop === undefined ? [] : (text.match(/[\p{L}\p{N}\p{Co}\p{M}]+/gu) ?? []);
+    return words.length > 1 ? nearMatch(words, node.slop) : quoted(text);
+};
+const nearMatch = (words, apart) => `NEAR(${words.map(quoted).join(" ")}, ${apart})`;
 export const combine = (parts, operator) => ({
     sql: parts.length ? `(${parts.map(({ sql }) => sql).join(` ${operator} `)})` : operator === "AND" ? "1" : "0",
     params: parts.flatMap(({ params }) => params),
@@ -59,20 +72,79 @@ const mentioned = ({ mail }, where, params) => mail
     : bound(`m.id IN (SELECT mn.source_id FROM mentions mn WHERE mn.source_type='message' AND ${where} ` +
         "UNION SELECT a.attachable_id FROM mentions mn JOIN attachments a ON a.id=mn.source_id " +
         `AND a.attachable_type='message' WHERE mn.source_type='attachment' AND ${where})`, ...params, ...params);
-const stemsOf = ({ execution }, text) => execution.stemmer?.phrases(text);
+const stemsOf = ({ execution: { stemmer } }, node) => {
+    if (node.slop === undefined)
+        return stemmer?.phrases(node.value);
+    const sequences = (stemmer?.stemSequences(node.value) ?? []).filter((sequence) => sequence.length);
+    return sequences.length
+        ? sequences
+            .map((sequence) => (sequence.length > 1 ? nearMatch(sequence, node.slop) : quoted(sequence.join(""))))
+            .join(" OR ")
+        : undefined;
+};
+/** Whether some stem sequence of a text holds every stem of `wanted` with at most `apart` other words between. */
+const near = (sequence, wanted, apart) => {
+    const events = sequence.flatMap((stem, index) => (wanted.includes(stem) ? [{ index, stem }] : []));
+    const held = new Map();
+    const distinct = new Set(wanted).size;
+    let left = 0;
+    for (const event of events) {
+        held.set(event.stem, (held.get(event.stem) ?? 0) + 1);
+        while (held.size === distinct) {
+            const first = events[left];
+            if (event.index - first.index + 1 - distinct <= apart)
+                return true;
+            const count = (held.get(first.stem) ?? 1) - 1;
+            if (count)
+                held.set(first.stem, count);
+            else
+                held.delete(first.stem);
+            left++;
+        }
+    }
+    return false;
+};
+/**
+ * `NEAR` over the stems can bridge `STEM_SEPARATOR`, which parts a text's Latin readings: the end of one and the
+ * start of the next are one word apart. A row without the separator is the index's answer; one with it is read.
+ */
+const nearStems = (scope, node, stems) => {
+    const { database, corpus, execution } = scope;
+    const stemmer = execution.stemmer;
+    const rows = database
+        .prepare(`SELECT m.id AS id, m.text AS text FROM ${corpus.rows} m WHERE m.id IN (SELECT rowid FROM ${corpus.stems} WHERE ${corpus.stems} MATCH ?) LIMIT ?`)
+        .all(`stems : ((${stems}) AND ${quoted(STEM_SEPARATOR)})`, QUERY_LIMITS.candidates + 1);
+    if (rows.length > QUERY_LIMITS.candidates)
+        exhausted("candidate rows");
+    const wanted = stemmer.stemSequences(node.value).filter((sequence) => sequence.length > 1);
+    const kept = rows.flatMap(({ id, text }) => {
+        scope.check();
+        const sequences = stemmer.stemSequences(String(text));
+        return wanted.some((query) => sequences.some((sequence) => near(sequence, query, node.slop)))
+            ? [Number(id)]
+            : [];
+    });
+    return bound(`m.id IN (SELECT rowid FROM ${corpus.stems} WHERE ${corpus.stems} MATCH ?) OR m.id IN (SELECT value FROM json_each(?))`, `stems : ((${stems}) NOT ${quoted(STEM_SEPARATOR)})`, JSON.stringify(kept));
+};
 const stemmedText = (node, scope) => {
     const { execution, corpus } = scope;
     if (!execution.stemmer && !execution.unstemmed)
         throw new Error("a stemmed leaf reached the compiler without the store's stemmer");
     const text = normalize(node.value);
-    const stems = stemsOf(scope, node.value);
+    const match = ftsMatch(node);
+    const stems = stemsOf(scope, node);
+    const byStems = stems && node.slop !== undefined
+        ? nearStems(scope, node, stems)
+        : stems
+            ? bound(`m.id IN (SELECT rowid FROM ${corpus.stems} WHERE ${corpus.stems} MATCH ?)`, `stems : (${stems})`)
+            : undefined;
     // Words OR stems: two spellings can fold one word to different stems, and every exact hit must stay.
     return always(/[\p{L}\p{N}]/u.test(text)
-        ? scope.everywhere(node, text, {
-            sql: `m.id IN (SELECT rowid FROM ${corpus.words} WHERE ${corpus.words} MATCH ?)${stems ? ` OR m.id IN (SELECT rowid FROM ${corpus.stems} WHERE ${corpus.stems} MATCH ?)` : ""}`,
-            params: [`normalized_text : (${quoted(text)})`, ...(stems ? [`stems : (${stems})`] : [])],
+        ? scope.everywhere(node, {
+            sql: `m.id IN (SELECT rowid FROM ${corpus.words} WHERE ${corpus.words} MATCH ?)${byStems ? ` OR ${byStems.sql}` : ""}`,
+            params: [`normalized_text : (${match})`, ...(byStems?.params ?? [])],
             exact: true,
-            fts: quoted(text),
+            fts: match,
             ...(stems ? { stems } : {}),
         })
         : bound("0"));
@@ -86,9 +158,9 @@ const words = (node, scope) => {
         return stemmedText(node, scope);
     if (operator === "term" || operator === "phrase") {
         const text = normalize(value);
-        const stems = stemsOf(scope, value);
+        const stems = stemsOf(scope, node);
         return always(/[\p{L}\p{N}]/u.test(text)
-            ? scope.everywhere(node, text, { ...wordMatch(corpus, quoted(text)), ...(stems ? { stems } : {}) })
+            ? scope.everywhere(node, { ...wordMatch(corpus, ftsMatch(node)), ...(stems ? { stems } : {}) })
             : bound("0"));
     }
     const pattern = operator === "wildcard" ? wildcardPattern(normalize(value)) : foldRegex(value, node.span);
@@ -181,10 +253,10 @@ const size = (node, { corpus }) => {
     const range = combine(conditions, "AND");
     return always(bound(`EXISTS (SELECT 1 FROM attachments att WHERE att.attachable_type='${corpus.item}' AND att.attachable_id=m.id AND att.size IS NOT NULL AND ${range.sql})`, ...range.params));
 };
-const attachmentText = (which) => ({ value }, scope) => {
-    const text = normalize(value);
+const attachmentText = (which) => (node, scope) => {
+    const text = normalize(node.value);
     // Never `fts`: the ranking ANDs every required word into message_words, where a file's words are not.
-    const set = scope.attachmentWords(quoted(text), which);
+    const set = scope.attachmentWords(ftsMatch(node), which);
     return always(/[\p{L}\p{N}]/u.test(text) ? bound(`m.id IN (${set.sql})`, ...set.params) : bound("0"));
 };
 const date = (node) => {