@wirecat/cli-core 0.495.0 → 0.497.0

package/dist/services/document-source.js

@@ -1,8 +1,8 @@
 import { isFolderProvider, kindIn } from "../domain/providers.js";
 /**
- * Where a hit came from. The document's preferred file says it: its account's provider, its link and its edit time.
+ * Where a hit came from. The document's preferred file says it: its account's provider and its link.
  * A Drive document under a notes folder (before version 31 moves it) is still known by its front matter.
- * Indexing time belongs to the local row.
+ * The hit's time is the store search's: the one `date:` reads.
  */
 export const documentSource = (note, accounts, profiles, file) => {
     const metadata = note.frontMatter;
@@ -16,15 +16,10 @@ export const documentSource = (note, accounts, profiles, file) => {
             (isFolderProvider(key.provider) && key.account === account.name))
         : undefined;
     const url = file?.url ?? (remote && typeof metadata?.sourceUrl === "string" ? metadata.sourceUrl : undefined);
-    const modified = file?.externalUpdatedAt ??
-        (remote && typeof metadata?.modifiedAt === "string" && Number.isFinite(Date.parse(metadata.modifiedAt))
-            ? metadata.modifiedAt
-            : undefined);
     return {
         kind: remote ? "document" : "note",
         provider: profile ? profile.command : "notes",
         account: profile?.name ?? note.folderId ?? "internal",
         ...(remote && url !== undefined ? { url } : {}),
-        updated_at: remote && modified !== undefined ? modified : note.updatedAt,
     };
 };

package/dist/services/messages-discovery.js

@@ -31,10 +31,15 @@ const FOLLOW_UP_CANDIDATES = 20;
 export const searchDiscovery = async (store, account, asked, messenger = {}, meaning, rerank) => {
     // A question's own "?" would parse as a one-character wildcard and send it to strict search (BUG-395); the
     // colon of a time or ratio ("22:00", "1:1") would parse as a field and fail the whole search, and escaped it is
-    // syntax again, so it splits the numbers (BUG-450).
+    // syntax again, so it splits the numbers (BUG-450). A quoted timestamp keeps its colons (BUG-458).
     const request = asked.ast !== undefined || asked.exact || !asked.text
         ? asked
-        : { ...asked, text: asked.text.replace(/(?<=\p{L})\?(?=\s|$)/gu, "").replace(/(?<=\d):(?=\d)/gu, " ") };
+        : {
+            ...asked,
+            text: asked.text
+                .replace(/(?<=\p{L})\?(?=\s|$)/gu, "")
+                .replace(/"(?:[^"\\]|\\.)*"|(?<=\d):(?=\d)/gu, (found) => (found === ":" ? " " : found)),
+        };
     const original = await prepareLucene(store, account, request, messenger);
     const text = request.text ?? "";
     const plan = planQuestion(text, request.ast !== undefined || !!request.exact || !!request.newest);

package/dist/services/notes-search.d.ts

@@ -1,3 +1,4 @@
+import { type QueryNode } from "@wirecat/cli-base/query";
 import { type Embedder, type WarmEmbedders } from "../embeddings/embed.js";
 import { type TextModel } from "../embeddings/models.js";
 import type { PictureEmbedder } from "../images/picture/embedder.js";
@@ -90,7 +91,7 @@ export interface FoundNote {
     /** The note's path inside its folder; `null` for a note written in memo. */
     path: string | null;
     title: string | null;
-    /** When the store's copy last changed. */
+    /** The time `date:` reads: a Drive document's edit time when the store has one, else when the store's copy changed. */
     updated_at: string;
     /** The first line holding a word of the query. */
     excerpt: string;
@@ -135,7 +136,7 @@ export interface NotesQuery {
     limit?: number;
     offset?: number;
     tag?: string;
-    /** A query in the same language that every hit must also match; it does not change the meaning half. */
+    /** A query in the same language that every hit must also match, as the query's own fields are. */
     filter?: string;
     folderIds?: string[];
     /** With `folderIds`, the notes written in memo too: one ranked list, so `offset` pages it. */
@@ -164,9 +165,18 @@ export declare const noteSourceOfQuery: (query: string) => {
     source?: Note["source"];
     rest: string;
 };
+/**
+ * What a note found by meaning must still match: the query with each word it asks for taken as matched, since the
+ * meaning half stands in for those. Words it excludes, and every other field, stay. `undefined` when nothing but
+ * wanted words is left, so any note found by meaning fits.
+ */
+export declare const fieldsOfQuery: (root: QueryNode) => QueryNode | undefined;
+/** The words and phrases the query asks for, for the meaning half to embed; none it excludes, no field. */
+export declare const meaningWords: (root: QueryNode) => string;
 /**
  * `search notes`: words and stems, and meaning when the local model is there, merged by reciprocal rank;
- * each hit says which half found it, and the answer lists what the notes found link to.
+ * each hit says which half found it, and the answer lists what the notes found link to. Every field of the query and
+ * of `filter` holds for every hit, found by words or by meaning; a query of fields alone is not searched by meaning.
  */
 export declare const searchNotes: (store: MessageStore, query: string, { limit, offset, tag, filter: expression, folderIds, withInternal, source: given, exact, timezone, env, command, embedders, pictures: withPictures, pictureLimit, pictureEmbedder, }?: NotesQuery) => Promise<NotesFound>;
 /** What `notes:internal` and `notes:<folder id>` ask of the store: one query, so a page of it is one page. */

package/dist/services/notes-search.js

@@ -19,8 +19,10 @@ import { listContextOf } from "./search-kind.js";
  * Text in the query language messages use, parsed for a store search: `date:` read in `timezone`, and the store's
  * stemmer when a word is stemmed.
  */
-export const storeQuery = async (store, text, { exact, timezone } = {}) => {
-    const ast = validateFields(parseLucene(text, { ...SEARCH_QUERY, defaultField: exact ? "exact" : "text" }));
+export const storeQuery = async (store, text, { exact, timezone } = {}) => resolvedQuery(store, parsedQuery(text, exact), timezone);
+const parsedQuery = (text, exact) => validateFields(parseLucene(text, { ...SEARCH_QUERY, defaultField: exact ? "exact" : "text" })).root;
+const resolvedQuery = async (store, root, timezone) => {
+    const ast = { root };
     const zone = timezoneOf(timezone);
     const resolve = (node) => {
         if (node.kind === "boolean")
@@ -145,8 +147,6 @@ const MAX_LINE = 200;
 const MAX_NOTES_OFFSET = 1000;
 /** How deep each half is read before the two are merged. */
 const CANDIDATES = 50;
-/** What a filter may narrow the meaning half to; past this the filter is applied to words only. */
-const FILTERED = 500;
 /** Reciprocal rank fusion's usual constant, as conversations use. */
 export const RRF_K = 60;
 const firstLine = (text, query) => {
@@ -225,17 +225,88 @@ export const noteSourceOfQuery = (query) => {
     }
     return { source, rest };
 };
+const WORD_FIELDS = new Set(["text", "exact"]);
+/**
+ * What a note found by meaning must still match: the query with each word it asks for taken as matched, since the
+ * meaning half stands in for those. Words it excludes, and every other field, stay. `undefined` when nothing but
+ * wanted words is left, so any note found by meaning fits.
+ */
+export const fieldsOfQuery = (root) => {
+    let narrows = false;
+    const kept = (node) => {
+        if (node.kind === "boolean")
+            return {
+                ...node,
+                clauses: node.clauses.map(({ occur, node }) => {
+                    if (occur !== "must_not")
+                        return { occur, node: kept(node) };
+                    narrows = true;
+                    return { occur, node };
+                }),
+            };
+        if (!WORD_FIELDS.has(node.field)) {
+            narrows = true;
+            return node;
+        }
+        // `in:notes` holds for every note and document; an empty clause list would match nothing.
+        return { kind: "predicate", field: "in", operator: "term", value: "notes", span: node.span };
+    };
+    const fields = kept(root);
+    return narrows ? fields : undefined;
+};
+/** The words and phrases the query asks for, for the meaning half to embed; none it excludes, no field. */
+export const meaningWords = (root) => {
+    const wanted = (node) => node.kind === "boolean"
+        ? node.clauses.flatMap(({ occur, node }) => (occur === "must_not" ? [] : wanted(node)))
+        : WORD_FIELDS.has(node.field) && (node.operator === "term" || node.operator === "phrase")
+            ? [node.value]
+            : [];
+    return wanted(root).join(" ").trim();
+};
+const allOf = (nodes) => nodes.length <= 1
+    ? nodes[0]
+    : {
+        kind: "boolean",
+        clauses: nodes.map((node) => ({ occur: "must", node })),
+        span: { start: 0, end: 0 },
+    };
+/**
+ * The notes of `refs` that match `root`, in the order of `refs`, each with the time `date:` read. One store query over
+ * just those notes, so a filter that matches thousands of notes still checks every candidate.
+ */
+const matching = async (store, refs, root, scope) => {
+    if (refs.length === 0)
+        return new Map();
+    const resolved = await resolvedQuery(store, root ?? { kind: "predicate", field: "in", operator: "term", value: "notes", span: { start: 0, end: 0 } }, scope.timezone);
+    const found = await store.notes.search({
+        root: resolved.root,
+        refs,
+        limit: refs.length,
+        ...(scope.folderIds ? { folderIds: scope.folderIds } : {}),
+        ...(scope.withInternal ? { withInternal: true } : {}),
+        ...(scope.source ? { source: scope.source } : {}),
+        ...(resolved.stemmer ? { stemmer: resolved.stemmer } : {}),
+    });
+    return new Map(found.items.map(({ ref, note, at }) => [ref, { note, at }]));
+};
 /**
  * `search notes`: words and stems, and meaning when the local model is there, merged by reciprocal rank;
- * each hit says which half found it, and the answer lists what the notes found link to.
+ * each hit says which half found it, and the answer lists what the notes found link to. Every field of the query and
+ * of `filter` holds for every hit, found by words or by meaning; a query of fields alone is not searched by meaning.
  */
 export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, filter: expression, folderIds, withInternal, source: given, exact = false, timezone, env, command, embedders, pictures: withPictures = false, pictureLimit = 5, pictureEmbedder, } = {}) => {
     const typed = noteSourceOfQuery(query);
     const source = given ?? typed.source;
-    const words = typed.source !== undefined && typed.source === source ? typed.rest : query;
+    const rest = typed.source !== undefined && typed.source === source ? typed.rest : query;
     const label = tag === undefined ? undefined : normalizeTag(tag);
     const filter = [expression ? `(${expression})` : "", label === undefined ? "" : `tag:${label}`].filter(Boolean).join(" AND ") ||
         undefined;
+    const parsed = query.trim() ? parsedQuery(query, exact) : undefined;
+    const words = parsed === undefined ? "" : meaningWords(parsed);
+    const narrowing = allOf([
+        ...(parsed === undefined ? [] : [fieldsOfQuery(parsed)].filter((node) => node !== undefined)),
+        ...(filter === undefined ? [] : [parsedQuery(filter)]),
+    ]);
     const text = [query.trim() ? `(${query})` : "", filter ?? ""].filter(Boolean).join(" AND ");
     const window = offset + limit;
     const scope = {
@@ -248,7 +319,7 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
     let meaning;
     if (exact)
         meaningSkipped = "--exact searches words as written";
-    else if (words.trim()) {
+    else if (words) {
         try {
             const nearest = await nearestNotes(store, words, {
                 limit: Math.max(window, CANDIDATES),
@@ -259,13 +330,11 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
                 ...(withInternal ? { withInternal: true } : {}),
                 ...(source === undefined ? {} : { source }),
             });
-            if (filter === undefined)
-                meaning = nearest;
-            else {
-                const allowed = await searchNotesQuery(store, { text: filter, limit: FILTERED, ...scope });
-                const refs = new Set(allowed.items.map(({ ref }) => ref));
-                meaning = nearest.filter(({ ref }) => refs.has(ref));
-            }
+            const allowed = await matching(store, nearest.map(({ ref }) => ref), narrowing, scope);
+            meaning = nearest.flatMap(({ ref }) => {
+                const found = allowed.get(ref);
+                return found ? [{ ref, ...found }] : [];
+            });
         }
         catch (error) {
             if (!(error instanceof CliError && error.code === "not_found"))
@@ -273,7 +342,7 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
             meaningSkipped = error.message;
         }
     }
-    const titleWords = plainQueryWords(words);
+    const titleWords = plainQueryWords(rest);
     const titled = titleWords === undefined
         ? { items: [], hasMore: false }
         : await searchNotesQuery(store, {
@@ -292,10 +361,10 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
         ...scope,
     });
     const ranked = new Map();
-    const add = (list, by) => list.forEach(({ ref, note }, rank) => {
+    const add = (list, by) => list.forEach(({ ref, note, at }, rank) => {
         if (tierRefs.has(ref))
             return;
-        const held = ranked.get(ref) ?? { note, score: 0, foundBy: [] };
+        const held = ranked.get(ref) ?? { note, at, score: 0, foundBy: [] };
         held.score += 1 / (RRF_K + rank + 1);
         held.foundBy.push(by);
         ranked.set(ref, held);
@@ -304,7 +373,7 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
     if (meaning !== undefined)
         add(meaning, "meaning");
     const ordered = [
-        ...tier.map(({ ref, note }) => [ref, { note, score: 0, foundBy: ["title"] }]),
+        ...tier.map(({ ref, note, at }) => [ref, { note, at, score: 0, foundBy: ["title"] }]),
         ...[...ranked.entries()].sort(([, a], [, b]) => b.score - a.score),
     ];
     const page = slice ? ordered : ordered.slice(offset, window);
@@ -313,7 +382,7 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
     const counts = new Map();
     const accounts = await store.accounts.storedAccounts();
     const profiles = await store.accounts.profiles();
-    for (const [ref, { note, foundBy }] of page) {
+    for (const [ref, { note, at, foundBy }] of page) {
         const links = (await store.notes.links({ from: ref })).map(({ to, targetText, kind, anchor }) => ({
             to,
             target_text: targetText,
@@ -327,6 +396,7 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
             path: note.path,
             title: note.title,
             ...documentSource(note, accounts, profiles, note.source === "file" ? (await store.files.ofDocument(note.ref))[0] : undefined),
+            updated_at: at,
             excerpt: firstLine(note.text, words),
             links,
             found_by: foundBy,
@@ -360,6 +430,10 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
             embedder: pictureEmbedder,
         })
         : {};
+    if (lane.pictures && narrowing !== undefined) {
+        const allowed = await matching(store, lane.pictures.map(({ ref }) => ref), narrowing, scope);
+        lane.pictures = lane.pictures.filter(({ ref }) => allowed.has(ref));
+    }
     return {
         query,
         ...(label === undefined ? {} : { tag: label }),

package/dist/store/sqlite/note-search.d.ts

@@ -13,6 +13,8 @@ export interface NoteQuery {
     /** With `folderIds`, the notes written in memo too, in the same ranked list. */
     withInternal?: boolean;
     source?: Note["source"];
+    /** Only these notes (`document:<id>`, `note:<id>`): checks a list found another way against the query. */
+    refs?: string[];
     /** Newest first instead of best first. */
     newest?: boolean;
     /** The store's stemmer, set when a leaf is stemmed. */
@@ -26,6 +28,8 @@ export interface NoteHit {
     relevance: number | null;
     /** Whether the note holds the exact forms, when the search was stemmed. */
     exact?: boolean;
+    /** The time `date:` read: a Drive document's edit time when the store has one, else when the store's copy changed. */
+    at: string;
 }
 /** The fields a note has. A message-only field (`from:`, `chat:`, …) is refused rather than matching nothing. */
 export declare const NOTE_FIELDS: readonly ["text", "exact", "body", "tag", "date", "in", "has", "title", "site", "lang", "author", "keyword", "source"];
@@ -68,6 +72,7 @@ export interface CorpusRow {
 /** What one corpus is searched by; `accountIds` narrows a corpus that has accounts. */
 export type CorpusQuery = Pick<NoteQuery, "root" | "limit" | "offset" | "newest" | "stemmer" | "signal"> & {
     accountIds?: string[];
+    ids?: number[];
 };
 /** Notes matching a parsed query, by the same language as messages: files and written notes, merged. */
 export declare const searchNotes: (context: StoreContext, query: NoteQuery) => {

package/dist/store/sqlite/note-search.js

@@ -4,6 +4,7 @@ import { compileAutomaton, foldRegex, wildcardPattern, } from "../../search/luce
 import { isStemmed } from "../../search/lucene/resolved.js";
 import { exhausted, QUERY_LIMITS, searchError } from "../../search/lucene/types.js";
 import { normalize } from "../normalize.js";
+import { contextDocumentTime } from "./context-document-source.js";
 import { documentPicturesOver } from "./document-pictures.js";
 import { linkKind } from "./link-kinds.js";
 import { prefixOf } from "./lucene.js";
@@ -42,7 +43,7 @@ const CORPORA = [
         type: "document",
         noun: "note",
         fields: NOTE_FIELDS,
-        view: "(SELECT id, account_id AS folder_id, external_id AS path, title, coalesce(body, '') AS text, metadata, updated_at AS at, deleted_at FROM documents)",
+        view: `(SELECT n.id, n.account_id AS folder_id, n.external_id AS path, n.title, coalesce(n.body, '') AS text, n.metadata, ${contextDocumentTime} AS at, n.deleted_at FROM documents n)`,
         words: "document_words",
         stems: "document_stems",
         live: "coalesce(json_extract(n.metadata, '$.catalog.active'), 1)=1",
@@ -67,7 +68,13 @@ export const searchNotes = (context, query) => {
     const offset = query.offset ?? 0;
     const chosen = CORPORA.filter(({ type }) => (query.source === undefined || (query.source === "file") === (type === "document")) &&
         (!query.folderIds?.length || query.withInternal === true || type === "document"));
-    const rows = chosen.flatMap((corpus) => searchCorpus(context, { ...query, ...(query.folderIds ? { accountIds: query.folderIds } : {}) }, corpus));
+    const idsOf = (type) => query.refs?.flatMap((ref) => (ref.startsWith(`${type}:`) ? [Number(ref.slice(type.length + 1))] : []));
+    const rows = chosen.flatMap((corpus) => {
+        const ids = idsOf(corpus.type);
+        if (ids?.length === 0)
+            return [];
+        return searchCorpus(context, { ...query, ...(query.folderIds ? { accountIds: query.folderIds } : {}), ...(ids ? { ids } : {}) }, corpus);
+    });
     const exactFirst = rows.some((row) => row.exact !== undefined);
     rows.sort((a, b) => query.newest
         ? (b.at ?? 0) - (a.at ?? 0) || a.id - b.id
@@ -89,6 +96,7 @@ export const searchNotes = (context, query) => {
                 note,
                 relevance: row.relevance,
                 ...(row.exact === undefined ? {} : { exact: row.exact }),
+                at: new Date(row.at ?? 0).toISOString(),
             };
         }),
         hasMore: rows.length > offset + query.limit,
@@ -247,6 +255,7 @@ export const searchCorpus = (context, query, corpus) => {
             bound("n.deleted_at IS NULL"),
             ...(corpus.live ? [bound(corpus.live)] : []),
             ...(query.accountIds?.length && corpus.inAccounts ? [corpus.inAccounts(JSON.stringify(query.accountIds))] : []),
+            ...(query.ids ? [bound("n.id IN (SELECT value FROM json_each(?))", JSON.stringify(query.ids))] : []),
         ], "AND");
         const where = combine([scope, expression], "AND");
         const ranking = query.newest ? undefined : required(query.root, stemmed ? "stems" : "fts");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.495.0",
+  "version": "0.497.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",

package/parity.json

@@ -387,7 +387,7 @@
     "--filter": {
       "value": "query",
       "meaning": "Strict Lucene filter: any message in the conversation must match; the meaning query stays unchanged",
-      "note": "search notes uses it for a query every note must also match, the meaning half unchanged"
+      "note": "search notes uses it for a query every note must also match, found by words or by meaning"
     },
     "--find-by-phone": {
       "value": "<who>",