@wirecat/cli-core 0.500.0 → 0.501.0

package/dist/services/notes-search.js

@@ -1,4 +1,4 @@
-import { CliError } from "@wirecat/cli-base";
+import { CliError, isCliError } from "@wirecat/cli-base";
 import { dateRange, parseLucene, timezoneOf } from "@wirecat/cli-base/query";
 import { INTERNAL_NOTES } from "../domain/providers.js";
 import { normalizeTag } from "../domain/tags.js";
@@ -10,6 +10,7 @@ import { validateFields } from "../search/lucene/registry.js";
 import { hasStems } from "../search/lucene/resolved.js";
 import { SEARCH_QUERY } from "../search/lucene/types.js";
 import { createStemmer, DEFAULT_STEMMERS } from "../search/stem.js";
+import { TOTAL_CAP, totalOf } from "../search/total.js";
 import { openPictureEmbedderFor } from "./document-pictures.js";
 import { documentSource } from "./document-source.js";
 import { plainQueryWords, titleQuery, titleTiers } from "./notes-title-tier.js";
@@ -122,7 +123,17 @@ export const embedEvents = (store, options = {}) => embedChunks(store, store.cal
 /** Embeds mail's chunks as notes' are embedded: same model and key, so search by meaning reads both. */
 export const embedMail = (store, options = {}) => embedChunks(store, store.mail, options);
 /** Notes nearest in meaning to the query, best first; only notes already embedded can be found. */
-export const nearestNotes = async (store, query, { model: choice, limit, folderIds, withInternal, source, env, threads, command, embedders, }) => {
+export const nearestNotes = async (store, query, options) => {
+    const { key, vector, floor } = await queryVector(store, query, options);
+    const nearest = await store.notes.nearest(key, vector, {
+        limit: options.limit,
+        ...(options.folderIds ? { folderIds: options.folderIds } : {}),
+        ...(options.withInternal ? { withInternal: true } : {}),
+        ...(options.source ? { source: options.source } : {}),
+    });
+    return nearest.filter(({ score }) => score > floor);
+};
+const queryVector = async (store, query, { model: choice, folderIds, withInternal, source, env, threads, command, embedders }) => {
     if (!query.trim())
         throw new CliError("validation_error", "say what to find in the notes");
     const { model, directory, key } = localModel(choice, env, command);
@@ -134,14 +145,12 @@ export const nearestNotes = async (store, query, { model: choice, limit, folderI
     if (!(await store.notes.embedded(key, scope)))
         throw new CliError("not_found", `nothing searched here has vectors of ${model.id} yet, so only words were matched`);
     const [vector] = await withLocal(model, directory, threads, (embedder) => embedder.embed([query], "query"), embedders);
-    const floor = meaningFloor(key);
-    const nearest = await store.notes.nearest(key, vector, {
-        limit,
-        ...(folderIds ? { folderIds } : {}),
-        ...(withInternal ? { withInternal: true } : {}),
-        ...(source ? { source } : {}),
-    });
-    return nearest.filter(({ score }) => score > floor);
+    return { key, vector: vector, floor: meaningFloor(key) };
+};
+/** Every note of `refs` nearer in meaning than the model's floor, best first. */
+const nearestAmong = async (store, query, refs, options) => {
+    const { key, vector, floor } = await queryVector(store, query, options);
+    return (await store.notes.scoreNotes(key, vector, refs)).filter(({ score }) => score > floor);
 };
 const MAX_LINE = 200;
 const MAX_NOTES_OFFSET = 1000;
@@ -289,6 +298,62 @@ const matching = async (store, refs, root, scope) => {
     });
     return new Map(found.items.map(({ ref, note, at }) => [ref, { note, at }]));
 };
+/**
+ * The most notes a filter may hold for meaning to rank all of them, each by its best chunk. Above it, meaning ranks
+ * the nearest notes overall and keeps those inside the filter, so a narrow filter over a large store loses some. On
+ * the owner's store (1,454 documents, 23,308 chunks) scoring all of them took ~0.5 s against ~0.17 s for the
+ * nearest 50 overall, and the cost grows with chunks, so 2,000 notes keeps it under a second at that density.
+ */
+export const RANK_WITHIN = 2000;
+/** The notes of a filter, newest first, when there are at most `RANK_WITHIN`; `undefined` when more, or too costly. */
+const filterRefs = async (store, narrowing, scope) => {
+    const resolved = await resolvedQuery(store, narrowing, scope.timezone);
+    try {
+        const found = await store.notes.matchingRefs({
+            root: resolved.root,
+            limit: RANK_WITHIN,
+            ...(scope.folderIds ? { folderIds: scope.folderIds } : {}),
+            ...(scope.withInternal ? { withInternal: true } : {}),
+            ...(scope.source ? { source: scope.source } : {}),
+            ...(resolved.stemmer ? { stemmer: resolved.stemmer } : {}),
+        });
+        return found.hasMore ? undefined : found.refs;
+    }
+    catch (error) {
+        if (tooLarge(error))
+            return undefined;
+        throw error;
+    }
+};
+const tooLarge = (error) => isCliError(error) && error.details.reason === "query.too-large";
+/**
+ * The notes the words find, with those found by meaning: every note the search would list, paged to the end. Never
+ * below what the page already showed; a count the store refuses as too costly is that lower bound.
+ */
+const notesTotal = async (store, text, { exact, scope, shown, hasMore, }, others, othersComplete) => {
+    let words;
+    try {
+        const { root, stemmer } = await storeQuery(store, text, {
+            exact,
+            ...(scope.timezone ? { timezone: scope.timezone } : {}),
+        });
+        words = await store.notes.matchingRefs({
+            root,
+            limit: TOTAL_CAP,
+            ...(scope.folderIds ? { folderIds: scope.folderIds } : {}),
+            ...(scope.withInternal ? { withInternal: true } : {}),
+            ...(scope.source ? { source: scope.source } : {}),
+            ...(stemmer ? { stemmer } : {}),
+        });
+    }
+    catch (error) {
+        if (tooLarge(error))
+            return totalOf(undefined, shown, hasMore);
+        throw error;
+    }
+    const count = new Set([...words.refs, ...others]).size + (words.hasMore ? 1 : 0);
+    return totalOf(count, shown, hasMore, othersComplete);
+};
 /**
  * `search notes`: words and stems, and meaning when the local model is there, merged by reciprocal rank;
  * each hit says which half found it, and the answer lists what the notes found link to. Every field of the query and
@@ -317,24 +382,39 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
     };
     let meaningSkipped;
     let meaning;
+    /** Every note found by meaning; `meaningComplete` is false when the nearest overall were cut at the depth. */
+    let meaningRefs;
+    let meaningComplete = true;
     if (exact)
         meaningSkipped = "--exact searches words as written";
     else if (words) {
+        const lane = {
+            command,
+            ...(env === undefined ? {} : { env }),
+            ...(embedders === undefined ? {} : { embedders }),
+            ...(folderIds === undefined ? {} : { folderIds }),
+            ...(withInternal ? { withInternal: true } : {}),
+            ...(source === undefined ? {} : { source }),
+        };
+        const depth = Math.max(window, CANDIDATES);
         try {
-            const nearest = await nearestNotes(store, words, {
-                limit: Math.max(window, CANDIDATES),
-                command,
-                ...(env === undefined ? {} : { env }),
-                ...(embedders === undefined ? {} : { embedders }),
-                ...(folderIds === undefined ? {} : { folderIds }),
-                ...(withInternal ? { withInternal: true } : {}),
-                ...(source === undefined ? {} : { source }),
-            });
+            const within = narrowing === undefined ? undefined : await filterRefs(store, narrowing, scope);
+            let nearest;
+            if (within === undefined) {
+                nearest = await nearestNotes(store, words, { ...lane, limit: depth });
+                meaningComplete = nearest.length < depth;
+            }
+            else {
+                const ranked = within.length ? await nearestAmong(store, words, within, lane) : [];
+                meaningRefs = new Set(ranked.map(({ ref }) => ref));
+                nearest = ranked.slice(0, depth);
+            }
             const allowed = await matching(store, nearest.map(({ ref }) => ref), narrowing, scope);
             meaning = nearest.flatMap(({ ref }) => {
                 const found = allowed.get(ref);
                 return found ? [{ ref, ...found }] : [];
             });
+            meaningRefs ??= new Set(meaning.map(({ ref }) => ref));
         }
         catch (error) {
             if (!(error instanceof CliError && error.code === "not_found"))
@@ -378,6 +458,7 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
     ];
     const page = slice ? ordered : ordered.slice(offset, window);
     const hasMore = slice ? matched.hasMore : ordered.length > window || matched.hasMore || titled.hasMore;
+    const total = await notesTotal(store, text, { exact, scope, shown: offset + page.length, hasMore }, [...tier.map(({ ref }) => ref), ...(meaningRefs ?? [])], meaningComplete);
     const hits = [];
     const counts = new Map();
     const accounts = await store.accounts.storedAccounts();
@@ -441,6 +522,7 @@ export const searchNotes = async (store, query, { limit = 20, offset = 0, tag, f
         by: meaning === undefined ? "words" : "words and meaning",
         ...(meaningSkipped === undefined ? {} : { meaning_skipped: meaningSkipped }),
         hits,
+        total,
         has_more: hasMore,
         ...(hasMore && window <= MAX_NOTES_OFFSET ? { next_offset: window } : {}),
         linked: [...counts.values()].sort((a, b) => b.notes - a.notes || (a.name ?? a.ref ?? "").localeCompare(b.name ?? b.ref ?? "")),

package/dist/store/queries.d.ts

@@ -13,6 +13,8 @@ export interface QueriesStore {
     matchQuery(execution: QueryExecution): Promise<Page<ScoredHit>>;
     /** The same matches as `matchQuery`, each counted once, grouped by chat, sender or quarter hour. */
     countQuery(execution: QueryExecution, by: QueryGrouping): Promise<QueryGroup[]>;
+    /** How many messages match, counted no further than `cap + 1`; `undefined` when only a full search could tell. */
+    countMatches(execution: QueryExecution, cap: number): Promise<number | undefined>;
     conversationEligibility(execution: QueryExecution): Promise<ConversationEligibility>;
     rankQuery(execution: QueryExecution, request: RankingRequest): Promise<RankedStoreFound>;
     rankingEvidence(execution: QueryExecution, request: RankingEvidenceRequest): Promise<RankedEvidence>;

package/dist/store/queries.js

@@ -13,6 +13,7 @@ export const queriesStoreOver = (context, stemmerFor) => {
     return {
         matchQuery: async (execution) => lucene.matchQuery(context, mailIndexed(execution)),
         countQuery: async (execution, by) => lucene.countQuery(context, mailIndexed(execution), by),
+        countMatches: async (execution, cap) => lucene.countMatches(context, mailIndexed(execution), cap),
         conversationEligibility: async (execution) => conversationEligibility(context, execution),
         rankQuery: async (execution, request) => rankQuery(context, execution, request),
         rankingEvidence: async (execution, request) => rankingEvidence(context, execution, request),

package/dist/store/sqlite/lucene.d.ts

@@ -18,6 +18,11 @@ export declare const matchQuery: (context: StoreContext, execution: QueryExecuti
 export declare const countQuery: (context: StoreContext, execution: QueryExecution, by: QueryGrouping) => Promise<QueryGroup[]>;
 /** All matching message keys, without materializing message bodies or imposing a hit-page limit. */
 export declare const queryMessagePks: (context: StoreContext, execution: QueryExecution) => Promise<number[]>;
+/**
+ * How many messages match, counting no further than `cap + 1`: one SQL count, unranked. `undefined` when a leaf is
+ * checked row by row (a preset, a pattern), where a count would cost as much as the search itself.
+ */
+export declare const countMatches: (context: StoreContext, execution: QueryExecution, cap: number) => number | undefined;
 interface QuerySelection {
     sql: string;
     params: SqlValue[];

package/dist/store/sqlite/lucene.js

@@ -48,6 +48,23 @@ export const matchQuery = (context, execution) => runQuery(context, execution);
 export const countQuery = (context, execution, by) => runQuery(context, execution, by);
 /** All matching message keys, without materializing message bodies or imposing a hit-page limit. */
 export const queryMessagePks = (context, execution) => runQuery(context, execution, "pks");
+/**
+ * How many messages match, counting no further than `cap + 1`: one SQL count, unranked. `undefined` when a leaf is
+ * checked row by row (a preset, a pattern), where a count would cost as much as the search itself.
+ */
+export const countMatches = (context, execution, cap) => {
+    const { database, check, where, from, withRanking, ranked, leaves } = compileQuery(context, {
+        ...execution,
+        newest: true,
+    });
+    if (leaves.some(({ test }) => test))
+        return undefined;
+    const row = database
+        .prepare(`SELECT count(*) AS n FROM (${withRanking}SELECT m.id ${from} WHERE ${where.sql} LIMIT ?)`)
+        .get(...ranked.params, ...where.params, cap + 1);
+    check();
+    return Number(row?.n ?? 0);
+};
 const compileQuery = (context, execution, boundedAttachments = false) => {
     const { database } = context;
     const corpus = execution.corpus === "mail" ? MAIL : MESSAGES;

package/dist/store/sqlite/note-search.d.ts

@@ -79,6 +79,11 @@ export declare const searchNotes: (context: StoreContext, query: NoteQuery) => {
     items: NoteHit[];
     hasMore: boolean;
 };
+/** The refs of the newest `limit` notes matching a query, without reading the notes themselves. */
+export declare const matchingNoteRefs: (context: StoreContext, query: Omit<NoteQuery, "offset" | "newest">) => {
+    refs: string[];
+    hasMore: boolean;
+};
 export declare const searchCorpus: (context: StoreContext, query: CorpusQuery, corpus: Corpus) => CorpusRow[];
 export interface NoteChunkToEmbed {
     hash: string;
@@ -101,6 +106,11 @@ export interface NoteSearch extends DocumentPictureStore {
         items: NoteHit[];
         hasMore: boolean;
     }>;
+    /** The refs of the newest `limit` notes matching the query; `hasMore` when more match. */
+    matchingRefs(query: Omit<NoteQuery, "offset" | "newest">): Promise<{
+        refs: string[];
+        hasMore: boolean;
+    }>;
     indexState(): Promise<NoteIndexState | undefined>;
     /** Live notes' chunks with no vector of `model`, by hash from `after`, each with its text cut again. */
     chunksToEmbed(model: string, options: {
@@ -120,6 +130,14 @@ export interface NoteSearch extends DocumentPictureStore {
         withInternal?: boolean;
         source?: Note["source"];
     }): Promise<NearestNote[]>;
+    /**
+     * Each of `refs` scored by its best chunk, best first, every chunk compared: a filter's own notes ranked, not the
+     * nearest overall. A note with no vector of `model` is left out; the notes themselves are not read.
+     */
+    scoreNotes(model: string, query: Float32Array, refs: string[]): Promise<{
+        ref: string;
+        score: number;
+    }[]>;
 }
 export declare const noteSearchOver: (context: StoreContext) => NoteSearch;
 export {};

package/dist/store/sqlite/note-search.js

@@ -63,9 +63,7 @@ const CORPORA = [
         tagged: (tag) => bound("n.id IN (SELECT g.taggable_id FROM taggings g JOIN resolved_tags t ON t.id=g.tag_id WHERE t.name = ? AND g.taggable_type = 'note')", tag),
     },
 ];
-/** Notes matching a parsed query, by the same language as messages: files and written notes, merged. */
-export const searchNotes = (context, query) => {
-    const offset = query.offset ?? 0;
+const matchedRows = (context, query) => {
     const chosen = CORPORA.filter(({ type }) => (query.source === undefined || (query.source === "file") === (type === "document")) &&
         (!query.folderIds?.length || query.withInternal === true || type === "document"));
     const idsOf = (type) => query.refs?.flatMap((ref) => (ref.startsWith(`${type}:`) ? [Number(ref.slice(type.length + 1))] : []));
@@ -83,6 +81,12 @@ export const searchNotes = (context, query) => {
             (a.relevance ?? 0) - (b.relevance ?? 0) ||
             (b.at ?? 0) - (a.at ?? 0) ||
             a.id - b.id);
+    return rows;
+};
+/** Notes matching a parsed query, by the same language as messages: files and written notes, merged. */
+export const searchNotes = (context, query) => {
+    const offset = query.offset ?? 0;
+    const rows = matchedRows(context, query);
     const read = {
         document: context.database.prepare("SELECT * FROM documents WHERE id = ?"),
         note: context.database.prepare("SELECT * FROM notes WHERE id = ?"),
@@ -102,6 +106,11 @@ export const searchNotes = (context, query) => {
         hasMore: rows.length > offset + query.limit,
     };
 };
+/** The refs of the newest `limit` notes matching a query, without reading the notes themselves. */
+export const matchingNoteRefs = (context, query) => {
+    const rows = matchedRows(context, { ...query, newest: true });
+    return { refs: rows.slice(0, query.limit).map(({ type, id }) => `${type}:${id}`), hasMore: rows.length > query.limit };
+};
 export const searchCorpus = (context, query, corpus) => {
     const { database } = context;
     const wordMatch = (match) => ({
@@ -282,7 +291,7 @@ export const searchCorpus = (context, query, corpus) => {
             : "";
         const window = tests.size ? QUERY_LIMITS.candidates + 1 : query.limit + offset + 1;
         const rows = database
-            .prepare(`${ctes.length ? `WITH ${ctes.join(", ")} ` : ""}SELECT n.*, ${ranking ? "f.rank" : "NULL"} AS relevance${exactTier ? ", n.id IN (SELECT id FROM x) AS exact" : ""}${projection}
+            .prepare(`${ctes.length ? `WITH ${ctes.join(", ")} ` : ""}SELECT ${tests.size ? "n.*" : "n.id, n.at"}, ${ranking ? "f.rank" : "NULL"} AS relevance${exactTier ? ", n.id IN (SELECT id FROM x) AS exact" : ""}${projection}
            FROM ${corpus.view} n${ranking ? " LEFT JOIN f ON f.id = n.id" : ""} WHERE ${where.sql} ORDER BY ${order} LIMIT ?`)
             .all(...cteParams, ...leaves.flatMap(([, fragment]) => (tests.size ? fragment.params : [])), ...where.params, window);
         if (tests.size && rows.length > QUERY_LIMITS.candidates)
@@ -346,6 +355,10 @@ export const noteSearchOver = (context) => {
             drain();
             return searchNotes(context, query);
         },
+        matchingRefs: async (query) => {
+            drain();
+            return matchingNoteRefs(context, query);
+        },
         indexState: async () => noteIndexState(database),
         chunksToEmbed: async (model, { after, limit }) => {
             drain();
@@ -456,5 +469,28 @@ export const noteSearchOver = (context) => {
                 return note ? [{ ref: note.ref, note, score, range: { start, end } }] : [];
             });
         },
+        scoreNotes: async (model, query, refs) => {
+            drain();
+            const scope = live(undefined, undefined, true);
+            const owned = database.prepare(`SELECT k.chunkable_type AS type, k.chunkable_id AS owner, v.vector
+           FROM chunks k JOIN embeddings v ON v.model = ? AND v.content_hash = k.content_hash
+          WHERE k.chunkable_type = ? AND k.chunkable_id IN (SELECT value FROM json_each(?))${scope.sql}`);
+            const best = new Map();
+            for (const type of ["document", "note"]) {
+                const ids = refs.flatMap((ref) => (ref.startsWith(`${type}:`) ? [Number(ref.slice(type.length + 1))] : []));
+                if (!ids.length)
+                    continue;
+                for (const row of owned.all(model, type, JSON.stringify(ids), ...scope.params)) {
+                    const key = `${row.type}:${row.owner}`;
+                    const score = dot(query, row.vector);
+                    if (score > (best.get(key) ?? Number.NEGATIVE_INFINITY))
+                        best.set(key, score);
+                }
+            }
+            const order = new Map(refs.map((ref, index) => [ref, index]));
+            return [...best]
+                .map(([ref, score]) => ({ ref, score }))
+                .sort((a, b) => b.score - a.score || (order.get(a.ref) ?? 0) - (order.get(b.ref) ?? 0));
+        },
     };
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.500.0",
+  "version": "0.501.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",
@@ -247,7 +247,7 @@
     "@types/node": "^22.20.5",
     "@types/pngjs": "6.0.5",
     "@vitest/coverage-v8": "^5.0.5",
-    "@wirecat/cli-base": "0.65.5",
+    "@wirecat/cli-base": "0.67.0",
     "@wirecat/cli-core-legacy-tags": "npm:@wirecat/cli-core@0.414.0",
     "commander": "15.0.0",
     "cspell": "10.4.0",
@@ -267,7 +267,7 @@
   },
   "peerDependencies": {
     "@napi-rs/canvas": ">=0.1.69 <2",
-    "@wirecat/cli-base": ">=0.65.0 <0.66.0",
+    "@wirecat/cli-base": ">=0.67.0 <0.68.0",
     "commander": ">=15"
   },
   "bin": {