@wirecat/cli-core 0.431.0 → 0.433.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/dist/cli/messenger/conversation-answers.d.ts +1 -0
  3. package/dist/cli/messenger/conversations-command.js +11 -3
  4. package/dist/cli/messenger/message-answers.d.ts +1 -0
  5. package/dist/cli/messenger/message-writes.d.ts +2 -0
  6. package/dist/cli/messenger/messages-search-command.js +11 -1
  7. package/dist/cli/messenger/search-answers.js +13 -0
  8. package/dist/cli/steps.js +6 -0
  9. package/dist/cli/upgrade-command.js +1 -1
  10. package/dist/entities/conversation-preparation.d.ts +10 -0
  11. package/dist/entities/conversation-preparation.js +2 -0
  12. package/dist/entities/conversation-results.d.ts +10 -0
  13. package/dist/entities/evidence.d.ts +8 -0
  14. package/dist/entities/index.d.ts +45 -0
  15. package/dist/entities/listener.d.ts +4 -0
  16. package/dist/entities/message-context.d.ts +3 -0
  17. package/dist/entities/message-transfer.d.ts +8 -0
  18. package/dist/entities/message-write.d.ts +4 -0
  19. package/dist/entities/message.d.ts +2 -0
  20. package/dist/entities/message.js +2 -0
  21. package/dist/entities/store-records.d.ts +5 -0
  22. package/dist/models/anthropic.js +2 -4
  23. package/dist/services/embeddings-maintenance.js +71 -2
  24. package/dist/services/embeddings-search.js +3 -1
  25. package/dist/services/embeddings.d.ts +7 -0
  26. package/dist/services/messages-archive-search.d.ts +2 -1
  27. package/dist/services/messages-archive-search.js +4 -4
  28. package/dist/services/messages-discovery.d.ts +24 -1
  29. package/dist/services/messages-discovery.js +61 -4
  30. package/dist/services/messages-meaning.d.ts +7 -0
  31. package/dist/services/messages-meaning.js +53 -0
  32. package/dist/services/messages-search.d.ts +5 -0
  33. package/dist/services/messages.d.ts +5 -0
  34. package/dist/services/messages.js +2 -1
  35. package/dist/services/search-catchup.js +1 -0
  36. package/dist/store/drivers/node-sqlite.js +1 -0
  37. package/dist/store/sqlite/drizzle/bun.js +58 -14
  38. package/dist/store/sqlite/drizzle/bun.js.map +3 -3
  39. package/dist/store/sqlite/drizzle/core.js +173 -7
  40. package/dist/store/sqlite/drizzle/core.js.map +4 -4
  41. package/dist/store/sqlite/drizzle/node.js +57 -13
  42. package/dist/store/sqlite/drizzle/node.js.map +3 -3
  43. package/dist/store/sqlite/drizzle/shared-55XJMTPA.js +6514 -0
  44. package/dist/store/sqlite/drizzle/shared-55XJMTPA.js.map +7 -0
  45. package/dist/store/sqlite/message-chunks.d.ts +44 -0
  46. package/dist/store/sqlite/message-chunks.js +76 -0
  47. package/dist/store/sqlite/open.js +1 -0
  48. package/dist/store/sqlite/schema/canonical-tags.d.ts +1 -0
  49. package/dist/store/sqlite/schema/document-index-pending.d.ts +1 -0
  50. package/dist/store/sqlite/schema/document-transfers.d.ts +1 -0
  51. package/dist/store/sqlite/schema/tag-consolidations.d.ts +2 -0
  52. package/dist/store/sqlite/schema.d.ts +97 -0
  53. package/dist/store/sqlite/vectors.js +4 -1
  54. package/dist/store/vectors.d.ts +36 -0
  55. package/dist/store/vectors.js +54 -0
  56. package/package.json +23 -24
  57. package/dist/store/sqlite/drizzle/shared-HKXF6HKX.js +0 -4846
  58. package/dist/store/sqlite/drizzle/shared-HKXF6HKX.js.map +0 -7
@@ -36,6 +36,7 @@ export declare const messageThread: v.StrictObjectSchema<{
36
36
  readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
37
37
  readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
38
38
  readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
39
+ readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
39
40
  }, undefined>, undefined>;
40
41
  readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
41
42
  readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -246,6 +247,7 @@ export declare const messageContext: v.UnionSchema<[v.StrictObjectSchema<{
246
247
  readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
247
248
  readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
248
249
  readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
250
+ readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
249
251
  }, undefined>, undefined>;
250
252
  readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
251
253
  readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -437,6 +439,7 @@ export declare const messageContext: v.UnionSchema<[v.StrictObjectSchema<{
437
439
  readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
438
440
  readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
439
441
  readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
442
+ readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
440
443
  }, undefined>, undefined>;
441
444
  readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
442
445
  readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -184,6 +184,10 @@ export declare const messageSyncChat: v.StrictObjectSchema<{
184
184
  readonly chat: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 1, undefined>]>;
185
185
  readonly needs: v.PicklistSchema<["build", "embed"], undefined>;
186
186
  }, undefined>, undefined>;
187
+ readonly messages: v.OptionalSchema<v.StrictObjectSchema<{
188
+ readonly embedded: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
189
+ readonly left: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
190
+ }, undefined>, undefined>;
187
191
  }, undefined>, undefined>;
188
192
  }, undefined>, undefined>;
189
193
  readonly window_complete: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
@@ -304,6 +308,10 @@ export declare const messageSync: v.UnionSchema<[v.StrictObjectSchema<{
304
308
  readonly chat: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 1, undefined>]>;
305
309
  readonly needs: v.PicklistSchema<["build", "embed"], undefined>;
306
310
  }, undefined>, undefined>;
311
+ readonly messages: v.OptionalSchema<v.StrictObjectSchema<{
312
+ readonly embedded: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
313
+ readonly left: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
314
+ }, undefined>, undefined>;
307
315
  }, undefined>, undefined>;
308
316
  }, undefined>, undefined>;
309
317
  readonly window_complete: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
@@ -32,6 +32,7 @@ export declare const messageWrite: v.StrictObjectSchema<{
32
32
  readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
33
33
  readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
34
34
  readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
35
+ readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
35
36
  }, undefined>, undefined>;
36
37
  readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
37
38
  readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -219,6 +220,7 @@ export declare const messageWrite: v.StrictObjectSchema<{
219
220
  readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
220
221
  readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
221
222
  readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
223
+ readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
222
224
  }, undefined>, undefined>;
223
225
  readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
224
226
  readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -414,6 +416,7 @@ export declare const messageSend: v.UnionSchema<[v.StrictObjectSchema<{
414
416
  readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
415
417
  readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
416
418
  readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
419
+ readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
417
420
  }, undefined>, undefined>;
418
421
  readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
419
422
  readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -601,6 +604,7 @@ export declare const messageSend: v.UnionSchema<[v.StrictObjectSchema<{
601
604
  readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
602
605
  readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
603
606
  readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
607
+ readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
604
608
  }, undefined>, undefined>;
605
609
  readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
606
610
  readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -42,6 +42,8 @@ export declare const message: v.StrictObjectSchema<{
42
42
  readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
43
43
  readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
44
44
  readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
45
+ /** Present when meaning was searched too: which of the two lists found it. */
46
+ readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
45
47
  }, undefined>, undefined>;
46
48
  /** The messages around a search hit, or its thread, oldest first; the hit itself is among them. */
47
49
  readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
@@ -51,6 +51,8 @@ export const message = v.strictObject({
51
51
  matched_terms: v.array(text),
52
52
  missing_terms: v.array(text),
53
53
  parent: v.optional(reference),
54
+ /** Present when meaning was searched too: which of the two lists found it. */
55
+ found_by: v.optional(v.array(v.picklist(["words", "meaning"]))),
54
56
  })),
55
57
  /** The messages around a search hit, or its thread, oldest first; the hit itself is among them. */
56
58
  context: v.optional(v.array(v.strictObject({ ref: reference, timestamp: time, sender_name: v.nullable(text), text }))),
@@ -55,6 +55,7 @@ export declare const storeExport: v.UnionSchema<[v.StrictObjectSchema<{
55
55
  readonly matched_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
56
56
  readonly missing_terms: v.ArraySchema<v.StringSchema<undefined>, undefined>;
57
57
  readonly parent: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>, undefined>;
58
+ readonly found_by: v.OptionalSchema<v.ArraySchema<v.PicklistSchema<["words", "meaning"], undefined>, undefined>, undefined>;
58
59
  }, undefined>, undefined>;
59
60
  readonly context: v.OptionalSchema<v.ArraySchema<v.StrictObjectSchema<{
60
61
  readonly ref: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, "a reference such as `note:42`">]>;
@@ -352,6 +353,10 @@ export declare const storeGapRepair: v.UnionSchema<[v.StrictObjectSchema<{
352
353
  readonly chat: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.MinLengthAction<string, 1, undefined>]>;
353
354
  readonly needs: v.PicklistSchema<["build", "embed"], undefined>;
354
355
  }, undefined>, undefined>;
356
+ readonly messages: v.OptionalSchema<v.StrictObjectSchema<{
357
+ readonly embedded: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
358
+ readonly left: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
359
+ }, undefined>, undefined>;
355
360
  }, undefined>, undefined>;
356
361
  }, undefined>, undefined>;
357
362
  readonly stopped: v.OptionalSchema<v.PicklistSchema<["bound", "partial_or_inaccessible", "fetch_failed"], undefined>, undefined>;
@@ -1,11 +1,9 @@
1
1
  import Anthropic from "@anthropic-ai/sdk";
2
2
  import * as v from "valibot";
3
- import { checkedOptions, range } from "./options.js";
3
+ import { checkedOptions } from "./options.js";
4
4
  import { messagesFor, systemFor } from "./types.js";
5
+ // API version 2026-10-21 refuses temperature, top_p and top_k on every model.
5
6
  const options = v.strictObject({
6
- temperature: range(0, 1),
7
- top_p: range(0, 1),
8
- top_k: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1))),
9
7
  stop_sequences: v.optional(v.array(v.string())),
10
8
  });
11
9
  export const anthropic = {
@@ -11,6 +11,8 @@ import { conversationsService } from "./conversations.js";
11
11
  import { storedChatId } from "./message-resolution.js";
12
12
  /** How much one run without `--chat` does at most, by default. */
13
13
  export const REFRESH_BOUNDS = { maxChats: 20, maxChunks: 2_000 };
14
+ /** Messages read per page while looking for those without a current vector. */
15
+ const MESSAGE_PAGE = 5_000;
14
16
  /** Chunks per batch for one local session; one short transaction writes each batch. */
15
17
  const PER_SESSION = 8;
16
18
  /** Texts per remote request; `concurrency` of them run at once. */
@@ -169,6 +171,58 @@ export const embeddingsMaintenance = (deps) => {
169
171
  })), described.map(({ attachmentPk }) => attachmentPk));
170
172
  return missing.length;
171
173
  };
174
+ /**
175
+ * The messages of the account, or of one chat, whose own vector is missing or older than their text: at most
176
+ * `budget` of them in hand, and how many more there are. Gone messages lose their chunks first.
177
+ */
178
+ const messagesToEmbed = async (chatId, model, budget, check) => {
179
+ const store = await deps.store();
180
+ const account = await deps.account();
181
+ await store.vectors.dropGoneMessages(account);
182
+ const items = [];
183
+ let beyond = 0;
184
+ let after = 0;
185
+ for (;;) {
186
+ check?.();
187
+ const page = await store.vectors.messagesToEmbed(account, {
188
+ ...(chatId === undefined ? {} : { chatId }),
189
+ model,
190
+ after,
191
+ limit: MESSAGE_PAGE,
192
+ });
193
+ const last = page.at(-1);
194
+ if (!last)
195
+ break;
196
+ after = last.messagePk;
197
+ for (const { messagePk, text, sentAt, hash: held, embedded } of page) {
198
+ const hash = chunkHash(text);
199
+ if (hash === held && embedded)
200
+ continue;
201
+ if (items.length < budget)
202
+ items.push({ messagePk, text, hash, occurredAt: sentAt });
203
+ else
204
+ beyond++;
205
+ }
206
+ }
207
+ return { items, beyond };
208
+ };
209
+ const embedMessages = async (items, target, embedder, check) => {
210
+ const store = await deps.store();
211
+ const account = await deps.account();
212
+ for (let at = 0; at < items.length; at += target.perBatch) {
213
+ check?.();
214
+ const batch = items.slice(at, at + target.perBatch);
215
+ const vectors = await embedder.embed(batch.map(({ text }) => text), "passage");
216
+ await store.vectors.saveMessageVectors(account, target.key, target.dims, batch.map(({ messagePk, hash, text, occurredAt }, index) => ({
217
+ messagePk,
218
+ hash,
219
+ length: text.length,
220
+ occurredAt,
221
+ vector: vectors[index],
222
+ })));
223
+ }
224
+ return items.length;
225
+ };
172
226
  return {
173
227
  status: async (chat, choice) => {
174
228
  const { store, account, chatId } = await found(chat);
@@ -204,7 +258,7 @@ export const embeddingsMaintenance = (deps) => {
204
258
  return { model: target.id, chats };
205
259
  return { model: target.id, chats, unbuiltGroups: (await store.vectors.unbuiltGroups(account)).length };
206
260
  },
207
- refresh: async ({ chat, model: choice = DEFAULT_TEXT_MODEL, build = true, embed = true, maxChats = REFRESH_BOUNDS.maxChats, maxChunks = REFRESH_BOUNDS.maxChunks, threads, progress, maxMessages, check, }) => {
261
+ refresh: async ({ chat, model: choice = DEFAULT_TEXT_MODEL, build = true, embed = true, maxChats = REFRESH_BOUNDS.maxChats, maxChunks = REFRESH_BOUNDS.maxChunks, threads, progress, maxMessages, check, messages = true, }) => {
208
262
  check?.();
209
263
  const store = await deps.store();
210
264
  const account = await deps.account();
@@ -236,13 +290,16 @@ export const embeddingsMaintenance = (deps) => {
236
290
  }
237
291
  }
238
292
  const embedded = [];
293
+ const wantsMessages = embed && messages && target.installed && choice === DEFAULT_TEXT_MODEL;
294
+ const toEmbed = wantsMessages ? await messagesToEmbed(chatId, target.key, maxChunks, check) : undefined;
295
+ let messagesEmbedded = 0;
239
296
  if (embed && target.installed) {
240
297
  const fresh = new Set(built.map(({ chat }) => chat));
241
298
  const queue = (await scan()).chats
242
299
  .filter(needsEmbed)
243
300
  .sort((a, b) => Number(fresh.has(b.chat)) - Number(fresh.has(a.chat)))
244
301
  .slice(0, maxChats);
245
- if (queue.length > 0) {
302
+ if (queue.length > 0 || toEmbed?.items.length) {
246
303
  await withEmbedder(deps, target, choice, { threads }, async (embedder) => {
247
304
  let budget = maxChunks;
248
305
  for (const { chat } of queue) {
@@ -254,6 +311,10 @@ export const embeddingsMaintenance = (deps) => {
254
311
  embedded.push(done);
255
312
  progress?.(`chat ${chat}: ${done.embedded} chunks embedded with ${target.id}`);
256
313
  }
314
+ if (toEmbed?.items.length && budget > 0) {
315
+ messagesEmbedded = await embedMessages(toEmbed.items.slice(0, budget), target, embedder, check);
316
+ progress?.(`${messagesEmbedded} messages embedded with ${target.id}`);
317
+ }
257
318
  });
258
319
  }
259
320
  }
@@ -273,6 +334,14 @@ export const embeddingsMaintenance = (deps) => {
273
334
  .map(({ chat }) => ({ chat, needs: "embed" }))
274
335
  : []),
275
336
  ],
337
+ ...(toEmbed
338
+ ? {
339
+ messages: {
340
+ embedded: messagesEmbedded,
341
+ left: toEmbed.items.length + toEmbed.beyond - messagesEmbedded,
342
+ },
343
+ }
344
+ : {}),
276
345
  };
277
346
  },
278
347
  clear: async (chat, choice) => {
@@ -122,7 +122,9 @@ export const embeddingsSearch = (deps) => {
122
122
  source,
123
123
  timezone,
124
124
  });
125
- const locallyRefreshed = refresh ? await embeddingsMaintenance(deps).refresh(refresh) : undefined;
125
+ const locallyRefreshed = refresh
126
+ ? await embeddingsMaintenance(deps).refresh({ ...refresh, messages: false })
127
+ : undefined;
126
128
  const store = await deps.store();
127
129
  const account = await deps.account();
128
130
  const target = embeddingTarget(deps, choice);
@@ -105,6 +105,8 @@ export interface RefreshOptions {
105
105
  maxChunks?: number;
106
106
  threads?: number;
107
107
  progress?: (note: string) => void;
108
+ /** With the default local model, also each message's own vector, which `--discover` reads; `false` skips it. */
109
+ messages?: boolean;
108
110
  }
109
111
  export interface Refreshed {
110
112
  model: string;
@@ -117,6 +119,11 @@ export interface Refreshed {
117
119
  chat: Id;
118
120
  needs: "build" | "embed";
119
121
  }[];
122
+ /** Messages given their own vector by this run, and how many still lack one; only when they were looked at. */
123
+ messages?: {
124
+ embedded: number;
125
+ left: number;
126
+ };
120
127
  }
121
128
  export interface FoundConversations {
122
129
  accounts?: AccountKey[];
@@ -1,6 +1,7 @@
1
1
  import type { AccountKey, MessageStore, StoredHit } from "../store/store.js";
2
2
  import type { Saved } from "./message-resolution.js";
3
3
  import type { SearchFound, SearchQuery } from "./messages.js";
4
+ import { type MeaningLane } from "./messages-discovery.js";
4
5
  import { type MessageStats, type StatsGrouping } from "./messages-search.js";
5
6
  /** How long a search may spend building the word index, then the stems, first (phase 2 plan S3: about 200 ms). */
6
7
  export declare const SEARCH_FILL_MS = 200;
@@ -18,6 +19,6 @@ export declare const MAX_SEARCH_OFFSET = 1000;
18
19
  * Search the store, one page at a time: `offset` skips the hits a page before held, and `nextOffset` says where
19
20
  * the next page starts. A page is the hits up to `offset + limit` read again and cut, so deep pages cost more.
20
21
  */
21
- export declare const searchStore: (store: MessageStore, account: AccountKey, request: SearchQuery, messenger?: Saved) => Promise<SearchFound>;
22
+ export declare const searchStore: (store: MessageStore, account: AccountKey, request: SearchQuery, messenger?: Saved, meaning?: MeaningLane) => Promise<SearchFound>;
22
23
  export declare const accountOf: (hit: StoredHit) => AccountKey;
23
24
  export declare const keyOf: ({ provider, account }: AccountKey) => string;
@@ -43,18 +43,18 @@ export const MAX_SEARCH_OFFSET = 1000;
43
43
  * Search the store, one page at a time: `offset` skips the hits a page before held, and `nextOffset` says where
44
44
  * the next page starts. A page is the hits up to `offset + limit` read again and cut, so deep pages cost more.
45
45
  */
46
- export const searchStore = async (store, account, request, messenger = {}) => {
46
+ export const searchStore = async (store, account, request, messenger = {}, meaning) => {
47
47
  const offset = request.offset ?? 0;
48
48
  if (!Number.isInteger(offset) || offset < 0 || offset > MAX_SEARCH_OFFSET)
49
49
  throw new CliError("validation_error", `offset takes a whole number from 0 to ${MAX_SEARCH_OFFSET}`);
50
- const found = await searchPage(store, account, { ...request, limit: request.limit + offset }, messenger);
50
+ const found = await searchPage(store, account, { ...request, limit: request.limit + offset }, messenger, meaning);
51
51
  return {
52
52
  ...found,
53
53
  items: found.items.slice(offset),
54
54
  ...(found.hasMore && offset + request.limit <= MAX_SEARCH_OFFSET ? { nextOffset: offset + request.limit } : {}),
55
55
  };
56
56
  };
57
- const searchPage = async (store, account, request, messenger) => {
57
+ const searchPage = async (store, account, request, messenger, meaning) => {
58
58
  if (request.signal?.aborted)
59
59
  throw new CliError("validation_error", "search was aborted", { reason: "query.aborted", complete: false });
60
60
  if (request.thread)
@@ -65,7 +65,7 @@ const searchPage = async (store, account, request, messenger) => {
65
65
  throw new CliError("validation_error", "discovery searches the local archive; use backend archive");
66
66
  await topUp(store);
67
67
  const found = request.discover
68
- ? await searchDiscovery(store, account, { ...request, thread: undefined }, messenger)
68
+ ? await searchDiscovery(store, account, { ...request, thread: undefined }, messenger, meaning)
69
69
  : await searchLucene(store, account, request, messenger);
70
70
  return withThreads(store, found, request);
71
71
  };
@@ -1,4 +1,27 @@
1
+ import type { Id } from "../domain/models.js";
1
2
  import type { MessengerTraits } from "../ports/traits.js";
2
3
  import type { AccountKey, MessageStore } from "../store/store.js";
3
4
  import type { SearchFound, SearchQuery } from "./messages.js";
4
- export declare const searchDiscovery: (store: MessageStore, account: AccountKey, asked: SearchQuery, messenger?: Partial<Pick<MessengerTraits, "savedChatId" | "app" | "history" | "provider">>) => Promise<SearchFound>;
5
+ /** How many of the messages nearest in meaning join the fusion: the benchmark's pool (performance/search/evidence-pipeline). */
6
+ export declare const MEANING_DEPTH = 100;
7
+ export type MeaningRanked = (AccountKey & {
8
+ chatId: Id;
9
+ id: Id;
10
+ score: number;
11
+ })[];
12
+ /** Why meaning was not searched, and the step that would make it searchable. */
13
+ export interface MeaningSkipped {
14
+ reason: string;
15
+ /** Absent when the model itself failed: nothing the owner runs would fix it. */
16
+ step?: "models.download" | "conversations.rebuild";
17
+ model: string;
18
+ }
19
+ /** The messages of these accounts nearest in meaning to the question, best first; or why there are none. */
20
+ export type MeaningLane = (question: string, accounts: AccountKey[]) => Promise<{
21
+ model: string;
22
+ ranked: MeaningRanked;
23
+ partly?: MeaningSkipped;
24
+ } | {
25
+ skipped: MeaningSkipped;
26
+ }>;
27
+ export declare const searchDiscovery: (store: MessageStore, account: AccountKey, asked: SearchQuery, messenger?: Partial<Pick<MessengerTraits, "savedChatId" | "app" | "history" | "provider">>, meaning?: MeaningLane) => Promise<SearchFound>;
@@ -5,7 +5,10 @@ import { exhausted, QUERY_LIMITS } from "../search/lucene/types.js";
5
5
  import { planQuestion, words } from "../search/question-plan.js";
6
6
  import { searchCombined } from "./messages-combined.js";
7
7
  import { prepareLucene, searchLucene } from "./messages-search.js";
8
- export const searchDiscovery = async (store, account, asked, messenger = {}) => {
8
+ /** How many of the messages nearest in meaning join the fusion: the benchmark's pool (performance/search/evidence-pipeline). */
9
+ export const MEANING_DEPTH = 100;
10
+ const RRF_K = 60;
11
+ export const searchDiscovery = async (store, account, asked, messenger = {}, meaning) => {
9
12
  // A question's own "?" would parse as a one-character wildcard and send it to strict search (BUG-395).
10
13
  const request = asked.ast !== undefined || asked.exact || !asked.text
11
14
  ? asked
@@ -25,6 +28,8 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
25
28
  return searchLucene(store, account, request, messenger);
26
29
  if (terms.length > 64)
27
30
  exhausted("discovery terms");
31
+ // Before the clock starts: loading the model alone takes about a second of the word search's budget.
32
+ const lane = meaning ? await meaning(text, original.scopeAccounts) : undefined;
28
33
  const started = performance.now();
29
34
  const check = () => {
30
35
  if (request.signal?.aborted)
@@ -115,8 +120,7 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
115
120
  const stem = (w) => original.execution.stemmer?.stemToken(w) ?? w;
116
121
  const stems = terms.map(stem);
117
122
  let bytes = 0;
118
- candidates = candidates
119
- .map((m) => {
123
+ const described = (m) => {
120
124
  check();
121
125
  const parent = parentByLocator.get(m.locator);
122
126
  const body = `${parent ? `${parent.text}\n` : ""}${m.text}`;
@@ -137,9 +141,17 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
137
141
  ...(parent ? { parent: parent.locator } : {}),
138
142
  },
139
143
  };
140
- })
144
+ };
145
+ candidates = candidates
146
+ .map(described)
141
147
  .sort((a, b) => (b.score ?? 0) - (a.score ?? 0) ||
142
148
  Number(parentByLocator.has(b.locator)) - Number(parentByLocator.has(a.locator)));
149
+ let meaningCandidates = 0;
150
+ if (lane && "ranked" in lane) {
151
+ const near = await nearEligible(store, account, request, messenger, lane.ranked, plan.scope);
152
+ meaningCandidates = near.length;
153
+ candidates = fusedWithMeaning(candidates, near, described);
154
+ }
143
155
  check();
144
156
  const items = await Promise.all(candidates.slice(0, request.limit).map(async ({ fused: _fused, ...m }) => {
145
157
  if (!request.context)
@@ -171,7 +183,52 @@ export const searchDiscovery = async (store, account, asked, messenger = {}) =>
171
183
  truncated,
172
184
  terms,
173
185
  queries: [...(plan.changed ? [] : plan.queries), relaxed],
186
+ ...(lane && "ranked" in lane ? { meaning: { model: lane.model, candidates: meaningCandidates } } : {}),
174
187
  },
175
188
  },
189
+ ...(lane && "skipped" in lane ? { meaningSkipped: lane.skipped } : {}),
190
+ ...(lane && "partly" in lane && lane.partly ? { meaningSkipped: lane.partly } : {}),
176
191
  };
177
192
  };
193
+ /** The nearest messages the question's filters allow, as hits, nearest first. */
194
+ const nearEligible = async (store, account, request, messenger, ranked, scope) => {
195
+ const keyOf = (one) => JSON.stringify([one.provider, one.account, one.chatId, one.id]);
196
+ const only = ranked
197
+ .slice(0, MEANING_DEPTH)
198
+ .filter((m) => !request.only || request.only.some((key) => keyOf(key) === keyOf(m)))
199
+ .map(({ provider, account, chatId, id }) => ({ provider, account, chatId, id }));
200
+ if (!only.length)
201
+ return [];
202
+ const eligible = await searchLucene(store, account, { ...request, text: scope || undefined, ast: undefined, limit: only.length, only, context: 0, thread: undefined }, messenger);
203
+ const hits = new Map(eligible.items.map((hit) => {
204
+ const { provider, account } = parseLocator(hit.locator);
205
+ return [keyOf({ provider, account, chatId: hit.chatId, id: hit.id }), hit];
206
+ }));
207
+ return ranked.flatMap((m) => {
208
+ const hit = hits.get(keyOf(m));
209
+ return hit ? [{ ...hit, cosine: m.score }] : [];
210
+ });
211
+ };
212
+ /**
213
+ * Reciprocal rank fusion of discovery's order and the meaning order, k = 60, as the benchmark's A2 arm: a hit
214
+ * scores 1 / (k + rank) in each list it is in. Coverage is left out of the fused score, so a message that answers
215
+ * in other words can rise above one that only repeats them.
216
+ */
217
+ const fusedWithMeaning = (words, near, described) => {
218
+ const merged = new Map();
219
+ const by = (hit, foundBy) => hit.discovery ? { ...hit, discovery: { ...hit.discovery, foundBy } } : hit;
220
+ words.forEach((hit, rank) => {
221
+ merged.set(hit.locator, { hit: by(hit, ["words"]), rrf: 1 / (RRF_K + rank + 1) });
222
+ });
223
+ near.forEach(({ cosine: _, ...m }, rank) => {
224
+ const share = 1 / (RRF_K + rank + 1);
225
+ const held = merged.get(m.locator);
226
+ if (held) {
227
+ held.rrf += share;
228
+ held.hit = by(held.hit, ["words", "meaning"]);
229
+ }
230
+ else
231
+ merged.set(m.locator, { hit: by(described({ ...m, fused: 0 }), ["meaning"]), rrf: share });
232
+ });
233
+ return [...merged.values()].sort((a, b) => b.rrf - a.rrf).map(({ hit, rrf }) => ({ ...hit, score: rrf }));
234
+ };
@@ -0,0 +1,7 @@
1
+ import type { ServiceDeps } from "./deps.js";
2
+ import { type MeaningLane } from "./messages-discovery.js";
3
+ /**
4
+ * Discovery's meaning half: the default local model over each message's own vector. It runs only when the model is
5
+ * on disk and vectors exist — never a download, never a remote model — and any failure leaves the words alone.
6
+ */
7
+ export declare const meaningLane: (deps: ServiceDeps) => MeaningLane;
@@ -0,0 +1,53 @@
1
+ import { DEFAULT_TEXT_MODEL } from "../embeddings/models.js";
2
+ import { embeddingTarget, withEmbedder } from "./embeddings-maintenance.js";
3
+ import { MEANING_DEPTH } from "./messages-discovery.js";
4
+ import { plainWords } from "./pictures.js";
5
+ /**
6
+ * Discovery's meaning half: the default local model over each message's own vector. It runs only when the model is
7
+ * on disk and vectors exist — never a download, never a remote model — and any failure leaves the words alone.
8
+ */
9
+ export const meaningLane = (deps) => async (question, accounts) => {
10
+ const target = embeddingTarget(deps, DEFAULT_TEXT_MODEL);
11
+ const skipped = (reason, step) => ({
12
+ skipped: { reason, model: target.id, ...(step ? { step } : {}) },
13
+ });
14
+ if (!target.installed)
15
+ return skipped(`${target.id} is not downloaded`, "models.download");
16
+ const store = await deps.store();
17
+ const ready = [];
18
+ const without = [];
19
+ for (const one of accounts)
20
+ ((await store.vectors.hasMessageVectors(one, target.key)) ? ready : without).push(one);
21
+ if (ready.length === 0)
22
+ return skipped("no message has a vector yet", "conversations.rebuild");
23
+ const words = plainWords(question);
24
+ try {
25
+ const ranked = [];
26
+ if (words) {
27
+ await withEmbedder(deps, target, DEFAULT_TEXT_MODEL, {}, async (embedder) => {
28
+ const [query] = await embedder.embed([words], "query");
29
+ for (const one of ready) {
30
+ const near = await store.vectors.nearestMessages(one, {
31
+ model: target.key,
32
+ limit: MEANING_DEPTH,
33
+ query: query,
34
+ });
35
+ for (const { chatId, messageId, score } of near)
36
+ ranked.push({ ...one, chatId, id: messageId, score });
37
+ }
38
+ });
39
+ }
40
+ return {
41
+ model: target.id,
42
+ ranked: ranked.sort((a, b) => b.score - a.score).slice(0, MEANING_DEPTH),
43
+ ...(without.length
44
+ ? {
45
+ partly: skipped(`${without.map(({ provider, account }) => `${provider}:${account}`).join(", ")} has no message vectors yet`, "conversations.rebuild").skipped,
46
+ }
47
+ : {}),
48
+ };
49
+ }
50
+ catch (error) {
51
+ return skipped(`${target.id} failed: ${error instanceof Error ? error.message : String(error)}`);
52
+ }
53
+ };
@@ -15,6 +15,11 @@ export interface QueryMetadata {
15
15
  truncated: boolean;
16
16
  terms: string[];
17
17
  queries: string[];
18
+ /** Present when meaning was searched too: the model, and how many of its nearest messages the filters allowed. */
19
+ meaning?: {
20
+ model: string;
21
+ candidates: number;
22
+ };
18
23
  };
19
24
  version: 1;
20
25
  fieldsVersion: number;
@@ -7,6 +7,7 @@ import { type Operated } from "../sends/guarded.js";
7
7
  import type { PermissionKey } from "../sends/permissions.js";
8
8
  import type { AccountKey, ChatCompleteness, MatchPlace, StoredHit } from "../store/store.js";
9
9
  import { type ServiceDeps } from "./deps.js";
10
+ import type { MeaningSkipped } from "./messages-discovery.js";
10
11
  import type { MessageStats, QueryMetadata, SearchCoverage, StatsGrouping } from "./messages-search.js";
11
12
  import type { PictureHit } from "./pictures.js";
12
13
  import { type SearchAllFound, type SearchAllRequest } from "./search-all.js";
@@ -79,6 +80,8 @@ export type FoundMessage = StoredHit & {
79
80
  matchedTerms: string[];
80
81
  missingTerms: string[];
81
82
  parent?: string;
83
+ /** Only when meaning was searched too: which of the two lists found it. */
84
+ foundBy?: ("words" | "meaning")[];
82
85
  };
83
86
  match?: "words" | "beginnings" | "corrected";
84
87
  /** The garbled spellings a `corrected` hit's transcript holds, per query word. */
@@ -116,6 +119,8 @@ export interface SearchFound extends Page<FoundMessage> {
116
119
  /** Per chat of the page: how much of its history the store holds. */
117
120
  completeness: (ChatCompleteness & AccountKey)[];
118
121
  /** `false` while the word index is being built. */
122
+ /** `--discover` searched by words alone: why meaning was not searched, and the step that would enable it. */
123
+ meaningSkipped?: MeaningSkipped;
119
124
  wordsReady: boolean;
120
125
  /** The same for the stems, in strict search: `false` until a stemmed search may run. */
121
126
  stemsReady?: boolean;
@@ -12,6 +12,7 @@ import { uploadSize } from "../sends/upload.js";
12
12
  import { crossAccount, fromStore, PUSHED } from "./deps.js";
13
13
  import { readChatId, storedChatId } from "./message-resolution.js";
14
14
  import { accountOf, keyOf, searchStore, statsStore, topUp } from "./messages-archive-search.js";
15
+ import { meaningLane } from "./messages-meaning.js";
15
16
  import { searchAll } from "./search-all.js";
16
17
  import { searchAllWithMeetings, } from "./search-all-meetings.js";
17
18
  import { searchRecordOf } from "./search-record.js";
@@ -31,7 +32,7 @@ export const messagesService = (deps) => {
31
32
  return { ...account, chatId, id };
32
33
  })
33
34
  : undefined;
34
- const found = await searchStore(store, account, only ? { ...query, only } : query, deps.messenger);
35
+ const found = await searchStore(store, account, only ? { ...query, only } : query, deps.messenger, query.discover && query.kind !== "mail" ? meaningLane(deps) : undefined);
35
36
  return { found, server };
36
37
  };
37
38
  const unifiedMessages = (store, account) => async (query) => {
@@ -36,6 +36,7 @@ export const catchUpSearch = async (deps, chat, options, signal, clock = Date.no
36
36
  maxMessages,
37
37
  check,
38
38
  threads: 1,
39
+ messages: false,
39
40
  });
40
41
  check();
41
42
  return {
@@ -1,3 +1,4 @@
1
+ // biome-ignore-all lint/plugin/no-sqlite-driver: cli-core owns the store, so it opens the driver itself
1
2
  import { DatabaseSync } from "node:sqlite";
2
3
  /**
3
4
  * Import this module only through a dynamic import that follows `quietSqliteWarning()`: a static import loads