@remit/search-service 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/package.json +66 -0
  2. package/src/anchor.test.ts +164 -0
  3. package/src/anchor.ts +127 -0
  4. package/src/backends/bedrock.test.ts +148 -0
  5. package/src/backends/bedrock.ts +105 -0
  6. package/src/backends/memory.test.ts +168 -0
  7. package/src/backends/memory.ts +152 -0
  8. package/src/backends/pgvector.integ.test.ts +174 -0
  9. package/src/backends/pgvector.ts +306 -0
  10. package/src/backends/runtime-import.ts +16 -0
  11. package/src/backends/s3-vectors.test.ts +929 -0
  12. package/src/backends/s3-vectors.ts +383 -0
  13. package/src/backends/sqlite-vec.integ.test.ts +144 -0
  14. package/src/backends/sqlite-vec.ts +250 -0
  15. package/src/bedrock.ts +4 -0
  16. package/src/chunking/chunker.test.ts +79 -0
  17. package/src/chunking/chunker.ts +56 -0
  18. package/src/chunking/entities.test.ts +82 -0
  19. package/src/chunking/entities.ts +74 -0
  20. package/src/chunking/entropy.test.ts +98 -0
  21. package/src/chunking/entropy.ts +161 -0
  22. package/src/chunking/keys.ts +22 -0
  23. package/src/chunking/structured.test.ts +120 -0
  24. package/src/chunking/structured.ts +79 -0
  25. package/src/content-hash.test.ts +27 -0
  26. package/src/content-hash.ts +10 -0
  27. package/src/embeddings.test.ts +28 -0
  28. package/src/embeddings.ts +149 -0
  29. package/src/from-env.test.ts +62 -0
  30. package/src/from-env.ts +130 -0
  31. package/src/index.ts +71 -0
  32. package/src/pgvector.ts +4 -0
  33. package/src/s3-vectors.ts +5 -0
  34. package/src/search.test.ts +772 -0
  35. package/src/search.ts +395 -0
  36. package/src/semantic-search.integ.test.ts +130 -0
  37. package/src/sqlite-vec.ts +4 -0
  38. package/src/types.ts +155 -0
  39. package/tsconfig.json +8 -0
@@ -0,0 +1,772 @@
1
+ import assert from "node:assert";
2
+ import { describe, it } from "node:test";
3
+ import {
4
+ MemoryVectorStore,
5
+ type VectorStoreService,
6
+ } from "./backends/memory.js";
7
+ import {
8
+ createDeterministicEmbeddingService,
9
+ type EmbeddingService,
10
+ } from "./embeddings.js";
11
+ import {
12
+ buildTextPreview,
13
+ DefaultSearchService,
14
+ literalMatchScore,
15
+ rerank,
16
+ tokenizeQuery,
17
+ truncateUtf8Bytes,
18
+ } from "./search.js";
19
+ import type {
20
+ ChunkMetadata,
21
+ EnvelopeChunkInput,
22
+ IndexEmailParams,
23
+ VectorMatch,
24
+ VectorQuery,
25
+ VectorRecord,
26
+ } from "./types.js";
27
+
28
+ const baseMetadata: IndexEmailParams["metadata"] = {
29
+ messageId: "msg-1",
30
+ threadId: "thread-1",
31
+ accountConfigId: "acct-1",
32
+ mailboxIds: ["mb-inbox"],
33
+ sentDate: 1_700_000_000,
34
+ isRead: false,
35
+ hasAttachment: false,
36
+ hasStars: false,
37
+ };
38
+
39
+ const aliceEnvelope: EnvelopeChunkInput = {
40
+ from: { name: "Alice", email: "alice@example.com" },
41
+ to: [{ name: "Bob", email: "bob@example.com" }],
42
+ cc: [],
43
+ bcc: [],
44
+ subject: "Q1 invoice review",
45
+ attachments: [],
46
+ };
47
+
48
+ const bobEnvelope: EnvelopeChunkInput = {
49
+ from: { name: "Bob", email: "bob@example.com" },
50
+ to: [{ name: "Carol", email: "carol@example.com" }],
51
+ cc: [],
52
+ bcc: [],
53
+ subject: "Project kickoff next week",
54
+ attachments: [
55
+ {
56
+ filename: "deck.pdf",
57
+ contentType: "application/pdf",
58
+ size: 100_000,
59
+ },
60
+ ],
61
+ };
62
+
63
+ const buildService = () => {
64
+ const store = new MemoryVectorStore();
65
+ const embedder = createDeterministicEmbeddingService({ dimensions: 128 });
66
+ const service = new DefaultSearchService({ embedder, store });
67
+ return { service, store };
68
+ };
69
+
70
+ const indexBoth = async (svc: DefaultSearchService): Promise<void> => {
71
+ await svc.index({
72
+ envelope: aliceEnvelope,
73
+ parsedBody: {
74
+ text: "I have reviewed the Q1 numbers in the spreadsheet and the team exceeded the renewal target by fourteen percent across the portfolio.",
75
+ html: null,
76
+ },
77
+ metadata: {
78
+ ...baseMetadata,
79
+ messageId: "msg-alice",
80
+ threadId: "thread-a",
81
+ fromName: "Alice",
82
+ subject: "Q1 invoice review",
83
+ },
84
+ });
85
+ await svc.index({
86
+ envelope: bobEnvelope,
87
+ parsedBody: {
88
+ text: "Kicking off the new platform migration project next quarter, please join the planning session on Friday and bring your roadmap notes.",
89
+ html: null,
90
+ },
91
+ metadata: {
92
+ ...baseMetadata,
93
+ messageId: "msg-bob",
94
+ threadId: "thread-b",
95
+ hasAttachment: true,
96
+ fromName: "Bob",
97
+ subject: "Project kickoff next week",
98
+ },
99
+ });
100
+ };
101
+
102
+ describe("DefaultSearchService", () => {
103
+ it("returns the email matching a sender name with the highest score", async () => {
104
+ const { service } = buildService();
105
+ await indexBoth(service);
106
+
107
+ const results = await service.search({
108
+ query: "alice",
109
+ accountConfigId: "acct-1",
110
+ });
111
+ assert.ok(results.length > 0);
112
+ assert.strictEqual(results[0].messageId, "msg-alice");
113
+ });
114
+
115
+ it("returns the email matching a recipient email address", async () => {
116
+ const { service } = buildService();
117
+ await indexBoth(service);
118
+
119
+ const results = await service.search({
120
+ query: "carol@example.com",
121
+ accountConfigId: "acct-1",
122
+ });
123
+ assert.ok(results.length > 0);
124
+ assert.strictEqual(results[0].messageId, "msg-bob");
125
+ });
126
+
127
+ it("dedupes by messageId so each message appears at most once", async () => {
128
+ const { service } = buildService();
129
+ await indexBoth(service);
130
+
131
+ const results = await service.search({
132
+ query: "alice",
133
+ accountConfigId: "acct-1",
134
+ });
135
+ const ids = results.map((r) => r.messageId);
136
+ assert.strictEqual(new Set(ids).size, ids.length);
137
+ });
138
+
139
+ it("filters by hasAttachment", async () => {
140
+ const { service } = buildService();
141
+ await indexBoth(service);
142
+
143
+ const results = await service.search({
144
+ query: "project",
145
+ accountConfigId: "acct-1",
146
+ hasAttachment: true,
147
+ });
148
+ assert.ok(results.every((r) => r.messageId === "msg-bob"));
149
+ });
150
+
151
+ it("removes all chunks for a deleted message", async () => {
152
+ const { service, store } = buildService();
153
+ await indexBoth(service);
154
+
155
+ const before = await store.query({
156
+ vector: new Array<number>(128).fill(0.1),
157
+ topK: 100,
158
+ });
159
+ assert.ok(before.some((m) => m.metadata.messageId === "msg-alice"));
160
+
161
+ await service.delete("msg-alice");
162
+
163
+ const after = await store.query({
164
+ vector: new Array<number>(128).fill(0.1),
165
+ topK: 100,
166
+ });
167
+ assert.ok(!after.some((m) => m.metadata.messageId === "msg-alice"));
168
+ });
169
+
170
+ it("scopes results to a single accountConfigId", async () => {
171
+ const { service } = buildService();
172
+ await indexBoth(service);
173
+ await service.index({
174
+ envelope: aliceEnvelope,
175
+ parsedBody: { text: null, html: null },
176
+ metadata: {
177
+ ...baseMetadata,
178
+ messageId: "msg-other-account",
179
+ threadId: "thread-other",
180
+ accountConfigId: "acct-2",
181
+ },
182
+ });
183
+
184
+ const results = await service.search({
185
+ query: "alice",
186
+ accountConfigId: "acct-2",
187
+ });
188
+ assert.ok(results.length > 0);
189
+ assert.ok(results.every((r) => r.messageId === "msg-other-account"));
190
+ });
191
+
192
+ it("returns fromName, subject, and sentDate in search results", async () => {
193
+ const { service } = buildService();
194
+ await indexBoth(service);
195
+
196
+ const results = await service.search({
197
+ query: "alice invoice",
198
+ accountConfigId: "acct-1",
199
+ });
200
+ assert.ok(results.length > 0);
201
+ const alice = results.find((r) => r.messageId === "msg-alice");
202
+ assert.ok(alice, "msg-alice should be in results");
203
+ assert.strictEqual(alice.fromName, "Alice");
204
+ assert.strictEqual(alice.subject, "Q1 invoice review");
205
+ assert.strictEqual(alice.sentDate, 1_700_000_000);
206
+ });
207
+
208
+ it("category filter returns only in-category hits", async () => {
209
+ const { service } = buildService();
210
+ await service.index({
211
+ envelope: aliceEnvelope,
212
+ parsedBody: {
213
+ text: "Quarterly renewal figures and the invoice reconciliation for the finance team review.",
214
+ html: null,
215
+ },
216
+ metadata: {
217
+ ...baseMetadata,
218
+ messageId: "msg-personal",
219
+ threadId: "thread-personal",
220
+ category: "personal",
221
+ },
222
+ });
223
+ await service.index({
224
+ envelope: aliceEnvelope,
225
+ parsedBody: {
226
+ text: "Weekly renewal newsletter roundup with the invoice highlights and finance stories for subscribers.",
227
+ html: null,
228
+ },
229
+ metadata: {
230
+ ...baseMetadata,
231
+ messageId: "msg-newsletter",
232
+ threadId: "thread-newsletter",
233
+ category: "newsletter",
234
+ },
235
+ });
236
+
237
+ const scoped = await service.search({
238
+ query: "renewal invoice finance",
239
+ accountConfigId: "acct-1",
240
+ category: "newsletter",
241
+ });
242
+ assert.ok(scoped.length > 0);
243
+ assert.ok(
244
+ scoped.every((r) => r.category === "newsletter"),
245
+ "every hit must be in the requested category",
246
+ );
247
+ });
248
+
249
+ // The real invariant is that a category filter only removes out-of-category
250
+ // hits from the candidate window — it never adds or reorders. It is NOT that
251
+ // `related(all)` limited to the top-N contains every scoped hit: with a large
252
+ // corpus both queries pull the same topK window then slice to `limit` by
253
+ // score, so an in-category hit ranked below the global top-N appears in the
254
+ // scoped result but not in the limited all-category result. That divergence
255
+ // is the point of the feature. We test the invariant against the full
256
+ // candidate window (a limit large enough that nothing is sliced off).
257
+ it("scoped hits are a subset of the same unsliced all-category window", async () => {
258
+ const { service } = buildService();
259
+ // Several strongly-matching personal messages that outrank the one
260
+ // newsletter, plus the newsletter itself. With a small all-limit the
261
+ // newsletter falls outside the top-N; the category scope surfaces it.
262
+ for (let i = 0; i < 5; i++) {
263
+ await service.index({
264
+ envelope: aliceEnvelope,
265
+ parsedBody: {
266
+ text: "Quarterly renewal invoice finance reconciliation for the finance team quarterly review.",
267
+ html: null,
268
+ },
269
+ metadata: {
270
+ ...baseMetadata,
271
+ messageId: `msg-personal-${i}`,
272
+ threadId: `thread-personal-${i}`,
273
+ category: "personal",
274
+ },
275
+ });
276
+ }
277
+ await service.index({
278
+ envelope: aliceEnvelope,
279
+ parsedBody: {
280
+ text: "Weekly roundup newsletter mentioning renewal and a finance story for subscribers.",
281
+ html: null,
282
+ },
283
+ metadata: {
284
+ ...baseMetadata,
285
+ messageId: "msg-newsletter",
286
+ threadId: "thread-newsletter",
287
+ category: "newsletter",
288
+ },
289
+ });
290
+
291
+ const allFull = await service.search({
292
+ query: "renewal invoice finance",
293
+ accountConfigId: "acct-1",
294
+ limit: 100,
295
+ });
296
+ const scoped = await service.search({
297
+ query: "renewal invoice finance",
298
+ accountConfigId: "acct-1",
299
+ category: "newsletter",
300
+ limit: 100,
301
+ });
302
+
303
+ const allIds = new Set(allFull.map((r) => r.messageId));
304
+ assert.ok(scoped.length > 0);
305
+ assert.ok(
306
+ scoped.every((r) => allIds.has(r.messageId)),
307
+ "every scoped hit must appear in the full (unsliced) all-category window",
308
+ );
309
+ assert.ok(
310
+ scoped.every((r) => r.category === "newsletter"),
311
+ "the scoped result is filtered to the requested category only",
312
+ );
313
+ });
314
+
315
+ it("omits fromName and subject when not stored in metadata (pre-enrichment vectors)", async () => {
316
+ const { service } = buildService();
317
+ // Index without display fields to simulate pre-enrichment vectors
318
+ await service.index({
319
+ envelope: aliceEnvelope,
320
+ parsedBody: {
321
+ text: "Pre-enrichment message content with enough substance to index",
322
+ html: null,
323
+ },
324
+ metadata: {
325
+ ...baseMetadata,
326
+ messageId: "msg-legacy",
327
+ threadId: "thread-legacy",
328
+ },
329
+ });
330
+
331
+ const results = await service.search({
332
+ query: "pre-enrichment message",
333
+ accountConfigId: "acct-1",
334
+ });
335
+ const legacy = results.find((r) => r.messageId === "msg-legacy");
336
+ assert.ok(legacy, "msg-legacy should be in results");
337
+ assert.strictEqual(legacy.sentDate, 1_700_000_000);
338
+ // fromName and subject should be absent for pre-enrichment vectors
339
+ assert.strictEqual(legacy.fromName, undefined);
340
+ assert.strictEqual(legacy.subject, undefined);
341
+ });
342
+ });
343
+
344
+ // Counts every vector handed to the store's upsert, so a re-index that writes
345
+ // nothing can be asserted as exactly zero PutVectors-bound records.
346
+ class SpyStore implements VectorStoreService {
347
+ private inner = new MemoryVectorStore();
348
+ writes: VectorRecord[][] = [];
349
+
350
+ upsert = async (vectors: VectorRecord[]): Promise<void> => {
351
+ this.writes.push(vectors);
352
+ await this.inner.upsert(vectors);
353
+ };
354
+ query = (params: VectorQuery): Promise<VectorMatch[]> =>
355
+ this.inner.query(params);
356
+ delete = (filter: { messageId: string }): Promise<void> =>
357
+ this.inner.delete(filter);
358
+ existingContentHashes = (chunkIds: string[]): Promise<Map<string, string>> =>
359
+ this.inner.existingContentHashes(chunkIds);
360
+ getByMessage = (messageId: string): Promise<VectorRecord[]> =>
361
+ this.inner.getByMessage(messageId);
362
+
363
+ written = (): number => this.writes.reduce((n, b) => n + b.length, 0);
364
+ reset = (): void => {
365
+ this.writes = [];
366
+ };
367
+ }
368
+
369
+ const invoiceBody =
370
+ "I have reviewed the Q1 numbers in the spreadsheet and the team exceeded the renewal target by fourteen percent across the portfolio.";
371
+
372
+ const aliceParams = (body: string): IndexEmailParams => ({
373
+ envelope: aliceEnvelope,
374
+ parsedBody: { text: body, html: null },
375
+ metadata: {
376
+ ...baseMetadata,
377
+ messageId: "msg-alice",
378
+ threadId: "thread-a",
379
+ fromName: "Alice",
380
+ subject: "Q1 invoice review",
381
+ },
382
+ });
383
+
384
+ describe("DefaultSearchService idempotent indexing", () => {
385
+ it("(a) a re-index of unchanged content writes zero vectors", async () => {
386
+ const store = new SpyStore();
387
+ const embedder = createDeterministicEmbeddingService({ dimensions: 128 });
388
+ const service = new DefaultSearchService({ embedder, store });
389
+
390
+ await service.index(aliceParams(invoiceBody));
391
+ assert.ok(store.written() > 0, "first index writes the message's chunks");
392
+
393
+ store.reset();
394
+ await service.index(aliceParams(invoiceBody));
395
+ assert.strictEqual(
396
+ store.written(),
397
+ 0,
398
+ "unchanged re-index must write nothing",
399
+ );
400
+ });
401
+
402
+ it("(b) changed body content re-PUTs", async () => {
403
+ const store = new SpyStore();
404
+ const embedder = createDeterministicEmbeddingService({ dimensions: 128 });
405
+ const service = new DefaultSearchService({ embedder, store });
406
+
407
+ await service.index(aliceParams(invoiceBody));
408
+ store.reset();
409
+
410
+ await service.index(
411
+ aliceParams(
412
+ "Completely different content: the renovation budget overran and we must escalate the vendor dispute before the quarter closes.",
413
+ ),
414
+ );
415
+ assert.ok(store.written() > 0, "changed content must re-PUT the chunks");
416
+ });
417
+
418
+ it("(c) an embedding model/version bump re-PUTs", async () => {
419
+ const store = new SpyStore();
420
+ const service128 = new DefaultSearchService({
421
+ embedder: createDeterministicEmbeddingService({ dimensions: 128 }),
422
+ store,
423
+ });
424
+ await service128.index(aliceParams(invoiceBody));
425
+ store.reset();
426
+
427
+ // Same store, same content, different embedding id (dimensions change).
428
+ const service256 = new DefaultSearchService({
429
+ embedder: createDeterministicEmbeddingService({ dimensions: 256 }),
430
+ store,
431
+ });
432
+ await service256.index(aliceParams(invoiceBody));
433
+ assert.ok(
434
+ store.written() > 0,
435
+ "a model/dimension change must invalidate the hash and re-embed",
436
+ );
437
+ });
438
+
439
+ it("(d) force re-PUTs unchanged content regardless", async () => {
440
+ const store = new SpyStore();
441
+ const embedder = createDeterministicEmbeddingService({ dimensions: 128 });
442
+ const service = new DefaultSearchService({ embedder, store });
443
+
444
+ const records = await service.prepareVectors(aliceParams(invoiceBody));
445
+ await service.upsertVectors(records);
446
+ store.reset();
447
+
448
+ const skipResult = await service.upsertVectors(records);
449
+ assert.strictEqual(skipResult.upserted, 0, "unchanged upsert skips all");
450
+ assert.strictEqual(store.written(), 0);
451
+
452
+ const forceResult = await service.upsertVectors(records, { force: true });
453
+ assert.strictEqual(
454
+ forceResult.upserted,
455
+ records.length,
456
+ "force re-PUTs every record",
457
+ );
458
+ assert.strictEqual(store.written(), records.length);
459
+ });
460
+ });
461
+
462
+ class CountingEmbedder implements EmbeddingService {
463
+ readonly embeddingId: string;
464
+ readonly dimensions: number;
465
+ private inner: EmbeddingService;
466
+ embedCalls = 0;
467
+ embeddedTexts = 0;
468
+ constructor(dimensions = 128) {
469
+ this.inner = createDeterministicEmbeddingService({ dimensions });
470
+ this.embeddingId = this.inner.embeddingId;
471
+ this.dimensions = this.inner.dimensions;
472
+ }
473
+ embed = async (texts: string[]): Promise<number[][]> => {
474
+ this.embedCalls += 1;
475
+ this.embeddedTexts += texts.length;
476
+ return this.inner.embed(texts);
477
+ };
478
+ }
479
+
480
+ describe("DefaultSearchService.indexIncremental", () => {
481
+ it("does not embed when content is unchanged and already indexed", async () => {
482
+ const store = new SpyStore();
483
+ const embedder = new CountingEmbedder();
484
+ const service = new DefaultSearchService({ embedder, store });
485
+
486
+ const first = await service.indexIncremental(aliceParams(invoiceBody));
487
+ assert.ok(first.upserted > 0, "first index embeds and writes");
488
+ assert.ok(embedder.embedCalls > 0);
489
+
490
+ embedder.embedCalls = 0;
491
+ store.reset();
492
+ const second = await service.indexIncremental(aliceParams(invoiceBody));
493
+ assert.strictEqual(embedder.embedCalls, 0, "unchanged: no embedding pass");
494
+ assert.strictEqual(store.written(), 0, "unchanged: nothing written");
495
+ assert.strictEqual(second.upserted, 0);
496
+ assert.ok(second.skipped > 0, "unchanged chunks are counted as skipped");
497
+ });
498
+
499
+ it("embeds only the changed chunks when the body changes", async () => {
500
+ const store = new SpyStore();
501
+ const embedder = new CountingEmbedder();
502
+ const service = new DefaultSearchService({ embedder, store });
503
+
504
+ await service.indexIncremental(aliceParams(invoiceBody));
505
+ embedder.embedCalls = 0;
506
+ store.reset();
507
+
508
+ const changed = await service.indexIncremental(
509
+ aliceParams(
510
+ "Completely different content: the renovation budget overran and we must escalate the vendor dispute before the quarter closes.",
511
+ ),
512
+ );
513
+ assert.ok(embedder.embedCalls > 0, "changed content re-embeds");
514
+ assert.ok(changed.upserted > 0, "changed content is written");
515
+ });
516
+
517
+ it("force re-embeds every chunk even when unchanged (move metadata refresh)", async () => {
518
+ const store = new SpyStore();
519
+ const embedder = new CountingEmbedder();
520
+ const service = new DefaultSearchService({ embedder, store });
521
+
522
+ await service.indexIncremental(aliceParams(invoiceBody));
523
+ embedder.embedCalls = 0;
524
+ store.reset();
525
+
526
+ const forced = await service.indexIncremental(aliceParams(invoiceBody), {
527
+ force: true,
528
+ });
529
+ assert.ok(embedder.embedCalls > 0, "force embeds regardless of hash");
530
+ assert.ok(forced.upserted > 0, "force re-writes every chunk");
531
+ assert.strictEqual(forced.skipped, 0);
532
+ });
533
+ });
534
+
535
+ const buildMatch = (
536
+ overrides: Omit<Partial<VectorMatch>, "metadata"> & {
537
+ metadata?: Partial<ChunkMetadata>;
538
+ },
539
+ ): VectorMatch => ({
540
+ chunkId: overrides.chunkId ?? "chunk-1",
541
+ score: overrides.score ?? 0.5,
542
+ metadata: {
543
+ messageId: "msg-1",
544
+ threadId: "thread-1",
545
+ accountConfigId: "acct-1",
546
+ mailboxIds: ["mb-inbox"],
547
+ chunkType: "body",
548
+ sentDate: 1_700_000_000,
549
+ isRead: false,
550
+ hasAttachment: false,
551
+ hasStars: false,
552
+ ...overrides.metadata,
553
+ },
554
+ });
555
+
556
+ describe("tokenizeQuery", () => {
557
+ it("lowercases and whitespace-splits", () => {
558
+ assert.deepStrictEqual(tokenizeQuery("Invoice NUMBER"), [
559
+ "invoice",
560
+ "number",
561
+ ]);
562
+ });
563
+
564
+ it("drops tokens shorter than 3 characters", () => {
565
+ assert.deepStrictEqual(tokenizeQuery("a to inv-98234"), ["inv-98234"]);
566
+ });
567
+
568
+ it("caps at 8 tokens", () => {
569
+ const query = Array.from({ length: 12 }, (_, i) => `word${i}`).join(" ");
570
+ assert.strictEqual(tokenizeQuery(query).length, 8);
571
+ });
572
+ });
573
+
574
+ describe("truncateUtf8Bytes", () => {
575
+ it("returns the string unchanged when it already fits the byte budget", () => {
576
+ assert.strictEqual(truncateUtf8Bytes("hello world", 100), "hello world");
577
+ });
578
+
579
+ it("truncates ASCII text to exactly the byte budget", () => {
580
+ const text = "a".repeat(100);
581
+ const truncated = truncateUtf8Bytes(text, 40);
582
+ assert.strictEqual(Buffer.byteLength(truncated, "utf8"), 40);
583
+ });
584
+
585
+ it("never splits a multi-byte CJK character (stays valid UTF-8, no replacement char)", () => {
586
+ // Each character is a 3-byte UTF-8 CJK ideograph; a byte budget that isn't a
587
+ // multiple of 3 forces the truncator to back off mid-sequence.
588
+ const text = "書".repeat(50);
589
+ const truncated = truncateUtf8Bytes(text, 41);
590
+ assert.ok(Buffer.byteLength(truncated, "utf8") <= 41);
591
+ assert.ok(
592
+ !truncated.includes("�"),
593
+ "must not contain the UTF-8 replacement character",
594
+ );
595
+ assert.strictEqual(
596
+ Buffer.from(truncated, "utf8").toString("utf8"),
597
+ truncated,
598
+ "must round-trip through UTF-8 unchanged",
599
+ );
600
+ });
601
+
602
+ it("never splits a surrogate pair (4-byte UTF-8 emoji)", () => {
603
+ const text = "😀".repeat(50);
604
+ const truncated = truncateUtf8Bytes(text, 41);
605
+ assert.ok(Buffer.byteLength(truncated, "utf8") <= 41);
606
+ assert.ok(!truncated.includes("�"));
607
+ assert.strictEqual(
608
+ Buffer.from(truncated, "utf8").toString("utf8"),
609
+ truncated,
610
+ );
611
+ // A lone surrogate half would fail a round-trip through encodeURIComponent.
612
+ assert.doesNotThrow(() => encodeURIComponent(truncated));
613
+ });
614
+
615
+ it("returns an empty string when the budget is smaller than any single character", () => {
616
+ assert.strictEqual(truncateUtf8Bytes("書".repeat(10), 2), "");
617
+ });
618
+ });
619
+
620
+ describe("buildTextPreview", () => {
621
+ it("keeps a 512+ char CJK chunk under the byte budget and valid UTF-8", () => {
622
+ // 3 bytes/char in UTF-8; 600 chars is well past both the 512-char cap and
623
+ // the byte budget, so both bounds are exercised.
624
+ const cjkChunk = "取引先への請求書を添付いたします。".repeat(40);
625
+ assert.ok(cjkChunk.length > 512);
626
+
627
+ const preview = buildTextPreview(cjkChunk);
628
+
629
+ assert.ok(
630
+ Buffer.byteLength(preview, "utf8") <= 700,
631
+ `preview is ${Buffer.byteLength(preview, "utf8")} bytes, expected <= 700`,
632
+ );
633
+ assert.ok(!preview.includes("�"));
634
+ assert.strictEqual(Buffer.from(preview, "utf8").toString("utf8"), preview);
635
+ });
636
+
637
+ it("does not shorten a plain-ASCII 512-char preview (byte cap does not bite the common case)", () => {
638
+ const asciiChunk = "invoice payment reconciliation ".repeat(20);
639
+ assert.ok(asciiChunk.length > 512);
640
+
641
+ const preview = buildTextPreview(asciiChunk);
642
+
643
+ assert.strictEqual(preview.length, 512);
644
+ assert.strictEqual(preview, asciiChunk.slice(0, 512));
645
+ });
646
+ });
647
+
648
+ describe("literalMatchScore", () => {
649
+ it("returns undefined when textPreview is absent (missing-preview neutrality)", () => {
650
+ assert.strictEqual(literalMatchScore(["invoice"], undefined), undefined);
651
+ });
652
+
653
+ it("returns undefined when there are no qualifying query tokens", () => {
654
+ assert.strictEqual(literalMatchScore([], "some preview text"), undefined);
655
+ });
656
+
657
+ it("is case-insensitive", () => {
658
+ assert.strictEqual(
659
+ literalMatchScore(["invoice"], "Your INVOICE is attached"),
660
+ 1,
661
+ );
662
+ });
663
+
664
+ it("scores the fraction of tokens found as substrings", () => {
665
+ assert.strictEqual(
666
+ literalMatchScore(
667
+ ["invoice", "number", "zzz"],
668
+ "the invoice number is 42",
669
+ ),
670
+ 2 / 3,
671
+ );
672
+ });
673
+
674
+ it("returns 0 when no tokens match", () => {
675
+ assert.strictEqual(
676
+ literalMatchScore(["invoice"], "completely unrelated content"),
677
+ 0,
678
+ );
679
+ });
680
+ });
681
+
682
+ describe("rerank", () => {
683
+ it("ranks an exact literal match above a semantically-similar but literal-miss chunk", () => {
684
+ const literalHit = buildMatch({
685
+ chunkId: "chunk-literal",
686
+ score: 0.5,
687
+ metadata: {
688
+ messageId: "msg-literal",
689
+ textPreview: "Please see invoice INV-98234 attached for payment.",
690
+ },
691
+ });
692
+ const semanticNearMiss = buildMatch({
693
+ chunkId: "chunk-semantic",
694
+ score: 0.9,
695
+ metadata: {
696
+ messageId: "msg-semantic",
697
+ textPreview:
698
+ "Here is the billing statement for this quarter's charges.",
699
+ },
700
+ });
701
+
702
+ const [first, second] = rerank(
703
+ [semanticNearMiss, literalHit],
704
+ "INV-98234",
705
+ ).sort((a, b) => b.score - a.score);
706
+
707
+ assert.strictEqual(first.metadata.messageId, "msg-literal");
708
+ assert.strictEqual(second.metadata.messageId, "msg-semantic");
709
+ });
710
+
711
+ it("leaves cosine score untouched when textPreview is missing (score-neutral)", () => {
712
+ const legacy = buildMatch({
713
+ score: 0.42,
714
+ metadata: { textPreview: undefined },
715
+ });
716
+
717
+ const [result] = rerank([legacy], "invoice INV-98234");
718
+ assert.strictEqual(result.score, 0.42);
719
+ });
720
+
721
+ it("blends 40% cosine and 60% literal when a preview is present", () => {
722
+ const match = buildMatch({
723
+ score: 0.5,
724
+ metadata: { textPreview: "invoice inv-98234 attached" },
725
+ });
726
+
727
+ const [result] = rerank([match], "inv-98234");
728
+ assert.strictEqual(result.score, 0.4 * 0.5 + 0.6 * 1);
729
+ });
730
+ });
731
+
732
+ describe("DefaultSearchService.search hybrid re-ranking (integration)", () => {
733
+ it("ranks a message containing a literal query string first", async () => {
734
+ const { service } = buildService();
735
+ await service.index({
736
+ envelope: aliceEnvelope,
737
+ parsedBody: {
738
+ text: "Please process invoice INV-98234 for the March renewal before month end.",
739
+ html: null,
740
+ },
741
+ metadata: {
742
+ ...baseMetadata,
743
+ messageId: "msg-literal-invoice",
744
+ threadId: "thread-literal",
745
+ fromName: "Alice",
746
+ subject: "Invoice INV-98234",
747
+ },
748
+ });
749
+ await service.index({
750
+ envelope: bobEnvelope,
751
+ parsedBody: {
752
+ text: "Billing and payment reconciliation for the quarterly renewal cycle across all accounts.",
753
+ html: null,
754
+ },
755
+ metadata: {
756
+ ...baseMetadata,
757
+ messageId: "msg-semantic-only",
758
+ threadId: "thread-semantic",
759
+ fromName: "Bob",
760
+ subject: "Quarterly billing reconciliation",
761
+ },
762
+ });
763
+
764
+ const results = await service.search({
765
+ query: "INV-98234",
766
+ accountConfigId: "acct-1",
767
+ });
768
+
769
+ assert.ok(results.length > 0);
770
+ assert.strictEqual(results[0].messageId, "msg-literal-invoice");
771
+ });
772
+ });