@remit/drizzle-service 0.0.102 → 0.0.104

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/package.json +4 -3
  2. package/src/batch.sqlite.test.ts +4 -4
  3. package/src/batch.ts +47 -21
  4. package/src/databases.ts +24 -6
  5. package/src/db.ts +20 -5
  6. package/src/index.ts +12 -0
  7. package/src/repair/junk-only-address.sqlite.test.ts +7 -2
  8. package/src/repos/account.conformance.d1.test.ts +20 -0
  9. package/src/repos/account.conformance.sqlite.test.ts +6 -2
  10. package/src/repos/calendar-collection.conformance.d1.test.ts +20 -0
  11. package/src/repos/calendar-collection.ts +2 -2
  12. package/src/repos/calendar-default.d1.test.ts +4 -0
  13. package/src/repos/calendar-default.sqlite.test.ts +3 -72
  14. package/src/repos/calendar-default.suite.ts +68 -0
  15. package/src/repos/calendar-event-index.conformance.d1.test.ts +20 -0
  16. package/src/repos/calendar-event-index.ts +2 -2
  17. package/src/repos/calendar-feed-token.conformance.d1.test.ts +20 -0
  18. package/src/repos/calendar-feed-token.ts +2 -2
  19. package/src/repos/calendar-object.conformance.d1.test.ts +20 -0
  20. package/src/repos/calendar-object.ts +2 -2
  21. package/src/repos/calendar-put.d1.test.ts +4 -0
  22. package/src/repos/calendar-put.sqlite.test.ts +3 -205
  23. package/src/repos/calendar-put.suite.ts +210 -0
  24. package/src/repos/calendar-store.ts +36 -40
  25. package/src/repos/calendar-suggestion.ts +2 -2
  26. package/src/repos/cascade-delete.test.ts +3 -2
  27. package/src/repos/cascade-delete.ts +12 -8
  28. package/src/repos/envelope.sqlite.test.ts +3 -2
  29. package/src/repos/envelope.test.ts +3 -2
  30. package/src/repos/envelope.ts +14 -9
  31. package/src/repos/filter.d1.test.ts +4 -0
  32. package/src/repos/filter.suite.ts +476 -0
  33. package/src/repos/filter.test.ts +3 -472
  34. package/src/repos/filter.ts +2 -2
  35. package/src/repos/i4-account-config.d1.test.ts +4 -0
  36. package/src/repos/i4-account-config.suite.ts +131 -0
  37. package/src/repos/i4-account-config.test.ts +3 -123
  38. package/src/repos/i4-account-config.ts +8 -10
  39. package/src/repos/i4-account-export-request.d1.test.ts +4 -0
  40. package/src/repos/i4-account-export-request.suite.ts +44 -0
  41. package/src/repos/i4-account-export-request.test.ts +3 -40
  42. package/src/repos/i4-account-export-request.ts +2 -2
  43. package/src/repos/i4-account-setting.d1.test.ts +4 -0
  44. package/src/repos/i4-account-setting.suite.ts +100 -0
  45. package/src/repos/i4-account-setting.test.ts +3 -93
  46. package/src/repos/i4-account-setting.ts +2 -2
  47. package/src/repos/i4-account.test.ts +12 -8
  48. package/src/repos/i4-account.ts +6 -10
  49. package/src/repos/i4-address-junk-move.sqlite.test.ts +7 -2
  50. package/src/repos/i4-address.ts +12 -2
  51. package/src/repos/i4-config-import.ts +2 -2
  52. package/src/repos/i4-mailbox-lock.ts +2 -2
  53. package/src/repos/i4-mailbox-special-use.sqlite.test.ts +12 -10
  54. package/src/repos/i4-mailbox-special-use.ts +29 -40
  55. package/src/repos/i4-mailbox.sqlite.test.ts +12 -10
  56. package/src/repos/i4-mailbox.test.ts +15 -14
  57. package/src/repos/i4-mailbox.ts +13 -7
  58. package/src/repos/i4-message-flag-push.ts +2 -2
  59. package/src/repos/i4-message-placement-move.ts +2 -2
  60. package/src/repos/i4-organize-job-request.ts +2 -2
  61. package/src/repos/i4-outbox-attachment.test.ts +7 -2
  62. package/src/repos/i4-outbox-attachment.ts +11 -5
  63. package/src/repos/i4-outbox-message.ts +2 -2
  64. package/src/repos/i4-sender-signer-standing.ts +2 -2
  65. package/src/repos/label.conformance.d1.test.ts +20 -0
  66. package/src/repos/label.ts +2 -2
  67. package/src/repos/message-flag.ts +3 -6
  68. package/src/repos/message-label.ts +2 -2
  69. package/src/repos/message.sqlite.test.ts +57 -3
  70. package/src/repos/message.test.ts +6 -5
  71. package/src/repos/message.ts +81 -10
  72. package/src/repos/quarantine.ts +2 -2
  73. package/src/repos/sync-warning.ts +2 -2
  74. package/src/repos/thread-message-body-muted.sqlite.test.ts +30 -22
  75. package/src/repos/thread-message-field-terms.sqlite.test.ts +16 -11
  76. package/src/repos/thread-message.sqlite.test.ts +13 -7
  77. package/src/repos/thread-message.test.ts +43 -27
  78. package/src/repos/thread-message.ts +26 -14
  79. package/src/repos/thread-search-predicates.ts +36 -63
  80. package/src/schema/message-data.ts +3 -0
  81. package/src/schema-sets/mail.ts +1 -0
  82. package/src/search/thread-message-search.sqlite.test.ts +57 -11
  83. package/src/search/thread-message-search.ts +26 -12
  84. package/src/sqlite-repositories.ts +48 -32
  85. package/src/test-d1.ts +40 -0
  86. package/src/test-db-sqlite.ts +1 -1
  87. package/src/test-shipped-sqlite-schema.ts +8 -6
  88. package/src/test-stores.ts +30 -0
@@ -6,9 +6,9 @@ import {
6
6
  } from "../id.js";
7
7
  import { messageDataSchema } from "../schema/message-data.js";
8
8
  import { threadMessageTable } from "../schema/thread-message.js";
9
- import { SearchIndex } from "../search/thread-message-search.js";
9
+ import { createSqliteRepositories } from "../sqlite-repositories.js";
10
10
  import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
11
- import { DrizzleMessageRepository } from "./message.js";
11
+ import type { DrizzleMessageRepository } from "./message.js";
12
12
 
13
13
  const SCHEMA = { ...messageDataSchema, threadMessages: threadMessageTable };
14
14
 
@@ -43,7 +43,7 @@ describe("DrizzleMessageRepository (sqlite)", () => {
43
43
 
44
44
  before(async () => {
45
45
  ({ db, close } = await createSqliteTestDb(SCHEMA));
46
- repo = new DrizzleMessageRepository(db, new SearchIndex(db, db));
46
+ repo = createSqliteRepositories({ app: db, mail: db, search: db }).message;
47
47
  });
48
48
 
49
49
  after(async () => {
@@ -279,4 +279,58 @@ describe("DrizzleMessageRepository (sqlite)", () => {
279
279
  await repo.delete(MESSAGE_ID);
280
280
  await assert.rejects(() => repo.get(MESSAGE_ID));
281
281
  });
282
+
283
+ test("listMissingSearchText names stored bodies without search text, after the cursor", async () => {
284
+ const id = (n: number): string => `00000000-0000-0000-3333-00000000000${n}`;
285
+ for (const n of [1, 2, 3, 4]) {
286
+ await repo.create({ ...BASE_INPUT, messageId: id(n), uid: 100 + n });
287
+ }
288
+ for (const n of [1, 2, 4]) {
289
+ await repo.update(id(n), { bodyStorageKey: `file:///bodies/${n}` });
290
+ }
291
+ await repo.setSearchText(id(2), "already indexed");
292
+
293
+ const ours = (rows: Array<{ messageId: string }>): string[] =>
294
+ rows
295
+ .map((row) => row.messageId)
296
+ .filter((messageId) => messageId.includes("-3333-"));
297
+
298
+ assert.deepEqual(ours(await repo.listMissingSearchText("", 100)), [
299
+ id(1),
300
+ id(4),
301
+ ]);
302
+ assert.deepEqual(ours(await repo.listMissingSearchText(id(1), 100)), [
303
+ id(4),
304
+ ]);
305
+ assert.deepEqual(
306
+ await repo
307
+ .listMissingSearchText(id(1), 100)
308
+ .then((rows) => rows.find((row) => row.messageId === id(4))),
309
+ {
310
+ messageId: id(4),
311
+ bodyStorageKey: "file:///bodies/4",
312
+ },
313
+ );
314
+ });
315
+
316
+ test("copySearchText gives a copy the source's search text", async () => {
317
+ const source = "00000000-0000-0000-4444-000000000001";
318
+ const copy = "00000000-0000-0000-4444-000000000002";
319
+ for (const [messageId, uid] of [
320
+ [source, 201],
321
+ [copy, 202],
322
+ ] as const) {
323
+ await repo.create({ ...BASE_INPUT, messageId, uid });
324
+ await repo.update(messageId, { bodyStorageKey: "file:///bodies/shared" });
325
+ }
326
+ await repo.setSearchText(source, "the shared body");
327
+
328
+ await repo.copySearchText(source, copy);
329
+
330
+ const missing = await repo.listMissingSearchText("", 100);
331
+ assert.equal(
332
+ missing.some((row) => row.messageId === copy),
333
+ false,
334
+ );
335
+ });
282
336
  });
@@ -4,9 +4,9 @@ import {
4
4
  envelopeId as deriveEnvelopeId,
5
5
  rootBodyPartId as deriveRootBodyPartId,
6
6
  } from "../id.js";
7
- import { SearchIndex } from "../search/thread-message-search.js";
8
- import { DrizzleEnvelopeRepository } from "./envelope.js";
9
- import { DrizzleMessageRepository } from "./message.js";
7
+ import { createSqliteRepositories } from "../sqlite-repositories.js";
8
+ import type { DrizzleEnvelopeRepository } from "./envelope.js";
9
+ import type { DrizzleMessageRepository } from "./message.js";
10
10
  import { createTestDb, type TestDb } from "./test-helpers.js";
11
11
 
12
12
  describe("DrizzleMessageRepository", () => {
@@ -37,8 +37,9 @@ describe("DrizzleMessageRepository", () => {
37
37
 
38
38
  before(async () => {
39
39
  ({ db, stop } = await createTestDb());
40
- messageRepo = new DrizzleMessageRepository(db, new SearchIndex(db, db));
41
- envelopeRepo = new DrizzleEnvelopeRepository(db);
40
+ const repos = createSqliteRepositories({ app: db, mail: db, search: db });
41
+ messageRepo = repos.message;
42
+ envelopeRepo = repos.envelope;
42
43
  });
43
44
 
44
45
  after(async () => {
@@ -18,8 +18,8 @@ import {
18
18
  or,
19
19
  type SQL,
20
20
  } from "drizzle-orm";
21
- import { runBatch, type SqliteBatchItem } from "../batch.js";
22
- import type { Db } from "../db.js";
21
+ import type { SqliteBatchItem, WriteBatch } from "../batch.js";
22
+ import type { RepositoryDb, SqliteStore } from "../db.js";
23
23
  import {
24
24
  CreateFailedConflictError,
25
25
  isUniqueViolation,
@@ -37,9 +37,9 @@ import {
37
37
  bodyStructureRefetchTable,
38
38
  envelopeAddressTable,
39
39
  envelopeTable,
40
- type MessageDataSchema,
41
40
  messageFlagTable,
42
41
  messageReferenceTable,
42
+ messageSearchTextTable,
43
43
  messageTable,
44
44
  rawMessageStorageTable,
45
45
  } from "../schema/message-data.js";
@@ -56,7 +56,7 @@ import {
56
56
  toRawMessageStorageItem,
57
57
  } from "./mappers.js";
58
58
 
59
- type DB = Db<MessageDataSchema>;
59
+ type DB = RepositoryDb<Record<string, unknown>>;
60
60
 
61
61
  function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
62
62
  return {
@@ -112,7 +112,7 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
112
112
  };
113
113
  }
114
114
 
115
- export type SubtreeDb = Pick<Db<Record<string, unknown>>, "delete">;
115
+ export type SubtreeDb = Pick<RepositoryDb<Record<string, unknown>>, "delete">;
116
116
 
117
117
  /**
118
118
  * The statements that delete a message and its whole per-message subtree by
@@ -154,6 +154,9 @@ export function deleteMessageSubtree(
154
154
  .where(
155
155
  and(inList(bodyStructureRefetchTable.messageId, messageIds), guard),
156
156
  ),
157
+ db
158
+ .delete(messageSearchTextTable)
159
+ .where(and(inList(messageSearchTextTable.messageId, messageIds), guard)),
157
160
  db
158
161
  .delete(envelopeTable)
159
162
  .where(and(inList(envelopeTable.messageId, messageIds), guard)),
@@ -193,10 +196,16 @@ const placementTerms = (expected: PlacementPredicate): SQL[] => {
193
196
  };
194
197
 
195
198
  export class DrizzleMessageRepository implements IMessageRepository {
199
+ private readonly db: DB;
200
+ private readonly batch: WriteBatch;
201
+
196
202
  constructor(
197
- private db: DB,
203
+ store: SqliteStore<Record<string, unknown>>,
198
204
  private search: SearchIndex,
199
- ) {}
205
+ ) {
206
+ this.db = store.db;
207
+ this.batch = store.batch;
208
+ }
200
209
 
201
210
  private rowOf(input: CreateMessageInput) {
202
211
  const now = Date.now();
@@ -439,6 +448,68 @@ export class DrizzleMessageRepository implements IMessageRepository {
439
448
  return this.get(messageId);
440
449
  }
441
450
 
451
+ async setSearchText(messageId: string, text: string): Promise<void> {
452
+ const now = Date.now();
453
+ await this.db
454
+ .insert(messageSearchTextTable)
455
+ .values({ messageId, text, createdAt: now, updatedAt: now })
456
+ .onConflictDoUpdate({
457
+ target: messageSearchTextTable.messageId,
458
+ set: { text, updatedAt: now },
459
+ });
460
+ }
461
+
462
+ async copySearchText(
463
+ sourceMessageId: string,
464
+ targetMessageId: string,
465
+ ): Promise<void> {
466
+ const [source] = await this.db
467
+ .select({ text: messageSearchTextTable.text })
468
+ .from(messageSearchTextTable)
469
+ .where(eq(messageSearchTextTable.messageId, sourceMessageId));
470
+ if (!source) return;
471
+ await this.setSearchText(targetMessageId, source.text);
472
+ }
473
+
474
+ async syncSearch(messageIds: string[]): Promise<void> {
475
+ if (messageIds.length === 0) return;
476
+ const listings = await this.db
477
+ .select({ threadMessageId: threadMessageTable.threadMessageId })
478
+ .from(threadMessageTable)
479
+ .where(inList(threadMessageTable.messageId, messageIds));
480
+ await this.search.sync(listings.map((row) => row.threadMessageId));
481
+ }
482
+
483
+ async listMissingSearchText(
484
+ after: string,
485
+ limit: number,
486
+ ): ReturnType<IMessageRepository["listMissingSearchText"]> {
487
+ const rows = await this.db
488
+ .select({
489
+ messageId: messageTable.messageId,
490
+ bodyStorageKey: messageTable.bodyStorageKey,
491
+ })
492
+ .from(messageTable)
493
+ .leftJoin(
494
+ messageSearchTextTable,
495
+ eq(messageSearchTextTable.messageId, messageTable.messageId),
496
+ )
497
+ .where(
498
+ and(
499
+ gt(messageTable.messageId, after),
500
+ isNotNull(messageTable.bodyStorageKey),
501
+ isNull(messageSearchTextTable.messageId),
502
+ ),
503
+ )
504
+ .orderBy(asc(messageTable.messageId))
505
+ .limit(limit);
506
+ return rows.flatMap((row) =>
507
+ row.bodyStorageKey
508
+ ? [{ messageId: row.messageId, bodyStorageKey: row.bodyStorageKey }]
509
+ : [],
510
+ );
511
+ }
512
+
442
513
  async delete(messageId: string): Promise<void> {
443
514
  await this.deleteMany([messageId]);
444
515
  }
@@ -462,7 +533,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
462
533
  .select({ threadMessageId: threadMessageTable.threadMessageId })
463
534
  .from(threadMessageTable)
464
535
  .where(eq(threadMessageTable.messageId, messageId));
465
- const results = await runBatch(this.db, [
536
+ const results = await this.batch([
466
537
  this.db
467
538
  .delete(threadMessageTable)
468
539
  .where(and(eq(threadMessageTable.messageId, messageId), placed)),
@@ -475,7 +546,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
475
546
 
476
547
  async deleteMany(messageIds: string[]): Promise<void> {
477
548
  if (messageIds.length === 0) return;
478
- await runBatch(this.db, deleteMessageSubtree(this.db, messageIds));
549
+ await this.batch(deleteMessageSubtree(this.db, messageIds));
479
550
  }
480
551
 
481
552
  async listByMailbox(
@@ -623,7 +694,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
623
694
  .select({ threadMessageId: threadMessageTable.threadMessageId })
624
695
  .from(threadMessageTable)
625
696
  .where(eq(threadMessageTable.messageId, messageId));
626
- const [moved] = await runBatch(this.db, [
697
+ const [moved] = await this.batch([
627
698
  this.db
628
699
  .update(messageTable)
629
700
  .set({
@@ -6,10 +6,10 @@ import type {
6
6
  } from "@remit/data-ports";
7
7
  import { deriveQuarantineId } from "@remit/data-ports/id";
8
8
  import { desc, eq } from "drizzle-orm";
9
- import type { Db } from "../db.js";
9
+ import type { RepositoryDb } from "../db.js";
10
10
  import { quarantineTable } from "../schema/quarantine.js";
11
11
 
12
- type DB = Db<Record<string, unknown>>;
12
+ type DB = RepositoryDb<Record<string, unknown>>;
13
13
 
14
14
  function rowToItem(row: typeof quarantineTable.$inferSelect): QuarantineItem {
15
15
  return {
@@ -6,11 +6,11 @@ import type {
6
6
  } from "@remit/data-ports";
7
7
  import { deriveSyncWarningId } from "@remit/data-ports/id";
8
8
  import { and, count, desc, eq, gte, sql } from "drizzle-orm";
9
- import type { Db } from "../db.js";
9
+ import type { RepositoryDb } from "../db.js";
10
10
  import { syncWarningTable } from "../schema/sync-warning.js";
11
11
  import { inList } from "./in-list.js";
12
12
 
13
- type DB = Db<Record<string, unknown>>;
13
+ type DB = RepositoryDb<Record<string, unknown>>;
14
14
 
15
15
  export class SyncWarningRepo implements ISyncWarningRepository {
16
16
  constructor(private db: DB) {}
@@ -11,15 +11,11 @@ import assert from "node:assert/strict";
11
11
  import { after, before, describe, test } from "node:test";
12
12
  import type { CreateThreadMessageInput } from "@remit/data-ports";
13
13
  import { addressTable } from "../schema/i4-address.js";
14
+ import { messageSearchTextTable } from "../schema/message-data.js";
14
15
  import { threadMessageTable } from "../schema/thread-message.js";
15
- import { SearchIndex } from "../search/thread-message-search.js";
16
+ import { createSqliteRepositories } from "../sqlite-repositories.js";
16
17
  import { createSqliteTestDb } from "../test-db-sqlite.js";
17
- import { shippedIndexDdl } from "../test-shipped-sqlite-schema.js";
18
- import { DrizzleThreadMessageRepository } from "./thread-message.js";
19
-
20
- // The index the mail baseline creates for this predicate, read from the one
21
- // committed source so the plan assertion below is about the real object.
22
- const mutedIndexDdl = shippedIndexDdl("address_by_normalized_email");
18
+ import type { DrizzleThreadMessageRepository } from "./thread-message.js";
23
19
 
24
20
  const ACCOUNT = "acct-seam";
25
21
  const MAILBOX = "mbx-seam";
@@ -57,11 +53,18 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
57
53
 
58
54
  before(async () => {
59
55
  ({ db, sqlite, close } = await createSqliteTestDb(
60
- { threadMessage: threadMessageTable, address: addressTable },
56
+ {
57
+ threadMessage: threadMessageTable,
58
+ address: addressTable,
59
+ messageSearchText: messageSearchTextTable,
60
+ },
61
61
  { searchIndex: true },
62
62
  ));
63
- sqlite.exec(mutedIndexDdl);
64
- repo = new DrizzleThreadMessageRepository(db, new SearchIndex(db, db));
63
+ repo = createSqliteRepositories({
64
+ app: db,
65
+ mail: db,
66
+ search: db,
67
+ }).threadMessage;
65
68
  });
66
69
 
67
70
  after(async () => {
@@ -82,15 +85,20 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
82
85
  }),
83
86
  );
84
87
  }
85
- await repo.create(
86
- makeInput({
87
- subject: "unrelated note",
88
- fromEmail: "noise@example.com",
89
- fromName: "Noise",
90
- snippet: "Your parcel was left with the concierge",
91
- sentDate: BASE_DATE,
92
- }),
93
- );
88
+ const parcel = makeInput({
89
+ subject: "unrelated note",
90
+ fromEmail: "noise@example.com",
91
+ fromName: "Noise",
92
+ snippet: "Your parcel was left with the concierge",
93
+ sentDate: BASE_DATE,
94
+ });
95
+ await db.insert(messageSearchTextTable).values({
96
+ messageId: parcel.messageId,
97
+ text: "Your parcel was left with the concierge",
98
+ createdAt: 0,
99
+ updatedAt: 0,
100
+ });
101
+ await repo.create(parcel);
94
102
  });
95
103
 
96
104
  test("the term reaches the body text, not just subject and From", async () => {
@@ -119,16 +127,16 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
119
127
  assert.equal(result.items.length, 1);
120
128
  });
121
129
 
122
- test("a term under the trigram floor reads the body too", async () => {
130
+ test("a short word prefix reads the body too", async () => {
123
131
  const result = await repo.searchByDate(
124
132
  ACCOUNT,
125
- { query: "ge" },
133
+ { query: "co" },
126
134
  { mailboxIds: SCOPE, limit: 50 },
127
135
  );
128
136
 
129
137
  assert.ok(
130
138
  result.items.some((item) => item.snippet?.includes("concierge")),
131
- "the folded scan matched the body preview",
139
+ "the prefix matched the body preview",
132
140
  );
133
141
  });
134
142
 
@@ -1,11 +1,12 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { after, before, describe, test } from "node:test";
3
3
  import type { CreateThreadMessageInput } from "@remit/data-ports";
4
+ import { messageSearchTextTable } from "../schema/message-data.js";
4
5
  import { threadMessageTable } from "../schema/thread-message.js";
5
- import { SearchIndex } from "../search/thread-message-search.js";
6
+ import { createSqliteRepositories } from "../sqlite-repositories.js";
6
7
  import { createSqliteTestDb } from "../test-db-sqlite.js";
7
8
  import {
8
- DrizzleThreadMessageRepository,
9
+ type DrizzleThreadMessageRepository,
9
10
  THREAD_SEARCH_MAX_LIMIT,
10
11
  } from "./thread-message.js";
11
12
 
@@ -51,14 +52,18 @@ describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () =>
51
52
 
52
53
  before(async () => {
53
54
  const created = await createSqliteTestDb(
54
- { threadMessage: threadMessageTable },
55
+ {
56
+ threadMessage: threadMessageTable,
57
+ messageSearchText: messageSearchTextTable,
58
+ },
55
59
  { searchIndex: true },
56
60
  );
57
61
  close = created.close;
58
- repo = new DrizzleThreadMessageRepository(
59
- created.db,
60
- new SearchIndex(created.db, created.db),
61
- );
62
+ repo = createSqliteRepositories({
63
+ app: created.db,
64
+ mail: created.db,
65
+ search: created.db,
66
+ }).threadMessage;
62
67
 
63
68
  await repo.create(
64
69
  makeInput({
@@ -159,10 +164,10 @@ describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () =>
159
164
  ]);
160
165
  });
161
166
 
162
- // Below the trigram floor the predicate is the folded LIKE, and sqlite's
163
- // lower() folds ASCII only — `é` never matches a stored `CAFÉ`. Applying it
164
- // anyway would drop a row the caller's own matcher accepts, which is #459
165
- // again for that clause shape, so a short non-ASCII term narrows nothing.
167
+ // A field term is the folded LIKE, and sqlite's lower() folds ASCII only —
168
+ // `é` never matches a stored `CAFÉ`. Applying it anyway would drop a row
169
+ // the caller's own matcher accepts, which is #459 again for that clause
170
+ // shape, so a non-ASCII term narrows nothing.
166
171
  test("drops a short accented term rather than missing the row it should match", async () => {
167
172
  const result = await repo.listByFieldTerms(
168
173
  ACCOUNT,
@@ -1,8 +1,9 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { after, before, describe, test } from "node:test";
3
3
  import type { CreateThreadMessageInput } from "@remit/data-ports";
4
+ import { messageSearchTextTable } from "../schema/message-data.js";
4
5
  import { threadMessageTable } from "../schema/thread-message.js";
5
- import { SearchIndex } from "../search/thread-message-search.js";
6
+ import { createSqliteRepositories } from "../sqlite-repositories.js";
6
7
  import { createSqliteTestDb } from "../test-db-sqlite.js";
7
8
  import {
8
9
  BASE_DATE,
@@ -12,14 +13,14 @@ import {
12
13
  totalRows,
13
14
  } from "./category-fixture.js";
14
15
  import {
15
- DrizzleThreadMessageRepository,
16
+ type DrizzleThreadMessageRepository,
16
17
  THREAD_SEARCH_MAX_LIMIT,
17
18
  } from "./thread-message.js";
18
19
 
19
20
  // The thread-message repo on sqlite (RFC 036 D1): CRUD, keyset pagination, and
20
- // text search — the FTS5 trigram index for terms of three characters or more
21
- // (D4), the folded-LIKE scan for shorter terms. The search contract is accent-
22
- // and case-insensitive substring match over subject and sender.
21
+ // text search — the FTS5 unicode61 index (D4), matching words by prefix, and
22
+ // the folded-LIKE scan for terms with no word. The search contract is accent-
23
+ // and case-insensitive word-prefix match over subject and sender.
23
24
 
24
25
  const ACCOUNT = "acct-thread";
25
26
  const MAILBOX = "mbx-thread";
@@ -55,10 +56,15 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
55
56
  ({ db, sqlite, close } = await createSqliteTestDb(
56
57
  {
57
58
  threadMessage: threadMessageTable,
59
+ messageSearchText: messageSearchTextTable,
58
60
  },
59
61
  { searchIndex: true },
60
62
  ));
61
- repo = new DrizzleThreadMessageRepository(db, new SearchIndex(db, db));
63
+ repo = createSqliteRepositories({
64
+ app: db,
65
+ mail: db,
66
+ search: db,
67
+ }).threadMessage;
62
68
  });
63
69
 
64
70
  after(async () => {
@@ -111,7 +117,7 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
111
117
  assert.ok(byName.items.some((i) => i.fromName === "Alice Kramer"));
112
118
  });
113
119
 
114
- test("FTS trigram search folds diacritics both ways", async () => {
120
+ test("FTS search folds diacritics both ways", async () => {
115
121
  await repo.create(makeInput({ subject: "Réunion budget" }));
116
122
  const folded = await repo.searchByMailbox(
117
123
  ACCOUNT,
@@ -1,13 +1,17 @@
1
1
  import assert from "node:assert";
2
2
  import { after, before, describe, test } from "node:test";
3
- import type { CreateThreadMessageInput } from "@remit/data-ports";
3
+ import type {
4
+ CreateThreadMessageInput,
5
+ IMessageRepository,
6
+ } from "@remit/data-ports";
4
7
  import shortUuid from "short-uuid";
8
+ import { messageSearchTextTable } from "../schema/message-data.js";
5
9
  import { threadMessageTable } from "../schema/thread-message.js";
6
- import { SearchIndex } from "../search/thread-message-search.js";
10
+ import { createSqliteRepositories } from "../sqlite-repositories.js";
7
11
  import { createSqliteTestDb } from "../test-db-sqlite.js";
8
12
  import {
9
13
  clampThreadSearchLimit,
10
- DrizzleThreadMessageRepository,
14
+ type DrizzleThreadMessageRepository,
11
15
  deriveThreadMessageId,
12
16
  THREAD_SEARCH_MAX_LIMIT,
13
17
  } from "./thread-message.js";
@@ -40,11 +44,14 @@ function makeInput(
40
44
  };
41
45
  }
42
46
 
43
- // A real better-sqlite3 database with the FTS5 trigram objects the migrator
47
+ // A real better-sqlite3 database with the FTS5 objects the migrator
44
48
  // installs, so the search predicates run their shipped path.
45
49
  const setupDb = () =>
46
50
  createSqliteTestDb(
47
- { threadMessage: threadMessageTable },
51
+ {
52
+ threadMessage: threadMessageTable,
53
+ messageSearchText: messageSearchTextTable,
54
+ },
48
55
  { searchIndex: true },
49
56
  );
50
57
 
@@ -87,10 +94,11 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
87
94
  before(async () => {
88
95
  const harness = await setupDb();
89
96
  close = harness.close;
90
- repo = new DrizzleThreadMessageRepository(
91
- harness.db,
92
- new SearchIndex(harness.db, harness.db),
93
- );
97
+ repo = createSqliteRepositories({
98
+ app: harness.db,
99
+ mail: harness.db,
100
+ search: harness.db,
101
+ }).threadMessage;
94
102
  });
95
103
 
96
104
  after(async () => {
@@ -193,7 +201,7 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
193
201
  });
194
202
 
195
203
  // ── Scenario 4 ────────────────────────────────────────────────────────────
196
- // Matching runs over the WHOLE mailbox via the trigram index, so an old
204
+ // Matching runs over the WHOLE mailbox via the full-text index, so an old
197
205
  // match well behind the recent rows is still found and paged (the DynamoDB
198
206
  // recent-window bound does not apply here — the improvement over #443 on the
199
207
  // DDB path).
@@ -385,16 +393,20 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
385
393
 
386
394
  describe("DrizzleThreadMessageRepository.searchByDate", () => {
387
395
  let repo: DrizzleThreadMessageRepository;
396
+ let messages: IMessageRepository;
388
397
  let close: () => Promise<void>;
389
398
  const cleanup: Array<() => Promise<void>> = [];
390
399
 
391
400
  before(async () => {
392
401
  const harness = await setupDb();
393
402
  close = harness.close;
394
- repo = new DrizzleThreadMessageRepository(
395
- harness.db,
396
- new SearchIndex(harness.db, harness.db),
397
- );
403
+ const repositories = createSqliteRepositories({
404
+ app: harness.db,
405
+ mail: harness.db,
406
+ search: harness.db,
407
+ });
408
+ repo = repositories.threadMessage;
409
+ messages = repositories.message;
398
410
  });
399
411
 
400
412
  after(async () => {
@@ -410,9 +422,11 @@ describe("DrizzleThreadMessageRepository.searchByDate", () => {
410
422
  rows: Array<Partial<CreateThreadMessageInput>>,
411
423
  ): Promise<void> {
412
424
  for (const overrides of rows) {
413
- const created = await repo.create(
414
- makeInput(accountConfigId, mailboxId, overrides),
415
- );
425
+ const input = makeInput(accountConfigId, mailboxId, overrides);
426
+ if (overrides.snippet) {
427
+ await messages.setSearchText(input.messageId, overrides.snippet);
428
+ }
429
+ const created = await repo.create(input);
416
430
  cleanup.push(() => repo.delete(accountConfigId, created.threadMessageId));
417
431
  }
418
432
  }
@@ -921,10 +935,11 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
921
935
  before(async () => {
922
936
  const harness = await setupDb();
923
937
  close = harness.close;
924
- repo = new DrizzleThreadMessageRepository(
925
- harness.db,
926
- new SearchIndex(harness.db, harness.db),
927
- );
938
+ repo = createSqliteRepositories({
939
+ app: harness.db,
940
+ mail: harness.db,
941
+ search: harness.db,
942
+ }).threadMessage;
928
943
  });
929
944
 
930
945
  after(async () => {
@@ -1050,7 +1065,7 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
1050
1065
  assert.deepEqual(subjectsOf(res), ["shared token here"]);
1051
1066
  });
1052
1067
 
1053
- test("LIKE metacharacters in the query match literally", async () => {
1068
+ test("LIKE metacharacters in a query with no word match literally", async () => {
1054
1069
  const acct = uuid();
1055
1070
  const mbx = uuid();
1056
1071
  await seed(acct, mbx, [
@@ -1058,7 +1073,7 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
1058
1073
  { subject: "50 percent off" },
1059
1074
  ]);
1060
1075
 
1061
- const res = await repo.searchByMailboxWindow(acct, mbx, { query: "50%" });
1076
+ const res = await repo.searchByMailboxWindow(acct, mbx, { query: "%" });
1062
1077
  assert.deepEqual(subjectsOf(res), ["50% off today"]);
1063
1078
  });
1064
1079
 
@@ -1096,10 +1111,11 @@ describe("DrizzleThreadMessageRepository — core CRUD", () => {
1096
1111
  before(async () => {
1097
1112
  const harness = await setupDb();
1098
1113
  close = harness.close;
1099
- repo = new DrizzleThreadMessageRepository(
1100
- harness.db,
1101
- new SearchIndex(harness.db, harness.db),
1102
- );
1114
+ repo = createSqliteRepositories({
1115
+ app: harness.db,
1116
+ mail: harness.db,
1117
+ search: harness.db,
1118
+ }).threadMessage;
1103
1119
  });
1104
1120
 
1105
1121
  after(async () => {