@remit/drizzle-service 0.0.98 → 0.0.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/package.json +1 -1
  2. package/src/batch.sqlite.test.ts +78 -0
  3. package/src/batch.ts +45 -0
  4. package/src/databases.ts +70 -0
  5. package/src/db.ts +1 -4
  6. package/src/generated-enum.test.ts +8 -0
  7. package/src/generated-id.test.ts +19 -0
  8. package/src/id.test.ts +0 -8
  9. package/src/id.ts +0 -6
  10. package/src/index.ts +16 -6
  11. package/src/repair/address-display-name.sqlite.test.ts +3 -5
  12. package/src/repair/address-display-name.ts +8 -3
  13. package/src/repair/junk-only-address.sqlite.test.ts +8 -36
  14. package/src/repair/stranded-sent-outbox.sqlite.test.ts +2 -9
  15. package/src/repos/account.conformance.sqlite.test.ts +1 -1
  16. package/src/repos/calendar-default.sqlite.test.ts +6 -6
  17. package/src/repos/calendar-put.sqlite.test.ts +40 -39
  18. package/src/repos/calendar-store.ts +273 -0
  19. package/src/repos/cascade-delete.sqlite.test.ts +7 -9
  20. package/src/repos/cascade-delete.test.ts +6 -13
  21. package/src/repos/cascade-delete.ts +127 -97
  22. package/src/repos/envelope.test.ts +1 -17
  23. package/src/repos/envelope.ts +83 -67
  24. package/src/repos/filter.test.ts +22 -2
  25. package/src/repos/filter.ts +1 -5
  26. package/src/repos/i4-account-config.test.ts +2 -2
  27. package/src/repos/i4-account-config.ts +5 -2
  28. package/src/repos/i4-account.test.ts +6 -2
  29. package/src/repos/i4-account.ts +12 -56
  30. package/src/repos/i4-address-junk-move.sqlite.test.ts +1 -1
  31. package/src/repos/i4-address-merge-flags.sqlite.test.ts +1 -2
  32. package/src/repos/i4-address.ts +58 -90
  33. package/src/repos/i4-mailbox-special-use.sqlite.test.ts +6 -2
  34. package/src/repos/i4-mailbox-special-use.ts +9 -6
  35. package/src/repos/i4-mailbox.sqlite.test.ts +12 -14
  36. package/src/repos/i4-mailbox.test.ts +17 -62
  37. package/src/repos/i4-mailbox.ts +121 -94
  38. package/src/repos/i4-organize-job-request.ts +0 -4
  39. package/src/repos/i4-outbox-attachment.ts +62 -33
  40. package/src/repos/mappers.ts +0 -16
  41. package/src/repos/message.sqlite.test.ts +62 -102
  42. package/src/repos/message.test.ts +6 -167
  43. package/src/repos/message.ts +227 -218
  44. package/src/repos/sync-warning.ts +76 -0
  45. package/src/repos/thread-message-body-muted.sqlite.test.ts +5 -10
  46. package/src/repos/thread-message-field-terms.sqlite.test.ts +5 -1
  47. package/src/repos/thread-message.sqlite.test.ts +2 -1
  48. package/src/repos/thread-message.test.ts +17 -4
  49. package/src/repos/thread-message.ts +502 -285
  50. package/src/repos/thread-search-predicates.ts +24 -26
  51. package/src/schema/message-data.ts +4 -8
  52. package/src/schema/sync-warning.ts +3 -0
  53. package/src/schema-sets/app.ts +17 -0
  54. package/src/schema-sets/mail.ts +32 -0
  55. package/src/schema-sets/search.ts +5 -0
  56. package/src/schema-version-accounting.sqlite.test.ts +56 -43
  57. package/src/schema.ts +3 -4
  58. package/src/search/thread-message-search.sqlite.test.ts +131 -0
  59. package/src/search/thread-message-search.ts +271 -0
  60. package/src/sqlite-client.ts +5 -7
  61. package/src/sqlite-repositories.ts +75 -0
  62. package/src/test-db-sqlite.ts +5 -13
  63. package/src/test-shipped-sqlite-schema.ts +46 -45
  64. package/src/test-sqlite.ts +35 -11
  65. package/src/vps-migrations-drift.sqlite.test.ts +81 -15
  66. package/src/account-granted-scopes-migration.sqlite.test.ts +0 -66
  67. package/src/account-synced-services-migration.sqlite.test.ts +0 -56
  68. package/src/mailbox-path-dedupe.sqlite.test.ts +0 -462
  69. package/src/mailbox-sync-status-backfill.sqlite.test.ts +0 -118
  70. package/src/outbox-created-cleanup.sqlite.test.ts +0 -83
  71. package/src/repair/duplicate-mailbox-mail.ts +0 -98
  72. package/src/repair/thread-message-category-contract.ts +0 -505
  73. package/src/repair/thread-message-category.sqlite.test.ts +0 -100
  74. package/src/repair/thread-message-category.ts +0 -415
  75. package/src/repos/calendar-migrations.sqlite.test.ts +0 -345
  76. package/src/repos/calendar-unit-of-work.ts +0 -31
  77. package/src/repos/config-import-atomicity.sqlite.test.ts +0 -255
  78. package/src/repos/filter-anchor-transaction.test.ts +0 -136
  79. package/src/repos/filter-anchor-transaction.ts +0 -45
  80. package/src/repos/filter-anchor.test.ts +0 -131
  81. package/src/repos/filter-anchor.ts +0 -105
  82. package/src/repos/i4-mailbox-special-use.structure.test.ts +0 -27
  83. package/src/repos/message-placement-backfill.sqlite.test.ts +0 -190
  84. package/src/repos/search-index-shape.test.ts +0 -82
  85. package/src/repos/search-index-shape.ts +0 -56
  86. package/src/repos/serialized-writes.sqlite.test.ts +0 -199
  87. package/src/repos/unit-of-work.sqlite.test.ts +0 -138
  88. package/src/repos/unit-of-work.test.ts +0 -109
  89. package/src/repos/unit-of-work.ts +0 -31
  90. package/src/schema/outbox.ts +0 -46
  91. package/src/schema-full-sqlite.ts +0 -13
  92. package/src/tx.sqlite.test.ts +0 -83
  93. package/src/tx.ts +0 -233
@@ -0,0 +1,271 @@
1
+ import {
2
+ and,
3
+ asc,
4
+ count,
5
+ desc,
6
+ eq,
7
+ gt,
8
+ inArray,
9
+ lt,
10
+ or,
11
+ type SQL,
12
+ sql,
13
+ } from "drizzle-orm";
14
+ import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
15
+ import { runBatch, type SqliteBatchItem } from "../batch.js";
16
+ import type { Db } from "../db.js";
17
+ import { threadMessageTable } from "../schema/thread-message.js";
18
+
19
+ // The FTS5 trigram table the search migration set creates
20
+ // (deploy/vps/migrations-sqlite/search). drizzle-kit cannot express a virtual
21
+ // table, so its DDL is that committed migration and this declaration only lets
22
+ // drizzle build statements against it; no drizzle-kit config reads it.
23
+ export const threadMessageSearchTable = sqliteTable("thread_message_fts", {
24
+ rowid: integer("rowid").notNull(),
25
+ threadMessageId: text("thread_message_id").notNull(),
26
+ accountConfigId: text("account_config_id").notNull(),
27
+ mailboxId: text("mailbox_id").notNull(),
28
+ sentDate: integer("sent_date").notNull(),
29
+ listId: text("list_id").notNull(),
30
+ subject: text("subject").notNull(),
31
+ sender: text("sender").notNull(),
32
+ body: text("body").notNull(),
33
+ });
34
+
35
+ type AnyDb = Db<Record<string, unknown>>;
36
+
37
+ type SearchDocument = typeof threadMessageSearchTable.$inferInsert;
38
+
39
+ // D1 refuses a statement with more than 100 bound parameters. A document is
40
+ // nine parameters, so ten documents per insert stay under it.
41
+ const DOCUMENTS_PER_STATEMENT = 10;
42
+ const DOCUMENTS_PER_REBUILD_PAGE = DOCUMENTS_PER_STATEMENT * 30;
43
+
44
+ // The document's rowid, derived from its thread message id so a write or a
45
+ // removal finds the document without a lookup. 53 bits, the integers a number
46
+ // carries exactly on every driver.
47
+ export const searchRowid = (threadMessageId: string): number => {
48
+ let h1 = 0xdeadbeef;
49
+ let h2 = 0x41c6ce57;
50
+ for (let index = 0; index < threadMessageId.length; index++) {
51
+ const code = threadMessageId.charCodeAt(index);
52
+ h1 = Math.imul(h1 ^ code, 2654435761);
53
+ h2 = Math.imul(h2 ^ code, 1597334677);
54
+ }
55
+ h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
56
+ h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
57
+ h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
58
+ h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
59
+ return 4294967296 * (2097151 & h2) + (h1 >>> 0);
60
+ };
61
+
62
+ const indexedColumns = {
63
+ threadMessageId: threadMessageTable.threadMessageId,
64
+ accountConfigId: threadMessageTable.accountConfigId,
65
+ mailboxId: threadMessageTable.mailboxId,
66
+ sentDate: threadMessageTable.sentDate,
67
+ listId: threadMessageTable.listId,
68
+ subject: threadMessageTable.subject,
69
+ fromName: threadMessageTable.fromName,
70
+ fromEmail: threadMessageTable.fromEmail,
71
+ snippet: threadMessageTable.snippet,
72
+ };
73
+
74
+ interface IndexedRow {
75
+ threadMessageId: string;
76
+ accountConfigId: string;
77
+ mailboxId: string;
78
+ sentDate: number;
79
+ listId: string | null;
80
+ subject: string | null;
81
+ fromName: string | null;
82
+ fromEmail: string | null;
83
+ snippet: string | null;
84
+ }
85
+
86
+ const toDocument = (row: IndexedRow): SearchDocument => ({
87
+ rowid: searchRowid(row.threadMessageId),
88
+ threadMessageId: row.threadMessageId,
89
+ accountConfigId: row.accountConfigId,
90
+ mailboxId: row.mailboxId,
91
+ sentDate: row.sentDate,
92
+ listId: row.listId ?? "",
93
+ subject: row.subject ?? "",
94
+ sender: `${row.fromName ?? ""} ${row.fromEmail ?? ""}`,
95
+ body: row.snippet ?? "",
96
+ });
97
+
98
+ const chunk = <T>(items: readonly T[], size: number): T[][] => {
99
+ const chunks: T[][] = [];
100
+ for (let start = 0; start < items.length; start += size) {
101
+ chunks.push(items.slice(start, start + size));
102
+ }
103
+ return chunks;
104
+ };
105
+
106
+ /** Where a page of candidates resumes: the last one the previous page held. */
107
+ export interface SearchPosition {
108
+ rank: number;
109
+ sentDate: number;
110
+ threadMessageId: string;
111
+ }
112
+
113
+ export type SearchCandidate = SearchPosition;
114
+
115
+ export interface CandidateQuery {
116
+ accountConfigId: string;
117
+ /** The mailboxes in scope; every mailbox of the config when absent. */
118
+ mailboxIds?: readonly string[];
119
+ /** The text condition, over the columns of {@link threadMessageSearchTable}. */
120
+ where: SQL;
121
+ /** Ranks first, highest first, when given. */
122
+ rank?: SQL<number>;
123
+ order: "asc" | "desc";
124
+ after: SearchPosition | null;
125
+ limit: number;
126
+ }
127
+
128
+ /**
129
+ * The thread-message search index, kept in its own database.
130
+ *
131
+ * It is written after the mail write it follows has committed, and never in
132
+ * the same batch: a write reads the rows it names back from the mail database
133
+ * and makes the index match them, so repeating one is harmless and a missed one
134
+ * is repaired by the next write to that row or by {@link rebuild}.
135
+ *
136
+ * A search is answered here — matched, scoped to its mailboxes, ordered and
137
+ * cut to a page — and the mail database is then asked for that page's rows
138
+ * only, so no statement carries more ids than a page holds.
139
+ */
140
+ export class SearchIndex {
141
+ constructor(
142
+ private readonly mail: AnyDb,
143
+ private readonly search: AnyDb,
144
+ ) {}
145
+
146
+ /** Make the index match the mail database for these thread messages. */
147
+ async sync(threadMessageIds: readonly string[]): Promise<void> {
148
+ const ids = [...new Set(threadMessageIds)];
149
+ for (const slice of chunk(ids, DOCUMENTS_PER_STATEMENT)) {
150
+ const rows = await this.mail
151
+ .select(indexedColumns)
152
+ .from(threadMessageTable)
153
+ .where(inArray(threadMessageTable.threadMessageId, slice));
154
+ const items: SqliteBatchItem[] = [
155
+ this.search
156
+ .delete(threadMessageSearchTable)
157
+ .where(
158
+ inArray(threadMessageSearchTable.rowid, slice.map(searchRowid)),
159
+ ),
160
+ ];
161
+ if (rows.length > 0) {
162
+ items.push(
163
+ this.search
164
+ .insert(threadMessageSearchTable)
165
+ .values(rows.map(toDocument)),
166
+ );
167
+ }
168
+ await runBatch(this.search, items);
169
+ }
170
+ }
171
+
172
+ async removeAccount(accountConfigId: string): Promise<void> {
173
+ await this.search
174
+ .delete(threadMessageSearchTable)
175
+ .where(eq(threadMessageSearchTable.accountConfigId, accountConfigId));
176
+ }
177
+
178
+ /** One page of matching thread messages, in the query's order. */
179
+ async candidates(query: CandidateQuery): Promise<SearchCandidate[]> {
180
+ const table = threadMessageSearchTable;
181
+ // The rank is computed once, in the inner select, and read by name
182
+ // after it: written out in the keyset and the ordering it would bind
183
+ // every term's parameters four times.
184
+ const matches = this.search
185
+ .select({
186
+ threadMessageId: table.threadMessageId,
187
+ sentDate: table.sentDate,
188
+ rank: (query.rank ?? sql<number>`0`).as("sender_rank"),
189
+ })
190
+ .from(table)
191
+ .where(
192
+ and(
193
+ eq(table.accountConfigId, query.accountConfigId),
194
+ query.mailboxIds
195
+ ? sql`${table.mailboxId} in (select value from json_each(${JSON.stringify(query.mailboxIds)}))`
196
+ : undefined,
197
+ query.where,
198
+ ),
199
+ )
200
+ .as("matches");
201
+ const { after } = query;
202
+ const dateAfter = after
203
+ ? or(
204
+ query.order === "desc"
205
+ ? lt(matches.sentDate, after.sentDate)
206
+ : gt(matches.sentDate, after.sentDate),
207
+ and(
208
+ eq(matches.sentDate, after.sentDate),
209
+ gt(matches.threadMessageId, after.threadMessageId),
210
+ ),
211
+ )
212
+ : undefined;
213
+ const keyset =
214
+ after && query.rank
215
+ ? or(
216
+ lt(matches.rank, after.rank),
217
+ and(eq(matches.rank, after.rank), dateAfter),
218
+ )
219
+ : dateAfter;
220
+ return this.search
221
+ .select()
222
+ .from(matches)
223
+ .where(keyset)
224
+ .orderBy(
225
+ ...(query.rank ? [desc(matches.rank)] : []),
226
+ query.order === "desc" ? desc(matches.sentDate) : asc(matches.sentDate),
227
+ asc(matches.threadMessageId),
228
+ )
229
+ .limit(query.limit);
230
+ }
231
+
232
+ /**
233
+ * Whether the index has fallen out of step with the mail database: a
234
+ * different number of documents than thread messages, as a new or deleted
235
+ * search database, or a write that died between the two, leaves it.
236
+ */
237
+ async needsRebuild(): Promise<boolean> {
238
+ const [[documents], [threads]] = await Promise.all([
239
+ this.search.select({ n: count() }).from(threadMessageSearchTable),
240
+ this.mail.select({ n: count() }).from(threadMessageTable),
241
+ ]);
242
+ return documents?.n !== threads?.n;
243
+ }
244
+
245
+ /** Drop every document and index the whole mail database again. */
246
+ async rebuild(): Promise<number> {
247
+ await this.search.delete(threadMessageSearchTable);
248
+ let indexed = 0;
249
+ let after = "";
250
+ for (;;) {
251
+ const rows = await this.mail
252
+ .select(indexedColumns)
253
+ .from(threadMessageTable)
254
+ .where(gt(threadMessageTable.threadMessageId, after))
255
+ .orderBy(asc(threadMessageTable.threadMessageId))
256
+ .limit(DOCUMENTS_PER_REBUILD_PAGE);
257
+ const last = rows.at(-1);
258
+ if (!last) return indexed;
259
+ await runBatch(
260
+ this.search,
261
+ chunk(rows, DOCUMENTS_PER_STATEMENT).map((slice) =>
262
+ this.search
263
+ .insert(threadMessageSearchTable)
264
+ .values(slice.map(toDocument)),
265
+ ),
266
+ );
267
+ indexed += rows.length;
268
+ after = last.threadMessageId;
269
+ }
270
+ }
271
+ }
@@ -1,5 +1,4 @@
1
1
  import type { Db } from "./db.js";
2
- import { serializeSqliteWrites } from "./tx.js";
3
2
 
4
3
  // The one place a SQLite connection is opened for the app (RFC 036 D3). Every
5
4
  // writer container (backend, imap-worker, smtp-worker, account-worker) opens the
@@ -7,10 +6,9 @@ import { serializeSqliteWrites } from "./tx.js";
7
6
  // readers, a 5 s busy_timeout so a writer waits out another's short transaction
8
7
  // instead of failing, synchronous=NORMAL (durable under WAL), foreign keys on.
9
8
  //
10
- // The handle is wrapped by `serializeSqliteWrites` before it leaves this
11
- // function, so a repo's insert/update/delete cannot bypass the in-process write
12
- // serialization (RFC 036 D3). `run`/`transaction`/reads pass through by design —
13
- // see the wrapper's comment.
9
+ // No write on this handle holds a transaction open across an `await`: a write
10
+ // set runs as one synchronous batch (`batch.ts`), so the shared connection is
11
+ // never inside another caller's transaction when a plain write lands.
14
12
  //
15
13
  // better-sqlite3 and its drizzle driver are imported dynamically so the whole
16
14
  // module stays out of the DynamoDB Lambda bundle (this package is `external`
@@ -40,10 +38,10 @@ export async function createSqliteDatabase<
40
38
  sqlite.pragma("synchronous = NORMAL");
41
39
  sqlite.pragma("foreign_keys = ON");
42
40
 
43
- const base: Db<TSchema> = drizzle(sqlite, { schema });
41
+ const db: Db<TSchema> = drizzle(sqlite, { schema });
44
42
 
45
43
  return {
46
- db: serializeSqliteWrites(base),
44
+ db,
47
45
  close: async () => {
48
46
  sqlite.close();
49
47
  },
@@ -0,0 +1,75 @@
1
+ import type { SqliteDatabases } from "./databases.js";
2
+ import { CalendarCollectionRepo } from "./repos/calendar-collection.js";
3
+ import { CalendarEventIndexRepo } from "./repos/calendar-event-index.js";
4
+ import { CalendarFeedTokenRepo } from "./repos/calendar-feed-token.js";
5
+ import { CalendarObjectRepo } from "./repos/calendar-object.js";
6
+ import { DrizzleCalendarStore } from "./repos/calendar-store.js";
7
+ import { CalendarSuggestionRepo } from "./repos/calendar-suggestion.js";
8
+ import { DrizzleEnvelopeRepository } from "./repos/envelope.js";
9
+ import { FilterRepo } from "./repos/filter.js";
10
+ import { AccountRepo } from "./repos/i4-account.js";
11
+ import { AccountConfigRepo } from "./repos/i4-account-config.js";
12
+ import { AccountExportRequestRepo } from "./repos/i4-account-export-request.js";
13
+ import { AccountSettingRepo } from "./repos/i4-account-setting.js";
14
+ import { AddressRepo } from "./repos/i4-address.js";
15
+ import { ConfigImportRepo } from "./repos/i4-config-import.js";
16
+ import { MailboxRepo } from "./repos/i4-mailbox.js";
17
+ import { MailboxLockRepo } from "./repos/i4-mailbox-lock.js";
18
+ import { MailboxSpecialUseRepo } from "./repos/i4-mailbox-special-use.js";
19
+ import { MessageFlagPushRepo } from "./repos/i4-message-flag-push.js";
20
+ import { MessagePlacementMoveRepo } from "./repos/i4-message-placement-move.js";
21
+ import { OrganizeJobRequestRepo } from "./repos/i4-organize-job-request.js";
22
+ import { OutboxAttachmentRepo } from "./repos/i4-outbox-attachment.js";
23
+ import { OutboxMessageRepo } from "./repos/i4-outbox-message.js";
24
+ import { SenderSignerStandingRepo } from "./repos/i4-sender-signer-standing.js";
25
+ import { LabelRepo } from "./repos/label.js";
26
+ import { DrizzleMessageRepository } from "./repos/message.js";
27
+ import { DrizzleMessageFlagRepository } from "./repos/message-flag.js";
28
+ import { MessageLabelRepo } from "./repos/message-label.js";
29
+ import { QuarantineRepo } from "./repos/quarantine.js";
30
+ import { SyncWarningRepo } from "./repos/sync-warning.js";
31
+ import { DrizzleThreadMessageRepository } from "./repos/thread-message.js";
32
+ import { SearchIndex } from "./search/thread-message-search.js";
33
+
34
+ // Every repo over the database that holds its tables (#1420).
35
+ export const createSqliteRepositories = ({
36
+ app,
37
+ mail,
38
+ search,
39
+ }: SqliteDatabases) => {
40
+ const searchIndex = new SearchIndex(mail, search);
41
+ return {
42
+ accountConfig: new AccountConfigRepo(app, mail),
43
+ account: new AccountRepo(app, mail),
44
+ accountSetting: new AccountSettingRepo(app),
45
+ address: new AddressRepo(mail),
46
+ mailbox: new MailboxRepo(mail, searchIndex),
47
+ mailboxSpecialUse: new MailboxSpecialUseRepo(app, mail),
48
+ mailboxLock: new MailboxLockRepo(mail),
49
+ message: new DrizzleMessageRepository(mail, searchIndex),
50
+ messageFlag: new DrizzleMessageFlagRepository(mail),
51
+ outboxMessage: new OutboxMessageRepo(mail),
52
+ outboxAttachment: new OutboxAttachmentRepo(mail),
53
+ threadMessage: new DrizzleThreadMessageRepository(mail, searchIndex),
54
+ envelope: new DrizzleEnvelopeRepository(mail),
55
+ accountExportRequest: new AccountExportRequestRepo(app),
56
+ configImport: new ConfigImportRepo(app),
57
+ quarantine: new QuarantineRepo(mail),
58
+ syncWarning: new SyncWarningRepo(mail),
59
+ organizeJobRequest: new OrganizeJobRequestRepo(mail),
60
+ placementMove: new MessagePlacementMoveRepo(mail),
61
+ flagPush: new MessageFlagPushRepo(mail),
62
+ filter: new FilterRepo(app),
63
+ label: new LabelRepo(app),
64
+ messageLabel: new MessageLabelRepo(mail),
65
+ senderSignerStanding: new SenderSignerStandingRepo(mail),
66
+ calendarCollection: new CalendarCollectionRepo(app),
67
+ calendarObject: new CalendarObjectRepo(app),
68
+ calendarEventIndex: new CalendarEventIndexRepo(app),
69
+ calendarFeedToken: new CalendarFeedTokenRepo(app),
70
+ calendarSuggestion: new CalendarSuggestionRepo(mail),
71
+ calendarStore: new DrizzleCalendarStore(app, mail),
72
+ };
73
+ };
74
+
75
+ export type SqliteRepositories = ReturnType<typeof createSqliteRepositories>;
@@ -1,16 +1,8 @@
1
- import { readFileSync } from "node:fs";
2
1
  import Database from "better-sqlite3";
3
2
  import { pushSQLiteSchema } from "drizzle-kit/api";
4
3
  import { drizzle } from "drizzle-orm/better-sqlite3";
5
4
  import type { Db } from "./db.js";
6
-
7
- // The exact FTS5 objects the migrator installs (RFC 036 D4), read from the one
8
- // committed source so a test runs the real search path, not a hand-copied twin.
9
- const searchIndexDdl = (): string =>
10
- readFileSync(
11
- new URL("../../../npm-scripts/sqlite-search-index.sql", import.meta.url),
12
- "utf8",
13
- );
5
+ import { applyMigrationSet } from "./test-shipped-sqlite-schema.js";
14
6
 
15
7
  // A real better-sqlite3 database (in-memory by default) with the schema pushed
16
8
  // from the drizzle table objects — the sqlite `pushSchema` — so a repo runs
@@ -53,11 +45,11 @@ export async function createSqliteTestDb<
53
45
  sqlite.exec(statement);
54
46
  }
55
47
 
56
- // Install the FTS5 search objects on top of the pushed schema so the
57
- // thread-message search predicates run their real trigram path (RFC 036 D4).
58
- // Opt-in: only schemas that include the `thread_message` table can carry it.
48
+ // Install the committed search set beside the pushed schema so the
49
+ // thread-message search predicates run their real trigram path. The one
50
+ // handle then plays both the mail and the search database.
59
51
  if (options?.searchIndex) {
60
- sqlite.exec(searchIndexDdl());
52
+ applyMigrationSet(sqlite, "search");
61
53
  }
62
54
 
63
55
  return {
@@ -1,7 +1,7 @@
1
- import { readdirSync, readFileSync } from "node:fs";
1
+ import { readFileSync } from "node:fs";
2
2
  import type Database from "better-sqlite3";
3
3
 
4
- // Read the committed SQLite entity migrations — the DDL a self-host deployment
4
+ // Read the committed SQLite migrations — the DDL a self-host deployment
5
5
  // actually runs — so a test can exercise the shipped shape instead of the one
6
6
  // `pushSQLiteSchema` derives from the drizzle table objects. The two are
7
7
  // generated from the same entities but only the pushed one is regenerated on
@@ -9,67 +9,68 @@ import type Database from "better-sqlite3";
9
9
  // (reader#73). Reading the files means a test fails when they drift, and
10
10
  // tracks them when they change.
11
11
 
12
- const MIGRATIONS_DIR = new URL(
13
- "../../../deploy/vps/migrations-sqlite/entities/",
14
- import.meta.url,
15
- );
12
+ /** The three databases a deployment migrates, one committed set each (#1420). */
13
+ export type MigrationSet = "app" | "mail" | "search";
16
14
 
17
- export const migrationSql = (tag: string): string =>
18
- readFileSync(new URL(`${tag}.sql`, MIGRATIONS_DIR), "utf8");
15
+ export const MIGRATION_SETS: readonly MigrationSet[] = [
16
+ "app",
17
+ "mail",
18
+ "search",
19
+ ];
19
20
 
20
- export interface MigrationJournalEntry {
21
+ const setDir = (set: MigrationSet): URL =>
22
+ new URL(`../../../deploy/vps/migrations-sqlite/${set}/`, import.meta.url);
23
+
24
+ const migrationSql = (set: MigrationSet, tag: string): string =>
25
+ readFileSync(new URL(`${tag}.sql`, setDir(set)), "utf8");
26
+
27
+ interface MigrationJournalEntry {
21
28
  idx: number;
22
29
  tag: string;
23
30
  }
24
31
 
25
- /** Every committed entity migration, in the order the migrator runs them. */
26
- export const migrationJournal = (): MigrationJournalEntry[] =>
32
+ /** Every committed migration of a set, in the order the migrator runs them. */
33
+ const migrationJournal = (set: MigrationSet): MigrationJournalEntry[] =>
27
34
  (
28
35
  JSON.parse(
29
- readFileSync(new URL("meta/_journal.json", MIGRATIONS_DIR), "utf8"),
36
+ readFileSync(new URL("meta/_journal.json", setDir(set)), "utf8"),
30
37
  ) as { entries: MigrationJournalEntry[] }
31
- ).entries;
32
-
33
- /** The `.sql` files present on disk, whatever the journal says about them. */
34
- export const migrationTagsOnDisk = (): string[] =>
35
- readdirSync(MIGRATIONS_DIR)
36
- .filter((name) => name.endsWith(".sql"))
37
- .map((name) => name.slice(0, -".sql".length))
38
- .sort();
39
-
40
- export interface MigrationSnapshot {
41
- id: string;
42
- prevId: string;
43
- }
38
+ ).entries.sort((left, right) => left.idx - right.idx);
44
39
 
45
- export const migrationSnapshot = (idx: number): MigrationSnapshot =>
46
- JSON.parse(
47
- readFileSync(
48
- new URL(
49
- `meta/${String(idx).padStart(4, "0")}_snapshot.json`,
50
- MIGRATIONS_DIR,
51
- ),
52
- "utf8",
53
- ),
54
- ) as MigrationSnapshot;
55
-
56
- /** The `CREATE TABLE` block for one table, as that migration declares it. */
57
- export const shippedTableDdl = (tag: string, table: string): string => {
58
- const match = migrationSql(tag).match(
40
+ /** The `CREATE TABLE` block for one mail table, as the mail baseline declares it. */
41
+ export const shippedTableDdl = (table: string): string => {
42
+ const [baseline] = migrationJournal("mail");
43
+ const match = migrationSql("mail", baseline.tag).match(
59
44
  new RegExp(`CREATE TABLE \`${table}\` \\([\\s\\S]*?\\n\\);`),
60
45
  );
61
46
  if (!match) {
62
- throw new Error(`${table} DDL not found in migration ${tag}`);
47
+ throw new Error(`${table} DDL not found in migration ${baseline.tag}`);
63
48
  }
64
49
  return match[0];
65
50
  };
66
51
 
67
- /** Run every statement of a committed migration against an open database. */
68
- export const applyMigration = (
52
+ /** The one statement of the mail baseline that creates a named index. */
53
+ export const shippedIndexDdl = (index: string): string => {
54
+ const [baseline] = migrationJournal("mail");
55
+ const statement = migrationSql("mail", baseline.tag)
56
+ .split("--> statement-breakpoint")
57
+ .find((candidate) => candidate.includes(`INDEX \`${index}\``));
58
+ if (!statement) {
59
+ throw new Error(`${index} not found in migration ${baseline.tag}`);
60
+ }
61
+ return statement;
62
+ };
63
+
64
+ /** Run every committed migration of a set against an open database. */
65
+ export const applyMigrationSet = (
69
66
  sqlite: Database.Database,
70
- tag: string,
67
+ set: MigrationSet,
71
68
  ): void => {
72
- for (const statement of migrationSql(tag).split("--> statement-breakpoint")) {
73
- sqlite.exec(statement);
69
+ for (const entry of migrationJournal(set)) {
70
+ for (const statement of migrationSql(set, entry.tag).split(
71
+ "--> statement-breakpoint",
72
+ )) {
73
+ sqlite.exec(statement);
74
+ }
74
75
  }
75
76
  };
@@ -1,23 +1,25 @@
1
+ import { join } from "node:path";
1
2
  import Database from "better-sqlite3";
2
3
  import { drizzle } from "drizzle-orm/better-sqlite3";
4
+ import { SQLITE_DATABASE_ENV, type SqliteDatabasePaths } from "./databases.js";
3
5
  import type { Db } from "./db.js";
4
6
  import {
5
- applyMigration,
6
- migrationJournal,
7
+ applyMigrationSet,
8
+ MIGRATION_SETS,
7
9
  } from "./test-shipped-sqlite-schema.js";
8
10
 
9
11
  /**
10
- * The store a test runs against: the committed SQLite entity migrations, in the
11
- * order the migrator runs them, over the same engine a self-host deployment
12
- * boots. Applying the shipped DDL rather than pushing the drizzle table objects
13
- * is the point — a test then fails when the two drift (reader#73).
12
+ * The store a test runs against: the committed SQLite migrations, in the order
13
+ * the migrator runs them, over the same engine a self-host deployment boots.
14
+ * Applying the shipped DDL rather than pushing the drizzle table objects is the
15
+ * point — a test then fails when the two drift (reader#73).
16
+ *
17
+ * All three sets land in the one database, which a test then hands to every
18
+ * repo whatever database the repo runs on in a deployment. Their tables do not
19
+ * overlap, so nothing collides.
14
20
  */
15
21
  export const applyShippedMigrations = (sqlite: Database.Database): void => {
16
- for (const entry of [...migrationJournal()].sort(
17
- (left, right) => left.idx - right.idx,
18
- )) {
19
- applyMigration(sqlite, entry.tag);
20
- }
22
+ for (const set of MIGRATION_SETS) applyMigrationSet(sqlite, set);
21
23
  };
22
24
 
23
25
  /**
@@ -37,3 +39,25 @@ export const createShippedSqliteDb = (): {
37
39
  close: () => sqlite.close(),
38
40
  };
39
41
  };
42
+
43
+ /**
44
+ * The three database files a deployment runs on, each migrated with its own
45
+ * set, under `directory` — and the environment pointed at them, so a
46
+ * composition root a test builds opens them exactly as a deployment does.
47
+ */
48
+ export const createShippedSqliteFiles = (
49
+ directory: string,
50
+ ): SqliteDatabasePaths => {
51
+ const paths: SqliteDatabasePaths = {
52
+ app: join(directory, "app.db"),
53
+ mail: join(directory, "mail.db"),
54
+ search: join(directory, "search.db"),
55
+ };
56
+ for (const set of MIGRATION_SETS) {
57
+ const sqlite = new Database(paths[set]);
58
+ applyMigrationSet(sqlite, set);
59
+ sqlite.close();
60
+ process.env[SQLITE_DATABASE_ENV[set]] = paths[set];
61
+ }
62
+ return paths;
63
+ };