@remit/drizzle-service 0.0.99 → 0.0.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/package.json +1 -1
  2. package/src/batch.sqlite.test.ts +78 -0
  3. package/src/batch.ts +45 -0
  4. package/src/databases.ts +70 -0
  5. package/src/db.ts +1 -4
  6. package/src/generated-enum.test.ts +8 -0
  7. package/src/generated-id.test.ts +19 -0
  8. package/src/id.test.ts +0 -8
  9. package/src/id.ts +0 -6
  10. package/src/index.ts +16 -6
  11. package/src/repair/address-display-name.sqlite.test.ts +3 -5
  12. package/src/repair/address-display-name.ts +8 -3
  13. package/src/repair/junk-only-address.sqlite.test.ts +8 -36
  14. package/src/repair/stranded-sent-outbox.sqlite.test.ts +2 -9
  15. package/src/repos/account.conformance.sqlite.test.ts +1 -1
  16. package/src/repos/calendar-default.sqlite.test.ts +6 -6
  17. package/src/repos/calendar-put.sqlite.test.ts +40 -39
  18. package/src/repos/calendar-store.ts +273 -0
  19. package/src/repos/cascade-delete.sqlite.test.ts +7 -9
  20. package/src/repos/cascade-delete.test.ts +6 -13
  21. package/src/repos/cascade-delete.ts +127 -97
  22. package/src/repos/envelope.test.ts +1 -17
  23. package/src/repos/envelope.ts +83 -67
  24. package/src/repos/filter.test.ts +22 -2
  25. package/src/repos/filter.ts +1 -5
  26. package/src/repos/i4-account-config.test.ts +2 -2
  27. package/src/repos/i4-account-config.ts +5 -2
  28. package/src/repos/i4-account.test.ts +6 -2
  29. package/src/repos/i4-account.ts +12 -56
  30. package/src/repos/i4-address-junk-move.sqlite.test.ts +1 -1
  31. package/src/repos/i4-address-merge-flags.sqlite.test.ts +1 -2
  32. package/src/repos/i4-address.ts +58 -90
  33. package/src/repos/i4-mailbox-special-use.sqlite.test.ts +6 -2
  34. package/src/repos/i4-mailbox-special-use.ts +9 -6
  35. package/src/repos/i4-mailbox.sqlite.test.ts +12 -14
  36. package/src/repos/i4-mailbox.test.ts +17 -62
  37. package/src/repos/i4-mailbox.ts +121 -94
  38. package/src/repos/i4-organize-job-request.ts +0 -4
  39. package/src/repos/i4-outbox-attachment.ts +62 -33
  40. package/src/repos/mappers.ts +0 -16
  41. package/src/repos/message.sqlite.test.ts +32 -106
  42. package/src/repos/message.test.ts +6 -167
  43. package/src/repos/message.ts +216 -228
  44. package/src/repos/sync-warning.ts +76 -0
  45. package/src/repos/thread-message-body-muted.sqlite.test.ts +5 -10
  46. package/src/repos/thread-message-field-terms.sqlite.test.ts +5 -1
  47. package/src/repos/thread-message.sqlite.test.ts +2 -1
  48. package/src/repos/thread-message.test.ts +17 -4
  49. package/src/repos/thread-message.ts +502 -285
  50. package/src/repos/thread-search-predicates.ts +24 -26
  51. package/src/schema/message-data.ts +4 -8
  52. package/src/schema/sync-warning.ts +3 -0
  53. package/src/schema-sets/app.ts +17 -0
  54. package/src/schema-sets/mail.ts +32 -0
  55. package/src/schema-sets/search.ts +5 -0
  56. package/src/schema-version-accounting.sqlite.test.ts +56 -43
  57. package/src/schema.ts +3 -4
  58. package/src/search/thread-message-search.sqlite.test.ts +131 -0
  59. package/src/search/thread-message-search.ts +271 -0
  60. package/src/sqlite-client.ts +5 -7
  61. package/src/sqlite-repositories.ts +75 -0
  62. package/src/test-db-sqlite.ts +5 -13
  63. package/src/test-shipped-sqlite-schema.ts +46 -45
  64. package/src/test-sqlite.ts +35 -11
  65. package/src/vps-migrations-drift.sqlite.test.ts +81 -15
  66. package/src/account-granted-scopes-migration.sqlite.test.ts +0 -66
  67. package/src/account-synced-services-migration.sqlite.test.ts +0 -56
  68. package/src/mailbox-path-dedupe.sqlite.test.ts +0 -462
  69. package/src/mailbox-sync-status-backfill.sqlite.test.ts +0 -118
  70. package/src/outbox-created-cleanup.sqlite.test.ts +0 -83
  71. package/src/repair/duplicate-mailbox-mail.ts +0 -98
  72. package/src/repair/thread-message-category-contract.ts +0 -505
  73. package/src/repair/thread-message-category.sqlite.test.ts +0 -100
  74. package/src/repair/thread-message-category.ts +0 -415
  75. package/src/repos/calendar-migrations.sqlite.test.ts +0 -345
  76. package/src/repos/calendar-unit-of-work.ts +0 -31
  77. package/src/repos/config-import-atomicity.sqlite.test.ts +0 -255
  78. package/src/repos/filter-anchor-transaction.test.ts +0 -136
  79. package/src/repos/filter-anchor-transaction.ts +0 -45
  80. package/src/repos/filter-anchor.test.ts +0 -131
  81. package/src/repos/filter-anchor.ts +0 -105
  82. package/src/repos/i4-mailbox-special-use.structure.test.ts +0 -27
  83. package/src/repos/message-placement-backfill.sqlite.test.ts +0 -190
  84. package/src/repos/search-index-shape.test.ts +0 -82
  85. package/src/repos/search-index-shape.ts +0 -56
  86. package/src/repos/serialized-writes.sqlite.test.ts +0 -199
  87. package/src/repos/unit-of-work.sqlite.test.ts +0 -138
  88. package/src/repos/unit-of-work.test.ts +0 -109
  89. package/src/repos/unit-of-work.ts +0 -31
  90. package/src/schema/outbox.ts +0 -46
  91. package/src/schema-full-sqlite.ts +0 -13
  92. package/src/tx.sqlite.test.ts +0 -83
  93. package/src/tx.ts +0 -233
@@ -1,462 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { describe, test } from "node:test";
3
- import * as entities from "@remit/drizzle-sqlite-schema";
4
- import Database from "better-sqlite3";
5
- import { asc, eq } from "drizzle-orm";
6
- import { drizzle } from "drizzle-orm/better-sqlite3";
7
- import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
8
- import type { Db } from "./db.js";
9
- import {
10
- PARKED_MAILBOX_ID,
11
- sweepDuplicateMailboxMail,
12
- } from "./repair/duplicate-mailbox-mail.js";
13
- import { MESSAGE_REMOVED_EVENT } from "./repos/message.js";
14
- import { outboxTable } from "./schema/outbox.js";
15
- import {
16
- applyMigration,
17
- migrationJournal,
18
- } from "./test-shipped-sqlite-schema.js";
19
-
20
- const DEDUPE = "0030_mailbox_path_dedupe";
21
- const UNIQUE = "0031_mailbox_path_unique";
22
- const MAILBOX_COLUMNS_ADDED_SINCE = ["0033_mailbox_sync_failure_reason"];
23
- const ACCOUNT = "acct";
24
- const CONFIG = "cfg";
25
-
26
- const filterAtDedupe = sqliteTable("filter", {
27
- filterId: text("filter_id").primaryKey(),
28
- accountConfigId: text("account_config_id").notNull(),
29
- name: text("name").notNull(),
30
- scope: text("scope").notNull(),
31
- ruleChangedAt: integer("rule_changed_at").notNull(),
32
- actionChangedAt: integer("action_changed_at").notNull(),
33
- actionMailboxId: text("action_mailbox_id").notNull(),
34
- createdAt: integer("created_at").notNull(),
35
- updatedAt: integer("updated_at").notNull(),
36
- });
37
-
38
- const organizeJobAtDedupe = sqliteTable("organize_job_request", {
39
- organizeJobId: text("organize_job_id").primaryKey(),
40
- accountConfigId: text("account_config_id").notNull(),
41
- userId: text("user_id").notNull(),
42
- actionMailboxId: text("action_mailbox_id").notNull(),
43
- ttl: integer("ttl").notNull(),
44
- createdAt: integer("created_at").notNull(),
45
- updatedAt: integer("updated_at").notNull(),
46
- });
47
-
48
- type Handle = { sqlite: Database.Database; db: Db<Record<string, unknown>> };
49
-
50
- const atPredecessor = (): Handle => {
51
- const sqlite = new Database(":memory:");
52
- for (const entry of migrationJournal()) {
53
- if (entry.tag === DEDUPE) {
54
- for (const tag of MAILBOX_COLUMNS_ADDED_SINCE)
55
- applyMigration(sqlite, tag);
56
- return {
57
- sqlite,
58
- db: drizzle(sqlite) as unknown as Db<Record<string, unknown>>,
59
- };
60
- }
61
- applyMigration(sqlite, entry.tag);
62
- }
63
- throw new Error(`${DEDUPE} is not in the journal`);
64
- };
65
-
66
- const seedMailbox = async (
67
- { db }: Handle,
68
- mailboxId: string,
69
- fullPath: string,
70
- createdAt: number,
71
- over: Partial<typeof entities.mailboxes.$inferInsert> = {},
72
- ): Promise<void> => {
73
- await db.insert(entities.mailboxes).values({
74
- mailboxId,
75
- accountId: ACCOUNT,
76
- namespacePrefix: "",
77
- hierarchyDelimiter: "/",
78
- fullPath,
79
- uidValidity: 1,
80
- uidNext: 1,
81
- highestModseq: "0",
82
- messageCount: 0,
83
- unseenCount: 0,
84
- deletedCount: 0,
85
- totalSize: 0,
86
- lastSyncUid: 0,
87
- highWaterMarkUid: 0,
88
- lastMessageSyncAt: 0,
89
- createdAt,
90
- updatedAt: createdAt,
91
- ...over,
92
- });
93
- };
94
-
95
- const seedMessage = async (
96
- { db }: Handle,
97
- messageId: string,
98
- mailboxId: string,
99
- uid = 1,
100
- ): Promise<void> => {
101
- await db.insert(entities.messages).values({
102
- messageId,
103
- mailboxId,
104
- uid,
105
- sequenceNumber: uid,
106
- rfc822Size: 1,
107
- internalDate: 0,
108
- envelopeId: `env-${messageId}`,
109
- rootBodyPartId: `bp-${messageId}`,
110
- createdAt: 0,
111
- updatedAt: 0,
112
- });
113
- await db.insert(entities.threadMessages).values({
114
- threadMessageId: `tm-${messageId}`,
115
- threadId: `thread-${messageId}`,
116
- messageId,
117
- accountConfigId: CONFIG,
118
- mailboxId,
119
- uid,
120
- referenceOrder: 0,
121
- internalDate: 0,
122
- sentDate: 0,
123
- isRead: false,
124
- hasAttachment: false,
125
- hasStars: false,
126
- isDeleted: false,
127
- createdAt: 0,
128
- updatedAt: 0,
129
- });
130
- };
131
-
132
- const seedSetting = async (
133
- { db }: Handle,
134
- name: string,
135
- value: typeof entities.accountSettings.$inferInsert.value,
136
- ): Promise<void> => {
137
- await db.insert(entities.accountSettings).values({
138
- accountSettingId: name,
139
- accountConfigId: CONFIG,
140
- name,
141
- value,
142
- createdAt: 0,
143
- updatedAt: 0,
144
- });
145
- };
146
-
147
- const dedupe = (handle: Handle): void => {
148
- applyMigration(handle.sqlite, DEDUPE);
149
- applyMigration(handle.sqlite, UNIQUE);
150
- };
151
-
152
- const mailboxIds = async ({ db }: Handle): Promise<string[]> =>
153
- (
154
- await db
155
- .select({ mailboxId: entities.mailboxes.mailboxId })
156
- .from(entities.mailboxes)
157
- .orderBy(asc(entities.mailboxes.mailboxId))
158
- ).map((row) => row.mailboxId);
159
-
160
- const messagesIn = async ({ db }: Handle, mailboxId: string): Promise<number> =>
161
- (
162
- await db
163
- .select({ messageId: entities.messages.messageId })
164
- .from(entities.messages)
165
- .where(eq(entities.messages.mailboxId, mailboxId))
166
- ).length;
167
-
168
- const threadsIn = async ({ db }: Handle, mailboxId: string): Promise<number> =>
169
- (
170
- await db
171
- .select({ messageId: entities.threadMessages.messageId })
172
- .from(entities.threadMessages)
173
- .where(eq(entities.threadMessages.mailboxId, mailboxId))
174
- ).length;
175
-
176
- const survivorCursor = async ({ db }: Handle, mailboxId: string) => {
177
- const [row] = await db
178
- .select({
179
- lastSyncUid: entities.mailboxes.lastSyncUid,
180
- highWaterMarkUid: entities.mailboxes.highWaterMarkUid,
181
- highestModseq: entities.mailboxes.highestModseq,
182
- })
183
- .from(entities.mailboxes)
184
- .where(eq(entities.mailboxes.mailboxId, mailboxId));
185
- return row;
186
- };
187
-
188
- describe("(account_id, full_path) becomes unique", () => {
189
- test("the loser's mail moves onto the survivor, so none of it is lost", async () => {
190
- const handle = atPredecessor();
191
- const cursor = {
192
- lastSyncUid: 1,
193
- highWaterMarkUid: 105,
194
- highestModseq: "900",
195
- };
196
- await seedMailbox(handle, "a", "Archive", 1, cursor);
197
- await seedMailbox(handle, "b", "Archive", 2, cursor);
198
- for (let uid = 1; uid <= 100; uid++) {
199
- await seedMessage(handle, `a-${uid}`, "a", uid);
200
- }
201
- for (let uid = 101; uid <= 105; uid++) {
202
- await seedMessage(handle, `b-${uid}`, "b", uid);
203
- }
204
-
205
- dedupe(handle);
206
-
207
- assert.deepEqual(await mailboxIds(handle), ["a"]);
208
- assert.equal(await messagesIn(handle, "a"), 105);
209
- assert.equal(await threadsIn(handle, "a"), 105);
210
- assert.equal(await messagesIn(handle, PARKED_MAILBOX_ID), 0);
211
- assert.deepEqual(await survivorCursor(handle, "a"), cursor);
212
- handle.sqlite.close();
213
- });
214
-
215
- test("what named the loser follows the survivor", async () => {
216
- const handle = atPredecessor();
217
- await seedMailbox(handle, "older-empty", "INBOX/Notifications", 1);
218
- await seedMailbox(handle, "with-mail", "INBOX/Notifications", 2);
219
- await seedMailbox(handle, "child", "INBOX/Notifications/Old", 3);
220
- await seedMailbox(handle, "elsewhere", "Receipts", 4);
221
- await seedMessage(handle, "m1", "with-mail");
222
- await seedMessage(handle, "copied", "elsewhere");
223
- await handle.db
224
- .update(entities.messages)
225
- .set({ originalMailboxId: "older-empty" })
226
- .where(eq(entities.messages.messageId, "copied"));
227
- await handle.db
228
- .update(entities.mailboxes)
229
- .set({ parentMailboxId: "older-empty" })
230
- .where(eq(entities.mailboxes.mailboxId, "child"));
231
- await handle.db.insert(filterAtDedupe).values({
232
- filterId: "f1",
233
- accountConfigId: CONFIG,
234
- name: "Alerts",
235
- scope: "Standing",
236
- ruleChangedAt: 0,
237
- actionChangedAt: 0,
238
- actionMailboxId: "older-empty",
239
- createdAt: 0,
240
- updatedAt: 0,
241
- });
242
- await handle.db.insert(organizeJobAtDedupe).values({
243
- organizeJobId: "job-1",
244
- accountConfigId: CONFIG,
245
- userId: "user",
246
- actionMailboxId: "older-empty",
247
- ttl: 0,
248
- createdAt: 0,
249
- updatedAt: 0,
250
- });
251
- await handle.db.insert(entities.messagePlacementMoves).values({
252
- messageId: "copied",
253
- accountId: ACCOUNT,
254
- accountConfigId: CONFIG,
255
- sourceMailboxId: "elsewhere",
256
- destinationMailboxId: "older-empty",
257
- createdAt: 0,
258
- updatedAt: 0,
259
- });
260
- await seedSetting(handle, "FolderRoleAppointment#acct#Archive", {
261
- kind: "String",
262
- value: "older-empty",
263
- });
264
- await handle.db.insert(entities.mailboxFlags).values({
265
- mailboxFlagId: "flag-1",
266
- mailboxId: "older-empty",
267
- flagName: "\\Seen",
268
- isPermanent: true,
269
- });
270
-
271
- dedupe(handle);
272
-
273
- assert.deepEqual(await mailboxIds(handle), [
274
- "child",
275
- "elsewhere",
276
- "with-mail",
277
- ]);
278
- const [child] = await handle.db
279
- .select({ parent: entities.mailboxes.parentMailboxId })
280
- .from(entities.mailboxes)
281
- .where(eq(entities.mailboxes.mailboxId, "child"));
282
- assert.equal(child?.parent, "with-mail");
283
- const [filter] = await handle.db.select().from(filterAtDedupe);
284
- assert.equal(filter?.actionMailboxId, "with-mail");
285
- const [job] = await handle.db.select().from(organizeJobAtDedupe);
286
- assert.equal(job?.actionMailboxId, "with-mail");
287
- const [copied] = await handle.db
288
- .select({ original: entities.messages.originalMailboxId })
289
- .from(entities.messages)
290
- .where(eq(entities.messages.messageId, "copied"));
291
- assert.equal(copied?.original, "with-mail");
292
- const [move] = await handle.db
293
- .select()
294
- .from(entities.messagePlacementMoves);
295
- assert.equal(move?.destinationMailboxId, "with-mail");
296
- const [setting] = await handle.db.select().from(entities.accountSettings);
297
- assert.deepEqual(setting?.value, { kind: "String", value: "with-mail" });
298
- assert.deepEqual(await handle.db.select().from(entities.mailboxFlags), []);
299
- const index = (
300
- handle.sqlite.prepare("PRAGMA index_list(mailbox)").all() as Array<{
301
- name: string;
302
- unique: number;
303
- }>
304
- ).find((entry) => entry.name === "mailbox_by_account_full_path");
305
- assert.equal(index?.unique, 1);
306
- handle.sqlite.close();
307
- });
308
-
309
- test("a muted losing folder stays muted, and the survivor's own name is kept", async () => {
310
- const handle = atPredecessor();
311
- await seedMailbox(handle, "survivor", "Newsletters", 1);
312
- await seedMailbox(handle, "loser", "Newsletters", 2);
313
- await seedSetting(handle, "MailboxMuted#loser", {
314
- kind: "Muted",
315
- value: true,
316
- });
317
- await seedSetting(handle, "MailboxDisplayName#loser", {
318
- kind: "String",
319
- value: "Loser name",
320
- });
321
- await seedSetting(handle, "MailboxDisplayName#survivor", {
322
- kind: "String",
323
- value: "Survivor name",
324
- });
325
-
326
- dedupe(handle);
327
-
328
- const settings = await handle.db
329
- .select({
330
- name: entities.accountSettings.name,
331
- value: entities.accountSettings.value,
332
- })
333
- .from(entities.accountSettings)
334
- .orderBy(asc(entities.accountSettings.name));
335
- assert.deepEqual(settings, [
336
- {
337
- name: "MailboxDisplayName#survivor",
338
- value: { kind: "String", value: "Survivor name" },
339
- },
340
- { name: "MailboxMuted#survivor", value: { kind: "Muted", value: true } },
341
- ]);
342
- handle.sqlite.close();
343
- });
344
-
345
- test("the survivor is chosen by the mail it holds, not by message_count", async () => {
346
- const handle = atPredecessor();
347
- await seedMailbox(handle, "claims-mail", "Archive", 1, {
348
- messageCount: 500,
349
- });
350
- await seedMailbox(handle, "holds-mail", "Archive", 2);
351
- await seedMessage(handle, "m1", "holds-mail", 1);
352
- await seedMessage(handle, "m2", "holds-mail", 2);
353
-
354
- dedupe(handle);
355
-
356
- assert.deepEqual(await mailboxIds(handle), ["holds-mail"]);
357
- handle.sqlite.close();
358
- });
359
-
360
- test("with equal mail the oldest row survives", async () => {
361
- const handle = atPredecessor();
362
- await seedMailbox(handle, "a-newer", "Archive", 2);
363
- await seedMailbox(handle, "b-older", "Archive", 1);
364
-
365
- dedupe(handle);
366
-
367
- assert.deepEqual(await mailboxIds(handle), ["b-older"]);
368
- handle.sqlite.close();
369
- });
370
-
371
- test("with equal mail and age the lowest mailbox_id survives", async () => {
372
- const handle = atPredecessor();
373
- await seedMailbox(handle, "b", "Archive", 1);
374
- await seedMailbox(handle, "a", "Archive", 1);
375
-
376
- dedupe(handle);
377
-
378
- assert.deepEqual(await mailboxIds(handle), ["a"]);
379
- handle.sqlite.close();
380
- });
381
-
382
- test("mail the survivor already holds at that uid is parked, removed at boot, and re-fetched", async () => {
383
- const handle = atPredecessor();
384
- const cursor = {
385
- lastSyncUid: 1,
386
- highWaterMarkUid: 9,
387
- highestModseq: "900",
388
- };
389
- await seedMailbox(handle, "older", "Archive", 1, cursor);
390
- await seedMailbox(handle, "newer", "Archive", 2, cursor);
391
- await seedMessage(handle, "kept", "older", 7);
392
- await seedMessage(handle, "stale", "newer", 7);
393
- await seedMessage(handle, "unrelated", "no-such-folder", 1);
394
-
395
- dedupe(handle);
396
-
397
- assert.deepEqual(await mailboxIds(handle), ["older"]);
398
- assert.equal(await messagesIn(handle, PARKED_MAILBOX_ID), 1);
399
- assert.equal(await threadsIn(handle, PARKED_MAILBOX_ID), 1);
400
- assert.deepEqual(await survivorCursor(handle, "older"), {
401
- lastSyncUid: 0,
402
- highWaterMarkUid: 0,
403
- highestModseq: "0",
404
- });
405
-
406
- const report = await sweepDuplicateMailboxMail(handle.db, "repair");
407
-
408
- assert.deepEqual(report, {
409
- mode: "repair",
410
- duplicatePaths: 0,
411
- parked: 1,
412
- removed: 1,
413
- });
414
- const messages = await handle.db
415
- .select({ messageId: entities.messages.messageId })
416
- .from(entities.messages)
417
- .orderBy(asc(entities.messages.messageId));
418
- assert.deepEqual(messages, [
419
- { messageId: "kept" },
420
- { messageId: "unrelated" },
421
- ]);
422
- const events = await handle.db
423
- .select({ messageId: outboxTable.messageId, event: outboxTable.event })
424
- .from(outboxTable);
425
- assert.deepEqual(events, [
426
- { messageId: "stale", event: MESSAGE_REMOVED_EVENT },
427
- ]);
428
- assert.deepEqual(await sweepDuplicateMailboxMail(handle.db, "repair"), {
429
- mode: "repair",
430
- duplicatePaths: 0,
431
- parked: 0,
432
- removed: 0,
433
- });
434
- handle.sqlite.close();
435
- });
436
-
437
- test("the check before migrating reports the paths the migration will merge", async () => {
438
- const handle = atPredecessor();
439
- await seedMailbox(handle, "a", "Archive", 1);
440
- await seedMailbox(handle, "b", "Archive", 2);
441
-
442
- assert.deepEqual(await sweepDuplicateMailboxMail(handle.db, "check"), {
443
- mode: "check",
444
- duplicatePaths: 1,
445
- parked: 0,
446
- removed: 0,
447
- });
448
- handle.sqlite.close();
449
- });
450
-
451
- test("the unique index is refused without the dedupe ahead of it", async () => {
452
- const handle = atPredecessor();
453
- await seedMailbox(handle, "a", "Archive", 1);
454
- await seedMailbox(handle, "b", "Archive", 2);
455
-
456
- assert.throws(
457
- () => applyMigration(handle.sqlite, UNIQUE),
458
- /UNIQUE constraint failed/,
459
- );
460
- handle.sqlite.close();
461
- });
462
- });
@@ -1,118 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { describe, test } from "node:test";
3
- import Database from "better-sqlite3";
4
- import {
5
- applyMigration,
6
- migrationJournal,
7
- } from "./test-shipped-sqlite-schema.js";
8
-
9
- /**
10
- * `sync_status` becomes NOT NULL, and on a real install almost every row is
11
- * NULL: the sweep's insert is the only create path that omitted it, and the
12
- * sweep is what discovers a folder in the first place.
13
- *
14
- * drizzle-kit does not emit data migrations, and SQLite cannot alter
15
- * nullability, so the schema change is a copy-and-swap whose `INSERT … SELECT`
16
- * would violate the new constraint on every one of those rows. Because drizzle
17
- * runs a folder's whole pending set inside one `BEGIN`/`COMMIT`, that failure
18
- * rolls the entities set back and the migrate one-shot that gates all six
19
- * services never completes.
20
- *
21
- * So the pair is proven here in the order the migrator runs it, against the
22
- * data it will actually meet.
23
- */
24
- const BACKFILL = "0026_mailbox_sync_status_backfill";
25
- const SCHEMA_CHANGE = "0027_mailbox_sync_status_total";
26
-
27
- const atPredecessor = (): Database.Database => {
28
- const sqlite = new Database(":memory:");
29
- for (const entry of migrationJournal()) {
30
- if (entry.tag === BACKFILL) return sqlite;
31
- applyMigration(sqlite, entry.tag);
32
- }
33
- throw new Error(`${BACKFILL} is not in the journal`);
34
- };
35
-
36
- const seed = (
37
- sqlite: Database.Database,
38
- rows: Array<{ mailboxId: string; syncStatus: string | null }>,
39
- ): void => {
40
- const insert = sqlite.prepare(
41
- `INSERT INTO mailbox (
42
- mailbox_id, account_id, namespace_type, namespace_prefix,
43
- hierarchy_delimiter, full_path, uid_validity, uid_next, highest_modseq,
44
- message_count, unseen_count, deleted_count, total_size, last_sync_uid,
45
- high_water_mark_uid, last_message_sync_at, parent_mailbox_id,
46
- sync_status, cursor_state, created_at, updated_at
47
- ) VALUES (?, 'acct', 'personal', '', '/', ?, 1, 1, '0', 0, 0, 0, 0, 0, 0, 0,
48
- 'None', ?, 'normal', 0, 0)`,
49
- );
50
- for (const row of rows) {
51
- insert.run(row.mailboxId, `Folder/${row.mailboxId}`, row.syncStatus);
52
- }
53
- };
54
-
55
- describe("mailbox sync_status becomes total", () => {
56
- test("the backfill runs first, so the rebuild sees no NULLs", () => {
57
- const sqlite = atPredecessor();
58
- seed(sqlite, [
59
- { mailboxId: "swept-1", syncStatus: null },
60
- { mailboxId: "swept-2", syncStatus: null },
61
- { mailboxId: "created", syncStatus: "pending" },
62
- { mailboxId: "failed", syncStatus: "failed" },
63
- ]);
64
-
65
- applyMigration(sqlite, BACKFILL);
66
- applyMigration(sqlite, SCHEMA_CHANGE);
67
-
68
- const rows = sqlite
69
- .prepare(
70
- "SELECT mailbox_id, sync_status FROM mailbox ORDER BY mailbox_id",
71
- )
72
- .all() as Array<{ mailbox_id: string; sync_status: string }>;
73
-
74
- assert.deepEqual(rows, [
75
- { mailbox_id: "created", sync_status: "pending" },
76
- { mailbox_id: "failed", sync_status: "failed" },
77
- { mailbox_id: "swept-1", sync_status: "synced" },
78
- { mailbox_id: "swept-2", sync_status: "synced" },
79
- ]);
80
- sqlite.close();
81
- });
82
-
83
- test("the rebuild is refused without the backfill ahead of it", () => {
84
- // The failure this ordering exists to prevent, made visible: without the
85
- // backfill the copy-and-swap cannot insert a NULL into the new column.
86
- const sqlite = atPredecessor();
87
- seed(sqlite, [{ mailboxId: "swept-1", syncStatus: null }]);
88
-
89
- assert.throws(
90
- () => applyMigration(sqlite, SCHEMA_CHANGE),
91
- /NOT NULL constraint failed/,
92
- );
93
- sqlite.close();
94
- });
95
-
96
- test("the rebuild keeps the account index and the new column", () => {
97
- const sqlite = atPredecessor();
98
- applyMigration(sqlite, BACKFILL);
99
- applyMigration(sqlite, SCHEMA_CHANGE);
100
- applyMigration(sqlite, "0028_mailbox_pending_path");
101
-
102
- const columns = (
103
- sqlite.prepare("PRAGMA table_info(mailbox)").all() as Array<{
104
- name: string;
105
- }>
106
- ).map((column) => column.name);
107
- assert.equal(columns.includes("pending_path"), true);
108
- assert.equal(columns.includes("old_path"), false);
109
-
110
- const indexes = (
111
- sqlite.prepare("PRAGMA index_list(mailbox)").all() as Array<{
112
- name: string;
113
- }>
114
- ).map((index) => index.name);
115
- assert.equal(indexes.includes("mailbox_by_account_id"), true);
116
- sqlite.close();
117
- });
118
- });
@@ -1,83 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { describe, test } from "node:test";
3
- import Database from "better-sqlite3";
4
- import {
5
- applyMigration,
6
- migrationJournal,
7
- } from "./test-shipped-sqlite-schema.js";
8
-
9
- /**
10
- * The data migration that clears the `message.created` backlog (reader#1063),
11
- * against a database that already holds those rows.
12
- *
13
- * Every other outbox test starts from an empty table, so the case that matters
14
- * — an instance upgrading with 30,906 undrained rows the drain filters out and
15
- * the column's type no longer admits — is only reachable by running the shipped
16
- * migrations over a table populated at the point the upgrade finds it.
17
- */
18
-
19
- const CLEANUP_TAG = "0022_drop_message_created_outbox";
20
-
21
- const seed = (sqlite: Database.Database, event: string, id: string): void => {
22
- sqlite
23
- .prepare(
24
- `INSERT INTO outbox (id, message_id, event, payload, created_at)
25
- VALUES (?, ?, ?, ?, ?)`,
26
- )
27
- .run(id, `message-${id}`, event, JSON.stringify({ messageId: id }), 1);
28
- };
29
-
30
- const unprocessed = (sqlite: Database.Database): string[] =>
31
- (
32
- sqlite
33
- .prepare(
34
- "SELECT event FROM outbox WHERE processed_at IS NULL ORDER BY event",
35
- )
36
- .all() as Array<{ event: string }>
37
- ).map((row) => row.event);
38
-
39
- const upgradedDatabase = (): Database.Database => {
40
- const entries = [...migrationJournal()].sort(
41
- (left, right) => left.idx - right.idx,
42
- );
43
- const cleanup = entries.findIndex((entry) => entry.tag === CLEANUP_TAG);
44
- assert.notEqual(cleanup, -1, `${CLEANUP_TAG} is not in the journal`);
45
-
46
- const sqlite = new Database(":memory:");
47
- for (const entry of entries.slice(0, cleanup)) {
48
- applyMigration(sqlite, entry.tag);
49
- }
50
-
51
- seed(sqlite, "message.created", "created-1");
52
- seed(sqlite, "message.created", "created-2");
53
- seed(sqlite, "message.body_synced", "synced-1");
54
- seed(sqlite, "message.moved", "moved-1");
55
-
56
- for (const entry of entries.slice(cleanup)) {
57
- applyMigration(sqlite, entry.tag);
58
- }
59
- return sqlite;
60
- };
61
-
62
- describe("the message.created outbox cleanup", () => {
63
- test("clears the rows an upgrading instance already has", () => {
64
- const sqlite = upgradedDatabase();
65
-
66
- const left = sqlite
67
- .prepare("SELECT COUNT(*) AS n FROM outbox WHERE event = ?")
68
- .get("message.created") as { n: number };
69
-
70
- assert.equal(left.n, 0);
71
- sqlite.close();
72
- });
73
-
74
- test("leaves the drained kinds alone", () => {
75
- const sqlite = upgradedDatabase();
76
-
77
- assert.deepEqual(unprocessed(sqlite), [
78
- "message.body_synced",
79
- "message.moved",
80
- ]);
81
- sqlite.close();
82
- });
83
- });