@remit/drizzle-service 0.0.87 → 0.0.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-granted-scopes-migration.sqlite.test.ts +66 -0
- package/src/account-synced-services-migration.sqlite.test.ts +8 -1
- package/src/mailbox-path-dedupe.sqlite.test.ts +436 -0
- package/src/repair/duplicate-mailbox-mail.ts +98 -0
- package/src/repos/i4-account.test.ts +20 -17
- package/src/repos/i4-account.ts +4 -0
- package/src/repos/i4-mailbox-special-use.sqlite.test.ts +67 -38
- package/src/repos/i4-mailbox.sqlite.test.ts +84 -21
- package/src/repos/i4-mailbox.test.ts +92 -39
- package/src/repos/i4-mailbox.ts +14 -3
- package/src/repos/test-helpers.ts +8 -0
package/package.json
CHANGED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, test } from "node:test";
|
|
3
|
+
import Database from "better-sqlite3";
|
|
4
|
+
import { drizzle } from "drizzle-orm/better-sqlite3";
|
|
5
|
+
import { AccountRepo } from "./repos/i4-account.js";
|
|
6
|
+
import { accountTable } from "./schema.js";
|
|
7
|
+
import {
|
|
8
|
+
applyMigration,
|
|
9
|
+
migrationJournal,
|
|
10
|
+
} from "./test-shipped-sqlite-schema.js";
|
|
11
|
+
|
|
12
|
+
const MIGRATION = "0032_account_granted_scopes";
|
|
13
|
+
|
|
14
|
+
const atPredecessor = (): Database.Database => {
|
|
15
|
+
const sqlite = new Database(":memory:");
|
|
16
|
+
for (const entry of migrationJournal()) {
|
|
17
|
+
if (entry.tag === MIGRATION) return sqlite;
|
|
18
|
+
applyMigration(sqlite, entry.tag);
|
|
19
|
+
}
|
|
20
|
+
throw new Error(`${MIGRATION} is not in the journal`);
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
const migrateOnward = (sqlite: Database.Database): void => {
|
|
24
|
+
const tags = migrationJournal().map((entry) => entry.tag);
|
|
25
|
+
for (const tag of tags.slice(tags.indexOf(MIGRATION))) {
|
|
26
|
+
applyMigration(sqlite, tag);
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
const insertAccount = (
|
|
31
|
+
sqlite: Database.Database,
|
|
32
|
+
accountId: string,
|
|
33
|
+
authType: string,
|
|
34
|
+
): void => {
|
|
35
|
+
sqlite
|
|
36
|
+
.prepare(
|
|
37
|
+
`INSERT INTO account (
|
|
38
|
+
account_id, account_config_id, username, email, auth_type,
|
|
39
|
+
imap_host, imap_port, imap_tls, imap_start_tls, smtp_port,
|
|
40
|
+
is_active, connection_state, created_at, updated_at
|
|
41
|
+
) VALUES (?, 'config', 'person', 'person@example.com', ?,
|
|
42
|
+
'outlook.office365.com', 993, 1, 0, 587, 1,
|
|
43
|
+
'not_authenticated', 0, 0)`,
|
|
44
|
+
)
|
|
45
|
+
.run(accountId, authType);
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
describe("an account written before granted scopes were recorded", () => {
|
|
49
|
+
test("a Microsoft account holds the mail grant its consent always asked for", async () => {
|
|
50
|
+
const sqlite = atPredecessor();
|
|
51
|
+
insertAccount(sqlite, "microsoft", "oauthMicrosoft");
|
|
52
|
+
insertAccount(sqlite, "password", "password");
|
|
53
|
+
|
|
54
|
+
migrateOnward(sqlite);
|
|
55
|
+
|
|
56
|
+
const repo = new AccountRepo(
|
|
57
|
+
drizzle(sqlite, { schema: { accounts: accountTable } }) as never,
|
|
58
|
+
);
|
|
59
|
+
assert.deepEqual((await repo.get("microsoft")).grantedScopes, [
|
|
60
|
+
"https://outlook.office.com/IMAP.AccessAsUser.All",
|
|
61
|
+
"https://outlook.office.com/SMTP.Send",
|
|
62
|
+
]);
|
|
63
|
+
assert.deepEqual((await repo.get("password")).grantedScopes, []);
|
|
64
|
+
sqlite.close();
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -21,6 +21,13 @@ const atPredecessor = (): Database.Database => {
|
|
|
21
21
|
throw new Error(`${MIGRATION} is not in the journal`);
|
|
22
22
|
};
|
|
23
23
|
|
|
24
|
+
const migrateOnward = (sqlite: Database.Database): void => {
|
|
25
|
+
const tags = migrationJournal().map((entry) => entry.tag);
|
|
26
|
+
for (const tag of tags.slice(tags.indexOf(MIGRATION))) {
|
|
27
|
+
applyMigration(sqlite, tag);
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
|
|
24
31
|
describe("an account written before the service selection existed", () => {
|
|
25
32
|
test("comes out of the migration syncing mail only", async () => {
|
|
26
33
|
const sqlite = atPredecessor();
|
|
@@ -36,7 +43,7 @@ describe("an account written before the service selection existed", () => {
|
|
|
36
43
|
)
|
|
37
44
|
.run();
|
|
38
45
|
|
|
39
|
-
|
|
46
|
+
migrateOnward(sqlite);
|
|
40
47
|
|
|
41
48
|
const repo = new AccountRepo(
|
|
42
49
|
drizzle(sqlite, { schema: { accounts: accountTable } }) as never,
|
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, test } from "node:test";
|
|
3
|
+
import * as entities from "@remit/drizzle-sqlite-schema";
|
|
4
|
+
import Database from "better-sqlite3";
|
|
5
|
+
import { asc, eq } from "drizzle-orm";
|
|
6
|
+
import { drizzle } from "drizzle-orm/better-sqlite3";
|
|
7
|
+
import type { Db } from "./db.js";
|
|
8
|
+
import {
|
|
9
|
+
PARKED_MAILBOX_ID,
|
|
10
|
+
sweepDuplicateMailboxMail,
|
|
11
|
+
} from "./repair/duplicate-mailbox-mail.js";
|
|
12
|
+
import { MESSAGE_REMOVED_EVENT } from "./repos/message.js";
|
|
13
|
+
import { outboxTable } from "./schema/outbox.js";
|
|
14
|
+
import {
|
|
15
|
+
applyMigration,
|
|
16
|
+
migrationJournal,
|
|
17
|
+
} from "./test-shipped-sqlite-schema.js";
|
|
18
|
+
|
|
19
|
+
const DEDUPE = "0030_mailbox_path_dedupe";
|
|
20
|
+
const UNIQUE = "0031_mailbox_path_unique";
|
|
21
|
+
const ACCOUNT = "acct";
|
|
22
|
+
const CONFIG = "cfg";
|
|
23
|
+
|
|
24
|
+
type Handle = { sqlite: Database.Database; db: Db<Record<string, unknown>> };
|
|
25
|
+
|
|
26
|
+
const atPredecessor = (): Handle => {
|
|
27
|
+
const sqlite = new Database(":memory:");
|
|
28
|
+
for (const entry of migrationJournal()) {
|
|
29
|
+
if (entry.tag === DEDUPE) {
|
|
30
|
+
return {
|
|
31
|
+
sqlite,
|
|
32
|
+
db: drizzle(sqlite) as unknown as Db<Record<string, unknown>>,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
applyMigration(sqlite, entry.tag);
|
|
36
|
+
}
|
|
37
|
+
throw new Error(`${DEDUPE} is not in the journal`);
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const seedMailbox = async (
|
|
41
|
+
{ db }: Handle,
|
|
42
|
+
mailboxId: string,
|
|
43
|
+
fullPath: string,
|
|
44
|
+
createdAt: number,
|
|
45
|
+
over: Partial<typeof entities.mailboxes.$inferInsert> = {},
|
|
46
|
+
): Promise<void> => {
|
|
47
|
+
await db.insert(entities.mailboxes).values({
|
|
48
|
+
mailboxId,
|
|
49
|
+
accountId: ACCOUNT,
|
|
50
|
+
namespacePrefix: "",
|
|
51
|
+
hierarchyDelimiter: "/",
|
|
52
|
+
fullPath,
|
|
53
|
+
uidValidity: 1,
|
|
54
|
+
uidNext: 1,
|
|
55
|
+
highestModseq: "0",
|
|
56
|
+
messageCount: 0,
|
|
57
|
+
unseenCount: 0,
|
|
58
|
+
deletedCount: 0,
|
|
59
|
+
totalSize: 0,
|
|
60
|
+
lastSyncUid: 0,
|
|
61
|
+
highWaterMarkUid: 0,
|
|
62
|
+
lastMessageSyncAt: 0,
|
|
63
|
+
createdAt,
|
|
64
|
+
updatedAt: createdAt,
|
|
65
|
+
...over,
|
|
66
|
+
});
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const seedMessage = async (
|
|
70
|
+
{ db }: Handle,
|
|
71
|
+
messageId: string,
|
|
72
|
+
mailboxId: string,
|
|
73
|
+
uid = 1,
|
|
74
|
+
): Promise<void> => {
|
|
75
|
+
await db.insert(entities.messages).values({
|
|
76
|
+
messageId,
|
|
77
|
+
mailboxId,
|
|
78
|
+
uid,
|
|
79
|
+
sequenceNumber: uid,
|
|
80
|
+
rfc822Size: 1,
|
|
81
|
+
internalDate: 0,
|
|
82
|
+
envelopeId: `env-${messageId}`,
|
|
83
|
+
rootBodyPartId: `bp-${messageId}`,
|
|
84
|
+
createdAt: 0,
|
|
85
|
+
updatedAt: 0,
|
|
86
|
+
});
|
|
87
|
+
await db.insert(entities.threadMessages).values({
|
|
88
|
+
threadMessageId: `tm-${messageId}`,
|
|
89
|
+
threadId: `thread-${messageId}`,
|
|
90
|
+
messageId,
|
|
91
|
+
accountConfigId: CONFIG,
|
|
92
|
+
mailboxId,
|
|
93
|
+
uid,
|
|
94
|
+
referenceOrder: 0,
|
|
95
|
+
internalDate: 0,
|
|
96
|
+
sentDate: 0,
|
|
97
|
+
isRead: false,
|
|
98
|
+
hasAttachment: false,
|
|
99
|
+
hasStars: false,
|
|
100
|
+
isDeleted: false,
|
|
101
|
+
createdAt: 0,
|
|
102
|
+
updatedAt: 0,
|
|
103
|
+
});
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
const seedSetting = async (
|
|
107
|
+
{ db }: Handle,
|
|
108
|
+
name: string,
|
|
109
|
+
value: typeof entities.accountSettings.$inferInsert.value,
|
|
110
|
+
): Promise<void> => {
|
|
111
|
+
await db.insert(entities.accountSettings).values({
|
|
112
|
+
accountSettingId: name,
|
|
113
|
+
accountConfigId: CONFIG,
|
|
114
|
+
name,
|
|
115
|
+
value,
|
|
116
|
+
createdAt: 0,
|
|
117
|
+
updatedAt: 0,
|
|
118
|
+
});
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
const dedupe = (handle: Handle): void => {
|
|
122
|
+
applyMigration(handle.sqlite, DEDUPE);
|
|
123
|
+
applyMigration(handle.sqlite, UNIQUE);
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
const mailboxIds = async ({ db }: Handle): Promise<string[]> =>
|
|
127
|
+
(
|
|
128
|
+
await db
|
|
129
|
+
.select({ mailboxId: entities.mailboxes.mailboxId })
|
|
130
|
+
.from(entities.mailboxes)
|
|
131
|
+
.orderBy(asc(entities.mailboxes.mailboxId))
|
|
132
|
+
).map((row) => row.mailboxId);
|
|
133
|
+
|
|
134
|
+
const messagesIn = async ({ db }: Handle, mailboxId: string): Promise<number> =>
|
|
135
|
+
(
|
|
136
|
+
await db
|
|
137
|
+
.select({ messageId: entities.messages.messageId })
|
|
138
|
+
.from(entities.messages)
|
|
139
|
+
.where(eq(entities.messages.mailboxId, mailboxId))
|
|
140
|
+
).length;
|
|
141
|
+
|
|
142
|
+
const threadsIn = async ({ db }: Handle, mailboxId: string): Promise<number> =>
|
|
143
|
+
(
|
|
144
|
+
await db
|
|
145
|
+
.select({ messageId: entities.threadMessages.messageId })
|
|
146
|
+
.from(entities.threadMessages)
|
|
147
|
+
.where(eq(entities.threadMessages.mailboxId, mailboxId))
|
|
148
|
+
).length;
|
|
149
|
+
|
|
150
|
+
const survivorCursor = async ({ db }: Handle, mailboxId: string) => {
|
|
151
|
+
const [row] = await db
|
|
152
|
+
.select({
|
|
153
|
+
lastSyncUid: entities.mailboxes.lastSyncUid,
|
|
154
|
+
highWaterMarkUid: entities.mailboxes.highWaterMarkUid,
|
|
155
|
+
highestModseq: entities.mailboxes.highestModseq,
|
|
156
|
+
})
|
|
157
|
+
.from(entities.mailboxes)
|
|
158
|
+
.where(eq(entities.mailboxes.mailboxId, mailboxId));
|
|
159
|
+
return row;
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
describe("(account_id, full_path) becomes unique", () => {
|
|
163
|
+
test("the loser's mail moves onto the survivor, so none of it is lost", async () => {
|
|
164
|
+
const handle = atPredecessor();
|
|
165
|
+
const cursor = {
|
|
166
|
+
lastSyncUid: 1,
|
|
167
|
+
highWaterMarkUid: 105,
|
|
168
|
+
highestModseq: "900",
|
|
169
|
+
};
|
|
170
|
+
await seedMailbox(handle, "a", "Archive", 1, cursor);
|
|
171
|
+
await seedMailbox(handle, "b", "Archive", 2, cursor);
|
|
172
|
+
for (let uid = 1; uid <= 100; uid++) {
|
|
173
|
+
await seedMessage(handle, `a-${uid}`, "a", uid);
|
|
174
|
+
}
|
|
175
|
+
for (let uid = 101; uid <= 105; uid++) {
|
|
176
|
+
await seedMessage(handle, `b-${uid}`, "b", uid);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
dedupe(handle);
|
|
180
|
+
|
|
181
|
+
assert.deepEqual(await mailboxIds(handle), ["a"]);
|
|
182
|
+
assert.equal(await messagesIn(handle, "a"), 105);
|
|
183
|
+
assert.equal(await threadsIn(handle, "a"), 105);
|
|
184
|
+
assert.equal(await messagesIn(handle, PARKED_MAILBOX_ID), 0);
|
|
185
|
+
assert.deepEqual(await survivorCursor(handle, "a"), cursor);
|
|
186
|
+
handle.sqlite.close();
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
test("what named the loser follows the survivor", async () => {
|
|
190
|
+
const handle = atPredecessor();
|
|
191
|
+
await seedMailbox(handle, "older-empty", "INBOX/Notifications", 1);
|
|
192
|
+
await seedMailbox(handle, "with-mail", "INBOX/Notifications", 2);
|
|
193
|
+
await seedMailbox(handle, "child", "INBOX/Notifications/Old", 3);
|
|
194
|
+
await seedMailbox(handle, "elsewhere", "Receipts", 4);
|
|
195
|
+
await seedMessage(handle, "m1", "with-mail");
|
|
196
|
+
await seedMessage(handle, "copied", "elsewhere");
|
|
197
|
+
await handle.db
|
|
198
|
+
.update(entities.messages)
|
|
199
|
+
.set({ originalMailboxId: "older-empty" })
|
|
200
|
+
.where(eq(entities.messages.messageId, "copied"));
|
|
201
|
+
await handle.db
|
|
202
|
+
.update(entities.mailboxes)
|
|
203
|
+
.set({ parentMailboxId: "older-empty" })
|
|
204
|
+
.where(eq(entities.mailboxes.mailboxId, "child"));
|
|
205
|
+
await handle.db.insert(entities.filters).values({
|
|
206
|
+
filterId: "f1",
|
|
207
|
+
accountConfigId: CONFIG,
|
|
208
|
+
name: "Alerts",
|
|
209
|
+
scope: "Standing",
|
|
210
|
+
ruleChangedAt: 0,
|
|
211
|
+
actionChangedAt: 0,
|
|
212
|
+
actionMailboxId: "older-empty",
|
|
213
|
+
createdAt: 0,
|
|
214
|
+
updatedAt: 0,
|
|
215
|
+
});
|
|
216
|
+
await handle.db.insert(entities.organizeJobRequests).values({
|
|
217
|
+
organizeJobId: "job-1",
|
|
218
|
+
accountConfigId: CONFIG,
|
|
219
|
+
userId: "user",
|
|
220
|
+
actionMailboxId: "older-empty",
|
|
221
|
+
ttl: 0,
|
|
222
|
+
createdAt: 0,
|
|
223
|
+
updatedAt: 0,
|
|
224
|
+
});
|
|
225
|
+
await handle.db.insert(entities.messagePlacementMoves).values({
|
|
226
|
+
messageId: "copied",
|
|
227
|
+
accountId: ACCOUNT,
|
|
228
|
+
accountConfigId: CONFIG,
|
|
229
|
+
sourceMailboxId: "elsewhere",
|
|
230
|
+
destinationMailboxId: "older-empty",
|
|
231
|
+
createdAt: 0,
|
|
232
|
+
updatedAt: 0,
|
|
233
|
+
});
|
|
234
|
+
await seedSetting(handle, "FolderRoleAppointment#acct#Archive", {
|
|
235
|
+
kind: "String",
|
|
236
|
+
value: "older-empty",
|
|
237
|
+
});
|
|
238
|
+
await handle.db.insert(entities.mailboxFlags).values({
|
|
239
|
+
mailboxFlagId: "flag-1",
|
|
240
|
+
mailboxId: "older-empty",
|
|
241
|
+
flagName: "\\Seen",
|
|
242
|
+
isPermanent: true,
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
dedupe(handle);
|
|
246
|
+
|
|
247
|
+
assert.deepEqual(await mailboxIds(handle), [
|
|
248
|
+
"child",
|
|
249
|
+
"elsewhere",
|
|
250
|
+
"with-mail",
|
|
251
|
+
]);
|
|
252
|
+
const [child] = await handle.db
|
|
253
|
+
.select({ parent: entities.mailboxes.parentMailboxId })
|
|
254
|
+
.from(entities.mailboxes)
|
|
255
|
+
.where(eq(entities.mailboxes.mailboxId, "child"));
|
|
256
|
+
assert.equal(child?.parent, "with-mail");
|
|
257
|
+
const [filter] = await handle.db.select().from(entities.filters);
|
|
258
|
+
assert.equal(filter?.actionMailboxId, "with-mail");
|
|
259
|
+
const [job] = await handle.db.select().from(entities.organizeJobRequests);
|
|
260
|
+
assert.equal(job?.actionMailboxId, "with-mail");
|
|
261
|
+
const [copied] = await handle.db
|
|
262
|
+
.select({ original: entities.messages.originalMailboxId })
|
|
263
|
+
.from(entities.messages)
|
|
264
|
+
.where(eq(entities.messages.messageId, "copied"));
|
|
265
|
+
assert.equal(copied?.original, "with-mail");
|
|
266
|
+
const [move] = await handle.db
|
|
267
|
+
.select()
|
|
268
|
+
.from(entities.messagePlacementMoves);
|
|
269
|
+
assert.equal(move?.destinationMailboxId, "with-mail");
|
|
270
|
+
const [setting] = await handle.db.select().from(entities.accountSettings);
|
|
271
|
+
assert.deepEqual(setting?.value, { kind: "String", value: "with-mail" });
|
|
272
|
+
assert.deepEqual(await handle.db.select().from(entities.mailboxFlags), []);
|
|
273
|
+
const index = (
|
|
274
|
+
handle.sqlite.prepare("PRAGMA index_list(mailbox)").all() as Array<{
|
|
275
|
+
name: string;
|
|
276
|
+
unique: number;
|
|
277
|
+
}>
|
|
278
|
+
).find((entry) => entry.name === "mailbox_by_account_full_path");
|
|
279
|
+
assert.equal(index?.unique, 1);
|
|
280
|
+
handle.sqlite.close();
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
test("a muted losing folder stays muted, and the survivor's own name is kept", async () => {
|
|
284
|
+
const handle = atPredecessor();
|
|
285
|
+
await seedMailbox(handle, "survivor", "Newsletters", 1);
|
|
286
|
+
await seedMailbox(handle, "loser", "Newsletters", 2);
|
|
287
|
+
await seedSetting(handle, "MailboxMuted#loser", {
|
|
288
|
+
kind: "Muted",
|
|
289
|
+
value: true,
|
|
290
|
+
});
|
|
291
|
+
await seedSetting(handle, "MailboxDisplayName#loser", {
|
|
292
|
+
kind: "String",
|
|
293
|
+
value: "Loser name",
|
|
294
|
+
});
|
|
295
|
+
await seedSetting(handle, "MailboxDisplayName#survivor", {
|
|
296
|
+
kind: "String",
|
|
297
|
+
value: "Survivor name",
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
dedupe(handle);
|
|
301
|
+
|
|
302
|
+
const settings = await handle.db
|
|
303
|
+
.select({
|
|
304
|
+
name: entities.accountSettings.name,
|
|
305
|
+
value: entities.accountSettings.value,
|
|
306
|
+
})
|
|
307
|
+
.from(entities.accountSettings)
|
|
308
|
+
.orderBy(asc(entities.accountSettings.name));
|
|
309
|
+
assert.deepEqual(settings, [
|
|
310
|
+
{
|
|
311
|
+
name: "MailboxDisplayName#survivor",
|
|
312
|
+
value: { kind: "String", value: "Survivor name" },
|
|
313
|
+
},
|
|
314
|
+
{ name: "MailboxMuted#survivor", value: { kind: "Muted", value: true } },
|
|
315
|
+
]);
|
|
316
|
+
handle.sqlite.close();
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
test("the survivor is chosen by the mail it holds, not by message_count", async () => {
|
|
320
|
+
const handle = atPredecessor();
|
|
321
|
+
await seedMailbox(handle, "claims-mail", "Archive", 1, {
|
|
322
|
+
messageCount: 500,
|
|
323
|
+
});
|
|
324
|
+
await seedMailbox(handle, "holds-mail", "Archive", 2);
|
|
325
|
+
await seedMessage(handle, "m1", "holds-mail", 1);
|
|
326
|
+
await seedMessage(handle, "m2", "holds-mail", 2);
|
|
327
|
+
|
|
328
|
+
dedupe(handle);
|
|
329
|
+
|
|
330
|
+
assert.deepEqual(await mailboxIds(handle), ["holds-mail"]);
|
|
331
|
+
handle.sqlite.close();
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
test("with equal mail the oldest row survives", async () => {
|
|
335
|
+
const handle = atPredecessor();
|
|
336
|
+
await seedMailbox(handle, "a-newer", "Archive", 2);
|
|
337
|
+
await seedMailbox(handle, "b-older", "Archive", 1);
|
|
338
|
+
|
|
339
|
+
dedupe(handle);
|
|
340
|
+
|
|
341
|
+
assert.deepEqual(await mailboxIds(handle), ["b-older"]);
|
|
342
|
+
handle.sqlite.close();
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
test("with equal mail and age the lowest mailbox_id survives", async () => {
|
|
346
|
+
const handle = atPredecessor();
|
|
347
|
+
await seedMailbox(handle, "b", "Archive", 1);
|
|
348
|
+
await seedMailbox(handle, "a", "Archive", 1);
|
|
349
|
+
|
|
350
|
+
dedupe(handle);
|
|
351
|
+
|
|
352
|
+
assert.deepEqual(await mailboxIds(handle), ["a"]);
|
|
353
|
+
handle.sqlite.close();
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
test("mail the survivor already holds at that uid is parked, removed at boot, and re-fetched", async () => {
|
|
357
|
+
const handle = atPredecessor();
|
|
358
|
+
const cursor = {
|
|
359
|
+
lastSyncUid: 1,
|
|
360
|
+
highWaterMarkUid: 9,
|
|
361
|
+
highestModseq: "900",
|
|
362
|
+
};
|
|
363
|
+
await seedMailbox(handle, "older", "Archive", 1, cursor);
|
|
364
|
+
await seedMailbox(handle, "newer", "Archive", 2, cursor);
|
|
365
|
+
await seedMessage(handle, "kept", "older", 7);
|
|
366
|
+
await seedMessage(handle, "stale", "newer", 7);
|
|
367
|
+
await seedMessage(handle, "unrelated", "no-such-folder", 1);
|
|
368
|
+
|
|
369
|
+
dedupe(handle);
|
|
370
|
+
|
|
371
|
+
assert.deepEqual(await mailboxIds(handle), ["older"]);
|
|
372
|
+
assert.equal(await messagesIn(handle, PARKED_MAILBOX_ID), 1);
|
|
373
|
+
assert.equal(await threadsIn(handle, PARKED_MAILBOX_ID), 1);
|
|
374
|
+
assert.deepEqual(await survivorCursor(handle, "older"), {
|
|
375
|
+
lastSyncUid: 0,
|
|
376
|
+
highWaterMarkUid: 0,
|
|
377
|
+
highestModseq: "0",
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
const report = await sweepDuplicateMailboxMail(handle.db, "repair");
|
|
381
|
+
|
|
382
|
+
assert.deepEqual(report, {
|
|
383
|
+
mode: "repair",
|
|
384
|
+
duplicatePaths: 0,
|
|
385
|
+
parked: 1,
|
|
386
|
+
removed: 1,
|
|
387
|
+
});
|
|
388
|
+
const messages = await handle.db
|
|
389
|
+
.select({ messageId: entities.messages.messageId })
|
|
390
|
+
.from(entities.messages)
|
|
391
|
+
.orderBy(asc(entities.messages.messageId));
|
|
392
|
+
assert.deepEqual(messages, [
|
|
393
|
+
{ messageId: "kept" },
|
|
394
|
+
{ messageId: "unrelated" },
|
|
395
|
+
]);
|
|
396
|
+
const events = await handle.db
|
|
397
|
+
.select({ messageId: outboxTable.messageId, event: outboxTable.event })
|
|
398
|
+
.from(outboxTable);
|
|
399
|
+
assert.deepEqual(events, [
|
|
400
|
+
{ messageId: "stale", event: MESSAGE_REMOVED_EVENT },
|
|
401
|
+
]);
|
|
402
|
+
assert.deepEqual(await sweepDuplicateMailboxMail(handle.db, "repair"), {
|
|
403
|
+
mode: "repair",
|
|
404
|
+
duplicatePaths: 0,
|
|
405
|
+
parked: 0,
|
|
406
|
+
removed: 0,
|
|
407
|
+
});
|
|
408
|
+
handle.sqlite.close();
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
test("the check before migrating reports the paths the migration will merge", async () => {
|
|
412
|
+
const handle = atPredecessor();
|
|
413
|
+
await seedMailbox(handle, "a", "Archive", 1);
|
|
414
|
+
await seedMailbox(handle, "b", "Archive", 2);
|
|
415
|
+
|
|
416
|
+
assert.deepEqual(await sweepDuplicateMailboxMail(handle.db, "check"), {
|
|
417
|
+
mode: "check",
|
|
418
|
+
duplicatePaths: 1,
|
|
419
|
+
parked: 0,
|
|
420
|
+
removed: 0,
|
|
421
|
+
});
|
|
422
|
+
handle.sqlite.close();
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
test("the unique index is refused without the dedupe ahead of it", async () => {
|
|
426
|
+
const handle = atPredecessor();
|
|
427
|
+
await seedMailbox(handle, "a", "Archive", 1);
|
|
428
|
+
await seedMailbox(handle, "b", "Archive", 2);
|
|
429
|
+
|
|
430
|
+
assert.throws(
|
|
431
|
+
() => applyMigration(handle.sqlite, UNIQUE),
|
|
432
|
+
/UNIQUE constraint failed/,
|
|
433
|
+
);
|
|
434
|
+
handle.sqlite.close();
|
|
435
|
+
});
|
|
436
|
+
});
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { count, eq, gt, inArray } from "drizzle-orm";
|
|
2
|
+
import type { Db } from "../db.js";
|
|
3
|
+
import { deleteMessageSubtree } from "../repos/message.js";
|
|
4
|
+
import { mailboxTable } from "../schema/i4-mailbox.js";
|
|
5
|
+
import { messageTable } from "../schema/message-data.js";
|
|
6
|
+
import { threadMessageTable } from "../schema/thread-message.js";
|
|
7
|
+
import { messageLabelTable } from "../schema.js";
|
|
8
|
+
import { runInTransaction } from "../tx.js";
|
|
9
|
+
|
|
10
|
+
export const PARKED_MAILBOX_ID = "DuplicateMailboxRemoved";
|
|
11
|
+
|
|
12
|
+
type DuplicateMailDb = Db<Record<string, unknown>>;
|
|
13
|
+
|
|
14
|
+
export type DuplicateMailRepairMode = "check" | "repair";
|
|
15
|
+
|
|
16
|
+
export type DuplicateMailReport = {
|
|
17
|
+
readonly mode: DuplicateMailRepairMode;
|
|
18
|
+
readonly duplicatePaths: number;
|
|
19
|
+
readonly parked: number;
|
|
20
|
+
readonly removed: number;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
const BATCH_SIZE = 100;
|
|
24
|
+
|
|
25
|
+
const countDuplicatePaths = async (db: DuplicateMailDb): Promise<number> => {
|
|
26
|
+
const groups = await db
|
|
27
|
+
.select({ rows: count() })
|
|
28
|
+
.from(mailboxTable)
|
|
29
|
+
.groupBy(mailboxTable.accountId, mailboxTable.fullPath)
|
|
30
|
+
.having(gt(count(), 1));
|
|
31
|
+
return groups.length;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
const countParked = async (db: DuplicateMailDb): Promise<number> => {
|
|
35
|
+
const [row] = await db
|
|
36
|
+
.select({ parked: count() })
|
|
37
|
+
.from(messageTable)
|
|
38
|
+
.where(eq(messageTable.mailboxId, PARKED_MAILBOX_ID));
|
|
39
|
+
return row?.parked ?? 0;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const nextParkedBatch = async (db: DuplicateMailDb): Promise<string[]> => {
|
|
43
|
+
const rows = await db
|
|
44
|
+
.select({ messageId: messageTable.messageId })
|
|
45
|
+
.from(messageTable)
|
|
46
|
+
.where(eq(messageTable.mailboxId, PARKED_MAILBOX_ID))
|
|
47
|
+
.limit(BATCH_SIZE);
|
|
48
|
+
return rows.map((row) => row.messageId);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
export const sweepDuplicateMailboxMail = async (
|
|
52
|
+
db: DuplicateMailDb,
|
|
53
|
+
mode: DuplicateMailRepairMode,
|
|
54
|
+
): Promise<DuplicateMailReport> => {
|
|
55
|
+
const duplicatePaths = await countDuplicatePaths(db);
|
|
56
|
+
const parked = await countParked(db);
|
|
57
|
+
if (mode === "check" || parked === 0) {
|
|
58
|
+
return { mode, duplicatePaths, parked, removed: 0 };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
let removed = 0;
|
|
62
|
+
for (;;) {
|
|
63
|
+
const messageIds = await nextParkedBatch(db);
|
|
64
|
+
if (messageIds.length === 0) break;
|
|
65
|
+
await runInTransaction(db, async (tx) => {
|
|
66
|
+
await deleteMessageSubtree(tx, messageIds);
|
|
67
|
+
await tx
|
|
68
|
+
.delete(threadMessageTable)
|
|
69
|
+
.where(inArray(threadMessageTable.messageId, messageIds));
|
|
70
|
+
await tx
|
|
71
|
+
.delete(messageLabelTable)
|
|
72
|
+
.where(inArray(messageLabelTable.messageId, messageIds));
|
|
73
|
+
});
|
|
74
|
+
removed += messageIds.length;
|
|
75
|
+
}
|
|
76
|
+
return { mode, duplicatePaths, parked, removed };
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
export const formatDuplicateMailReport = (
|
|
80
|
+
report: DuplicateMailReport,
|
|
81
|
+
): string[] => {
|
|
82
|
+
const lines: string[] = [];
|
|
83
|
+
if (report.duplicatePaths > 0) {
|
|
84
|
+
lines.push(
|
|
85
|
+
`${report.duplicatePaths} folder path(s) held by more than one row, merged by the pending migration`,
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
if (report.parked === 0) {
|
|
89
|
+
lines.push("No message is parked by the folder dedupe");
|
|
90
|
+
return lines;
|
|
91
|
+
}
|
|
92
|
+
lines.push(
|
|
93
|
+
report.mode === "check"
|
|
94
|
+
? `${report.parked} message(s) parked by the folder dedupe, would be removed`
|
|
95
|
+
: `${report.removed} of ${report.parked} message(s) parked by the folder dedupe removed`,
|
|
96
|
+
);
|
|
97
|
+
return lines;
|
|
98
|
+
};
|
|
@@ -3,6 +3,7 @@ import { after, before, describe, test } from "node:test";
|
|
|
3
3
|
import { createTestDb, randomId, type TestDb } from "../test-db.js";
|
|
4
4
|
import { AccountRepo } from "./i4-account.js";
|
|
5
5
|
import { MailboxRepo } from "./i4-mailbox.js";
|
|
6
|
+
import { mailboxCreated } from "./test-helpers.js";
|
|
6
7
|
|
|
7
8
|
function makeAccountInput(accountConfigId: string) {
|
|
8
9
|
return {
|
|
@@ -160,23 +161,25 @@ describe("AccountRepo", () => {
|
|
|
160
161
|
const account = await repo.create(makeAccountInput(accountConfigId));
|
|
161
162
|
|
|
162
163
|
const mailboxRepo = new MailboxRepo(db as never);
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
164
|
+
mailboxCreated(
|
|
165
|
+
await mailboxRepo.create({
|
|
166
|
+
accountId: account.accountId,
|
|
167
|
+
namespaceType: "personal",
|
|
168
|
+
namespacePrefix: "",
|
|
169
|
+
hierarchyDelimiter: "/",
|
|
170
|
+
fullPath: "INBOX",
|
|
171
|
+
uidValidity: 1,
|
|
172
|
+
uidNext: 1,
|
|
173
|
+
highestModseq: "0",
|
|
174
|
+
messageCount: 0,
|
|
175
|
+
unseenCount: 0,
|
|
176
|
+
deletedCount: 0,
|
|
177
|
+
totalSize: 0,
|
|
178
|
+
lastSyncUid: 0,
|
|
179
|
+
highWaterMarkUid: 0,
|
|
180
|
+
lastMessageSyncAt: Date.now(),
|
|
181
|
+
}),
|
|
182
|
+
);
|
|
180
183
|
|
|
181
184
|
const desc = await repo.describe(account.accountId);
|
|
182
185
|
assert.equal(desc.account.length, 1);
|