@remit/drizzle-service 0.0.49 → 0.0.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/repair/junk-only-address.sqlite.test.ts +535 -0
- package/src/repair/junk-only-address.ts +158 -0
- package/src/repos/address-search-predicates.ts +15 -0
- package/src/repos/i4-address-junk-move.sqlite.test.ts +203 -0
- package/src/repos/i4-address.test.ts +217 -0
- package/src/repos/i4-address.ts +105 -0
- package/src/repos/i4-mailbox-special-use.sqlite.test.ts +146 -5
- package/src/repos/i4-mailbox-special-use.ts +111 -69
package/package.json
CHANGED
|
@@ -0,0 +1,535 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { after, before, beforeEach, describe, test } from "node:test";
|
|
4
|
+
import Database from "better-sqlite3";
|
|
5
|
+
import { shippedTableDdl } from "../test-shipped-sqlite-schema.js";
|
|
6
|
+
import {
|
|
7
|
+
type JunkOnlyRepairClient,
|
|
8
|
+
sweepJunkOnlyAddresses,
|
|
9
|
+
} from "./junk-only-address.js";
|
|
10
|
+
|
|
11
|
+
const DDL_TAG = "0000_happy_roland_deschain";
|
|
12
|
+
|
|
13
|
+
const clientOver = (sqlite: Database.Database): JunkOnlyRepairClient => ({
|
|
14
|
+
all: async (sql, params) => sqlite.prepare(sql).all(...params),
|
|
15
|
+
run: async (sql, params) => sqlite.prepare(sql).run(...params).changes,
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
interface AddressRow {
|
|
19
|
+
flags: string;
|
|
20
|
+
updated_at: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
describe("addresses standing only on mail in Junk", () => {
|
|
24
|
+
let sqlite: Database.Database;
|
|
25
|
+
|
|
26
|
+
const mailbox = (mailboxId: string, specialUse: string | null): void => {
|
|
27
|
+
sqlite
|
|
28
|
+
.prepare(
|
|
29
|
+
`INSERT INTO mailbox (
|
|
30
|
+
mailbox_id, account_id, namespace_prefix, hierarchy_delimiter,
|
|
31
|
+
full_path, uid_validity, uid_next, highest_modseq, message_count,
|
|
32
|
+
unseen_count, deleted_count, total_size, last_sync_uid,
|
|
33
|
+
high_water_mark_uid, last_message_sync_at, special_use,
|
|
34
|
+
created_at, updated_at
|
|
35
|
+
) VALUES (?, 'acc', '', '/', ?, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, ?, 0, 0)`,
|
|
36
|
+
)
|
|
37
|
+
.run(mailboxId, mailboxId, specialUse);
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const specialUseEntry = (mailboxId: string, specialUse: string): void => {
|
|
41
|
+
sqlite
|
|
42
|
+
.prepare(
|
|
43
|
+
`INSERT INTO mailbox_special_use_entry (
|
|
44
|
+
mailbox_special_use_id, mailbox_id, special_use
|
|
45
|
+
) VALUES (?, ?, ?)`,
|
|
46
|
+
)
|
|
47
|
+
.run(`${mailboxId}-${specialUse}`, mailboxId, specialUse);
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
const message = (messageId: string, mailboxId: string): void => {
|
|
51
|
+
sqlite
|
|
52
|
+
.prepare(
|
|
53
|
+
`INSERT INTO message (
|
|
54
|
+
message_id, mailbox_id, uid, sequence_number, rfc822_size,
|
|
55
|
+
internal_date, envelope_id, root_body_part_id, created_at, updated_at
|
|
56
|
+
) VALUES (?, ?, 1, 1, 10, 0, 'env', 'body', 0, 0)`,
|
|
57
|
+
)
|
|
58
|
+
.run(messageId, mailboxId);
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
const address = (
|
|
62
|
+
addressId: string,
|
|
63
|
+
counters: { outbound?: number; reply?: number } = {},
|
|
64
|
+
flags = "{}",
|
|
65
|
+
): void => {
|
|
66
|
+
sqlite
|
|
67
|
+
.prepare(
|
|
68
|
+
`INSERT INTO address (
|
|
69
|
+
address_id, account_config_id, display_name, local_part, domain,
|
|
70
|
+
normalized_email, normalized_compound, flags, inbound_count,
|
|
71
|
+
outbound_count, reply_count, last_inbound_at, last_outbound_at,
|
|
72
|
+
last_reply_at, created_at, updated_at
|
|
73
|
+
) VALUES (?, 'cfg-1', 'Name', ?, 'example.com', ?, ?, ?, 0, ?, ?, 0,
|
|
74
|
+
NULL, 0, 0, 0)`,
|
|
75
|
+
)
|
|
76
|
+
.run(
|
|
77
|
+
addressId,
|
|
78
|
+
addressId,
|
|
79
|
+
`${addressId}@example.com`,
|
|
80
|
+
`name ${addressId}@example.com`,
|
|
81
|
+
flags,
|
|
82
|
+
counters.outbound ?? 0,
|
|
83
|
+
counters.reply ?? 0,
|
|
84
|
+
);
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const sighting = (addressId: string, messageId: string): void => {
|
|
88
|
+
sqlite
|
|
89
|
+
.prepare(
|
|
90
|
+
`INSERT INTO envelope_address (
|
|
91
|
+
envelope_address_id, message_id, address_id, display_name,
|
|
92
|
+
normalized_email, address_role, address_order, created_at, updated_at
|
|
93
|
+
) VALUES (?, ?, ?, 'Name', ?, 'From', 0, 0, 0)`,
|
|
94
|
+
)
|
|
95
|
+
.run(
|
|
96
|
+
`${addressId}-${messageId}`,
|
|
97
|
+
messageId,
|
|
98
|
+
addressId,
|
|
99
|
+
`${addressId}@example.com`,
|
|
100
|
+
);
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
const read = (addressId: string): AddressRow =>
|
|
104
|
+
sqlite
|
|
105
|
+
.prepare("SELECT flags, updated_at FROM address WHERE address_id = ?")
|
|
106
|
+
.get(addressId) as AddressRow;
|
|
107
|
+
|
|
108
|
+
const withheld = (addressId: string): boolean =>
|
|
109
|
+
JSON.parse(read(addressId).flags).junkOnly?.value === true;
|
|
110
|
+
|
|
111
|
+
const addressCount = (): number =>
|
|
112
|
+
(
|
|
113
|
+
sqlite.prepare("SELECT count(*) AS n FROM address").get() as {
|
|
114
|
+
n: number;
|
|
115
|
+
}
|
|
116
|
+
).n;
|
|
117
|
+
|
|
118
|
+
before(() => {
|
|
119
|
+
sqlite = new Database(":memory:");
|
|
120
|
+
for (const table of [
|
|
121
|
+
"address",
|
|
122
|
+
"envelope_address",
|
|
123
|
+
"message",
|
|
124
|
+
"mailbox",
|
|
125
|
+
"mailbox_special_use_entry",
|
|
126
|
+
]) {
|
|
127
|
+
sqlite.exec(shippedTableDdl(DDL_TAG, table));
|
|
128
|
+
}
|
|
129
|
+
sqlite.exec(
|
|
130
|
+
readFileSync(
|
|
131
|
+
new URL(
|
|
132
|
+
"../../../../npm-scripts/sqlite-address-sightings-index.sql",
|
|
133
|
+
import.meta.url,
|
|
134
|
+
),
|
|
135
|
+
"utf8",
|
|
136
|
+
),
|
|
137
|
+
);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
after(() => {
|
|
141
|
+
sqlite.close();
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
beforeEach(() => {
|
|
145
|
+
for (const table of [
|
|
146
|
+
"address",
|
|
147
|
+
"envelope_address",
|
|
148
|
+
"message",
|
|
149
|
+
"mailbox",
|
|
150
|
+
"mailbox_special_use_entry",
|
|
151
|
+
]) {
|
|
152
|
+
sqlite.exec(`DELETE FROM ${table}`);
|
|
153
|
+
}
|
|
154
|
+
mailbox("junk", '["Junk"]');
|
|
155
|
+
specialUseEntry("junk", "Junk");
|
|
156
|
+
mailbox("inbox", null);
|
|
157
|
+
mailbox("trash", '["Trash"]');
|
|
158
|
+
specialUseEntry("trash", "Trash");
|
|
159
|
+
message("spam-1", "junk");
|
|
160
|
+
message("spam-2", "junk");
|
|
161
|
+
message("mail-1", "inbox");
|
|
162
|
+
message("bin-1", "trash");
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
test("withholds an address seen only on mail in Junk", async () => {
|
|
166
|
+
address("spammer");
|
|
167
|
+
sighting("spammer", "spam-1");
|
|
168
|
+
sighting("spammer", "spam-2");
|
|
169
|
+
|
|
170
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
171
|
+
|
|
172
|
+
assert.equal(report.withholdable, 1);
|
|
173
|
+
assert.equal(report.withheld, 1);
|
|
174
|
+
assert.equal(withheld("spammer"), true);
|
|
175
|
+
// `int64` on the wire, so the timestamp must not land as a float.
|
|
176
|
+
assert.match(read("spammer").flags, /"setAt":\d+,/);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
test("keeps an address with one sighting outside Junk", async () => {
|
|
180
|
+
address("colleague");
|
|
181
|
+
sighting("colleague", "spam-1");
|
|
182
|
+
sighting("colleague", "mail-1");
|
|
183
|
+
|
|
184
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
185
|
+
|
|
186
|
+
assert.equal(report.withholdable, 0);
|
|
187
|
+
assert.equal(withheld("colleague"), false);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test("withholds an address whose only message moved into Junk", async () => {
|
|
191
|
+
address("newsletter");
|
|
192
|
+
sighting("newsletter", "mail-1");
|
|
193
|
+
sqlite
|
|
194
|
+
.prepare("UPDATE message SET mailbox_id = 'junk' WHERE message_id = ?")
|
|
195
|
+
.run("mail-1");
|
|
196
|
+
|
|
197
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
198
|
+
|
|
199
|
+
assert.equal(withheld("newsletter"), true);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
test("restores an address whose message moved out of Junk", async () => {
|
|
203
|
+
address("misfiled");
|
|
204
|
+
sighting("misfiled", "spam-1");
|
|
205
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
206
|
+
assert.equal(withheld("misfiled"), true);
|
|
207
|
+
|
|
208
|
+
sqlite
|
|
209
|
+
.prepare("UPDATE message SET mailbox_id = 'inbox' WHERE message_id = ?")
|
|
210
|
+
.run("spam-1");
|
|
211
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
212
|
+
|
|
213
|
+
assert.equal(report.restorable, 1);
|
|
214
|
+
assert.equal(report.restored, 1);
|
|
215
|
+
assert.equal(withheld("misfiled"), false);
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
test("never withholds an address the account has written to", async () => {
|
|
219
|
+
address("client", { outbound: 1 });
|
|
220
|
+
sighting("client", "spam-1");
|
|
221
|
+
|
|
222
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
223
|
+
|
|
224
|
+
assert.equal(report.withholdable, 0);
|
|
225
|
+
assert.equal(withheld("client"), false);
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
test("never withholds an address the account has replied to", async () => {
|
|
229
|
+
address("friend", { reply: 2 });
|
|
230
|
+
sighting("friend", "spam-1");
|
|
231
|
+
|
|
232
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
233
|
+
|
|
234
|
+
assert.equal(withheld("friend"), false);
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
test("never withholds a VIP", async () => {
|
|
238
|
+
address("boss", {}, '{"vip":{"value":true,"setAt":1}}');
|
|
239
|
+
sighting("boss", "spam-1");
|
|
240
|
+
|
|
241
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
242
|
+
|
|
243
|
+
assert.equal(withheld("boss"), false);
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
test("restores a withheld address once the account writes to it", async () => {
|
|
247
|
+
address("reformed", {}, '{"junkOnly":{"value":true,"setAt":1}}');
|
|
248
|
+
sighting("reformed", "spam-1");
|
|
249
|
+
|
|
250
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
251
|
+
assert.equal(withheld("reformed"), true);
|
|
252
|
+
|
|
253
|
+
sqlite
|
|
254
|
+
.prepare("UPDATE address SET outbound_count = 1 WHERE address_id = ?")
|
|
255
|
+
.run("reformed");
|
|
256
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
257
|
+
|
|
258
|
+
assert.equal(withheld("reformed"), false);
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
test("leaves mail in Trash feeding the address book", async () => {
|
|
262
|
+
address("ex-colleague");
|
|
263
|
+
sighting("ex-colleague", "bin-1");
|
|
264
|
+
|
|
265
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
266
|
+
|
|
267
|
+
assert.equal(report.withholdable, 0);
|
|
268
|
+
assert.equal(withheld("ex-colleague"), false);
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
test("one deleted message does not keep a spammer suggestible", async () => {
|
|
272
|
+
address("spammer");
|
|
273
|
+
sighting("spammer", "spam-1");
|
|
274
|
+
sighting("spammer", "bin-1");
|
|
275
|
+
|
|
276
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
277
|
+
|
|
278
|
+
assert.equal(withheld("spammer"), true);
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
test("Trash reads the same whichever move happened first", async () => {
|
|
282
|
+
address("junk-then-bin");
|
|
283
|
+
address("bin-then-junk");
|
|
284
|
+
sighting("junk-then-bin", "spam-1");
|
|
285
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
286
|
+
sighting("junk-then-bin", "bin-1");
|
|
287
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
288
|
+
|
|
289
|
+
sighting("bin-then-junk", "bin-1");
|
|
290
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
291
|
+
sighting("bin-then-junk", "spam-1");
|
|
292
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
293
|
+
|
|
294
|
+
assert.equal(withheld("junk-then-bin"), true);
|
|
295
|
+
assert.equal(withheld("bin-then-junk"), true);
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
test("a cold sweep agrees with the moves that built the state", async () => {
|
|
299
|
+
address("spammer");
|
|
300
|
+
sighting("spammer", "spam-1");
|
|
301
|
+
sighting("spammer", "bin-1");
|
|
302
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
303
|
+
|
|
304
|
+
sqlite
|
|
305
|
+
.prepare("UPDATE address SET flags = '{}' WHERE address_id = ?")
|
|
306
|
+
.run("spammer");
|
|
307
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
308
|
+
|
|
309
|
+
assert.equal(withheld("spammer"), true);
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
test("deleting every spam message leaves the mark standing", async () => {
|
|
313
|
+
address("spammer");
|
|
314
|
+
sighting("spammer", "spam-1");
|
|
315
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
316
|
+
|
|
317
|
+
sqlite
|
|
318
|
+
.prepare("UPDATE message SET mailbox_id = 'trash' WHERE message_id = ?")
|
|
319
|
+
.run("spam-1");
|
|
320
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
321
|
+
|
|
322
|
+
assert.equal(report.restorable, 0);
|
|
323
|
+
assert.equal(withheld("spammer"), true);
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
test("deleting the spam does not put its sender back", async () => {
|
|
327
|
+
address("spammer");
|
|
328
|
+
sighting("spammer", "spam-1");
|
|
329
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
330
|
+
|
|
331
|
+
sqlite
|
|
332
|
+
.prepare("UPDATE message SET mailbox_id = 'trash' WHERE message_id = ?")
|
|
333
|
+
.run("spam-1");
|
|
334
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
335
|
+
|
|
336
|
+
assert.equal(report.restorable, 0);
|
|
337
|
+
assert.equal(withheld("spammer"), true);
|
|
338
|
+
});
|
|
339
|
+
|
|
340
|
+
test("purging the spam does not put its sender back", async () => {
|
|
341
|
+
address("spammer");
|
|
342
|
+
sighting("spammer", "spam-1");
|
|
343
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
344
|
+
|
|
345
|
+
sqlite.prepare("DELETE FROM message WHERE message_id = ?").run("spam-1");
|
|
346
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
347
|
+
|
|
348
|
+
assert.equal(report.restorable, 0);
|
|
349
|
+
assert.equal(withheld("spammer"), true);
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
test("withholds a sender the account blocked or muted", async () => {
|
|
353
|
+
address("reported", {}, '{"blocked":{"value":true,"setAt":1}}');
|
|
354
|
+
address("hushed", {}, '{"muted":{"value":true,"setAt":1}}');
|
|
355
|
+
sighting("reported", "spam-1");
|
|
356
|
+
sighting("hushed", "spam-1");
|
|
357
|
+
|
|
358
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
359
|
+
|
|
360
|
+
assert.equal(withheld("reported"), true);
|
|
361
|
+
assert.equal(withheld("hushed"), true);
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
test("blocking a withheld sender never lifts the mark", async () => {
|
|
365
|
+
address("spammer");
|
|
366
|
+
sighting("spammer", "spam-1");
|
|
367
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
368
|
+
|
|
369
|
+
sqlite
|
|
370
|
+
.prepare(
|
|
371
|
+
`UPDATE address SET flags = json_set(flags, '$.blocked',
|
|
372
|
+
json_object('value', json('true'), 'setAt', 1))
|
|
373
|
+
WHERE address_id = ?`,
|
|
374
|
+
)
|
|
375
|
+
.run("spammer");
|
|
376
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
377
|
+
|
|
378
|
+
assert.equal(report.restorable, 0);
|
|
379
|
+
assert.equal(withheld("spammer"), true);
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
test("reads a Junk folder a server does not designate", async () => {
|
|
383
|
+
mailbox("named-spam", null);
|
|
384
|
+
message("spam-5", "named-spam");
|
|
385
|
+
address("by-name");
|
|
386
|
+
sighting("by-name", "spam-5");
|
|
387
|
+
sqlite
|
|
388
|
+
.prepare("UPDATE mailbox SET full_path = 'Spam' WHERE mailbox_id = ?")
|
|
389
|
+
.run("named-spam");
|
|
390
|
+
|
|
391
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
392
|
+
|
|
393
|
+
assert.equal(withheld("by-name"), true);
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
test("reads a Junk folder nested under any prefix", async () => {
|
|
397
|
+
mailbox("nested-spam", null);
|
|
398
|
+
message("spam-6", "nested-spam");
|
|
399
|
+
address("nested");
|
|
400
|
+
sighting("nested", "spam-6");
|
|
401
|
+
sqlite
|
|
402
|
+
.prepare(
|
|
403
|
+
"UPDATE mailbox SET full_path = 'INBOX/Spam' WHERE mailbox_id = ?",
|
|
404
|
+
)
|
|
405
|
+
.run("nested-spam");
|
|
406
|
+
|
|
407
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
408
|
+
|
|
409
|
+
assert.equal(withheld("nested"), true);
|
|
410
|
+
});
|
|
411
|
+
|
|
412
|
+
test("reads a Junk folder under a delimiter that is not a slash", async () => {
|
|
413
|
+
mailbox("dotted-spam", null);
|
|
414
|
+
message("spam-7", "dotted-spam");
|
|
415
|
+
address("dotted");
|
|
416
|
+
sighting("dotted", "spam-7");
|
|
417
|
+
sqlite
|
|
418
|
+
.prepare(
|
|
419
|
+
`UPDATE mailbox SET full_path = 'Mail.Junk E-mail',
|
|
420
|
+
hierarchy_delimiter = '.' WHERE mailbox_id = ?`,
|
|
421
|
+
)
|
|
422
|
+
.run("dotted-spam");
|
|
423
|
+
|
|
424
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
425
|
+
|
|
426
|
+
assert.equal(withheld("dotted"), true);
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
test("never reads a prefix as the folder it names", async () => {
|
|
430
|
+
mailbox("spam-parent", null);
|
|
431
|
+
message("mail-2", "spam-parent");
|
|
432
|
+
address("under-spam");
|
|
433
|
+
sighting("under-spam", "mail-2");
|
|
434
|
+
sqlite
|
|
435
|
+
.prepare(
|
|
436
|
+
"UPDATE mailbox SET full_path = 'Spam/Receipts' WHERE mailbox_id = ?",
|
|
437
|
+
)
|
|
438
|
+
.run("spam-parent");
|
|
439
|
+
|
|
440
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
441
|
+
|
|
442
|
+
assert.equal(withheld("under-spam"), false);
|
|
443
|
+
});
|
|
444
|
+
|
|
445
|
+
test("reads the designation from either place it is stored", async () => {
|
|
446
|
+
mailbox("column-only", '["Junk"]');
|
|
447
|
+
mailbox("entry-only", null);
|
|
448
|
+
specialUseEntry("entry-only", "Junk");
|
|
449
|
+
message("spam-3", "column-only");
|
|
450
|
+
message("spam-4", "entry-only");
|
|
451
|
+
address("by-column");
|
|
452
|
+
address("by-entry");
|
|
453
|
+
sighting("by-column", "spam-3");
|
|
454
|
+
sighting("by-entry", "spam-4");
|
|
455
|
+
|
|
456
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
457
|
+
|
|
458
|
+
assert.equal(withheld("by-column"), true);
|
|
459
|
+
assert.equal(withheld("by-entry"), true);
|
|
460
|
+
});
|
|
461
|
+
|
|
462
|
+
test("leaves an address no message has ever carried alone", async () => {
|
|
463
|
+
address("orphan");
|
|
464
|
+
|
|
465
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
466
|
+
|
|
467
|
+
assert.equal(report.withholdable, 0);
|
|
468
|
+
assert.equal(withheld("orphan"), false);
|
|
469
|
+
});
|
|
470
|
+
|
|
471
|
+
test("removes no row", async () => {
|
|
472
|
+
address("spammer");
|
|
473
|
+
address("colleague");
|
|
474
|
+
address("client", { outbound: 3 });
|
|
475
|
+
sighting("spammer", "spam-1");
|
|
476
|
+
sighting("colleague", "mail-1");
|
|
477
|
+
sighting("client", "spam-2");
|
|
478
|
+
|
|
479
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
480
|
+
|
|
481
|
+
assert.equal(addressCount(), 3);
|
|
482
|
+
});
|
|
483
|
+
|
|
484
|
+
test("keeps the rest of an address's flags", async () => {
|
|
485
|
+
address("noisy", {}, '{"muted":{"value":true,"setAt":7}}');
|
|
486
|
+
sighting("noisy", "spam-1");
|
|
487
|
+
|
|
488
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
489
|
+
|
|
490
|
+
assert.deepEqual(JSON.parse(read("noisy").flags).muted, {
|
|
491
|
+
value: true,
|
|
492
|
+
setAt: 7,
|
|
493
|
+
});
|
|
494
|
+
});
|
|
495
|
+
|
|
496
|
+
test("a second run writes nothing", async () => {
|
|
497
|
+
address("spammer");
|
|
498
|
+
sighting("spammer", "spam-1");
|
|
499
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
500
|
+
const first = read("spammer");
|
|
501
|
+
|
|
502
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
503
|
+
|
|
504
|
+
assert.equal(report.withholdable, 0);
|
|
505
|
+
assert.equal(report.restorable, 0);
|
|
506
|
+
assert.equal(report.withheld, 0);
|
|
507
|
+
assert.equal(report.restored, 0);
|
|
508
|
+
assert.deepEqual(read("spammer"), first);
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
test("check mode reports what it would do and writes nothing", async () => {
|
|
512
|
+
address("spammer");
|
|
513
|
+
sighting("spammer", "spam-1");
|
|
514
|
+
address("misfiled", {}, '{"junkOnly":{"value":true,"setAt":1}}');
|
|
515
|
+
sighting("misfiled", "mail-1");
|
|
516
|
+
|
|
517
|
+
const report = await sweepJunkOnlyAddresses(clientOver(sqlite), "check");
|
|
518
|
+
|
|
519
|
+
assert.equal(report.withholdable, 1);
|
|
520
|
+
assert.equal(report.restorable, 1);
|
|
521
|
+
assert.equal(report.withheld, 0);
|
|
522
|
+
assert.equal(report.restored, 0);
|
|
523
|
+
assert.equal(withheld("spammer"), false);
|
|
524
|
+
assert.equal(withheld("misfiled"), true);
|
|
525
|
+
});
|
|
526
|
+
|
|
527
|
+
test("survives a row whose flags were never populated", async () => {
|
|
528
|
+
address("legacy", {}, "");
|
|
529
|
+
sighting("legacy", "spam-1");
|
|
530
|
+
|
|
531
|
+
await sweepJunkOnlyAddresses(clientOver(sqlite), "repair");
|
|
532
|
+
|
|
533
|
+
assert.equal(withheld("legacy"), true);
|
|
534
|
+
});
|
|
535
|
+
});
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import {
|
|
2
|
+
JUNK_FOLDER_NAMES,
|
|
3
|
+
TRASH_FOLDER_NAMES,
|
|
4
|
+
} from "@remit/data-ports/mailbox-role";
|
|
5
|
+
import { MailboxSpecialUse } from "@remit/domain-enums";
|
|
6
|
+
|
|
7
|
+
export interface JunkOnlyRepairClient {
|
|
8
|
+
all(sql: string, params: readonly unknown[]): Promise<unknown[]>;
|
|
9
|
+
run(sql: string, params: readonly unknown[]): Promise<number>;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export type JunkOnlyRepairMode = "check" | "repair";
|
|
13
|
+
|
|
14
|
+
export interface JunkOnlyReport {
|
|
15
|
+
readonly mode: JunkOnlyRepairMode;
|
|
16
|
+
readonly withholdable: number;
|
|
17
|
+
readonly withheld: number;
|
|
18
|
+
readonly restorable: number;
|
|
19
|
+
readonly restored: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export const JUNK_ONLY_FLAG = "junkOnly";
|
|
23
|
+
|
|
24
|
+
const STORED_FLAGS = "coalesce(nullif(address.flags, ''), '{}')";
|
|
25
|
+
|
|
26
|
+
const flagIsSet = (name: string): string =>
|
|
27
|
+
`coalesce(json_extract(${STORED_FLAGS}, '$.${name}.value'), 0) = 1`;
|
|
28
|
+
|
|
29
|
+
const quoted = (names: readonly string[]): string =>
|
|
30
|
+
names.map((name) => `'${name}'`).join(", ");
|
|
31
|
+
|
|
32
|
+
const MAILBOX_LEAF = `lower(substr(
|
|
33
|
+
mailbox.full_path,
|
|
34
|
+
length(rtrim(
|
|
35
|
+
mailbox.full_path,
|
|
36
|
+
replace(mailbox.full_path, mailbox.hierarchy_delimiter, '')
|
|
37
|
+
)) + 1
|
|
38
|
+
))`;
|
|
39
|
+
|
|
40
|
+
const mailboxCarriesRole = (
|
|
41
|
+
specialUse: string,
|
|
42
|
+
names: readonly string[],
|
|
43
|
+
): string => `(
|
|
44
|
+
exists (
|
|
45
|
+
SELECT 1 FROM mailbox_special_use_entry entry
|
|
46
|
+
WHERE entry.mailbox_id = message.mailbox_id
|
|
47
|
+
AND entry.special_use = '${specialUse}'
|
|
48
|
+
)
|
|
49
|
+
OR exists (
|
|
50
|
+
SELECT 1 FROM mailbox
|
|
51
|
+
WHERE mailbox.mailbox_id = message.mailbox_id
|
|
52
|
+
AND (
|
|
53
|
+
mailbox.special_use LIKE '%"${specialUse}"%'
|
|
54
|
+
OR ${MAILBOX_LEAF} IN (${quoted(names)})
|
|
55
|
+
)
|
|
56
|
+
)
|
|
57
|
+
)`;
|
|
58
|
+
|
|
59
|
+
const IN_JUNK = mailboxCarriesRole(MailboxSpecialUse.Junk, JUNK_FOLDER_NAMES);
|
|
60
|
+
const IN_TRASH = mailboxCarriesRole(
|
|
61
|
+
MailboxSpecialUse.Trash,
|
|
62
|
+
TRASH_FOLDER_NAMES,
|
|
63
|
+
);
|
|
64
|
+
|
|
65
|
+
const sightingWhere = (extra: string): string => `exists (
|
|
66
|
+
SELECT 1 FROM envelope_address
|
|
67
|
+
JOIN message ON message.message_id = envelope_address.message_id
|
|
68
|
+
WHERE envelope_address.address_id = address.address_id${extra}
|
|
69
|
+
)`;
|
|
70
|
+
|
|
71
|
+
const SIGHTING_IN_JUNK = sightingWhere(` AND ${IN_JUNK}`);
|
|
72
|
+
const SIGHTING_IN_LIVE_MAIL = sightingWhere(
|
|
73
|
+
` AND NOT ${IN_JUNK} AND NOT ${IN_TRASH}`,
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
export const ACCOUNT_HAS_CORRESPONDED = `(
|
|
77
|
+
address.outbound_count > 0
|
|
78
|
+
OR address.reply_count > 0
|
|
79
|
+
OR ${flagIsSet("vip")}
|
|
80
|
+
OR ${flagIsSet("trusted")}
|
|
81
|
+
)`;
|
|
82
|
+
|
|
83
|
+
export const WITHHOLDABLE = `NOT ${flagIsSet(JUNK_ONLY_FLAG)}
|
|
84
|
+
AND NOT ${ACCOUNT_HAS_CORRESPONDED}
|
|
85
|
+
AND ${SIGHTING_IN_JUNK}
|
|
86
|
+
AND NOT ${SIGHTING_IN_LIVE_MAIL}`;
|
|
87
|
+
|
|
88
|
+
export const RESTORABLE = `${flagIsSet(JUNK_ONLY_FLAG)}
|
|
89
|
+
AND (${ACCOUNT_HAS_CORRESPONDED} OR ${SIGHTING_IN_LIVE_MAIL})`;
|
|
90
|
+
|
|
91
|
+
export const withholdSql = (scope = ""): string =>
|
|
92
|
+
`UPDATE address
|
|
93
|
+
SET flags = json_set(${STORED_FLAGS}, '$.${JUNK_ONLY_FLAG}',
|
|
94
|
+
json_object('value', json('true'), 'setAt', CAST(? AS INTEGER), 'setBy', ?)),
|
|
95
|
+
updated_at = ?
|
|
96
|
+
WHERE ${WITHHOLDABLE}${scope}`;
|
|
97
|
+
|
|
98
|
+
export const restoreSql = (scope = ""): string =>
|
|
99
|
+
`UPDATE address
|
|
100
|
+
SET flags = json_remove(${STORED_FLAGS}, '$.${JUNK_ONLY_FLAG}'), updated_at = ?
|
|
101
|
+
WHERE ${RESTORABLE}${scope}`;
|
|
102
|
+
|
|
103
|
+
const REPAIR_SET_BY = "junk-only-repair";
|
|
104
|
+
|
|
105
|
+
const countWhere = async (
|
|
106
|
+
client: JunkOnlyRepairClient,
|
|
107
|
+
predicate: string,
|
|
108
|
+
): Promise<number> => {
|
|
109
|
+
const [row] = (await client.all(
|
|
110
|
+
`SELECT count(*) AS row_count FROM address WHERE ${predicate}`,
|
|
111
|
+
[],
|
|
112
|
+
)) as { row_count: number }[];
|
|
113
|
+
return row?.row_count ?? 0;
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
export const sweepJunkOnlyAddresses = async (
|
|
117
|
+
client: JunkOnlyRepairClient,
|
|
118
|
+
mode: JunkOnlyRepairMode,
|
|
119
|
+
now: number = Date.now(),
|
|
120
|
+
): Promise<JunkOnlyReport> => {
|
|
121
|
+
const withholdable = await countWhere(client, WITHHOLDABLE);
|
|
122
|
+
const restorable = await countWhere(client, RESTORABLE);
|
|
123
|
+
|
|
124
|
+
if (mode === "check") {
|
|
125
|
+
return { mode, withholdable, withheld: 0, restorable, restored: 0 };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const withheld =
|
|
129
|
+
withholdable === 0
|
|
130
|
+
? 0
|
|
131
|
+
: await client.run(withholdSql(), [now, REPAIR_SET_BY, now]);
|
|
132
|
+
|
|
133
|
+
const restored = restorable === 0 ? 0 : await client.run(restoreSql(), [now]);
|
|
134
|
+
|
|
135
|
+
return { mode, withholdable, withheld, restorable, restored };
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
export const formatJunkOnlyReport = (report: JunkOnlyReport): string[] => {
|
|
139
|
+
if (report.withholdable === 0 && report.restorable === 0) {
|
|
140
|
+
return ["no address stands only on mail in Junk"];
|
|
141
|
+
}
|
|
142
|
+
const lines: string[] = [];
|
|
143
|
+
if (report.withholdable > 0) {
|
|
144
|
+
lines.push(
|
|
145
|
+
report.mode === "check"
|
|
146
|
+
? `${report.withholdable} address(es) stand only on mail in Junk, would be withheld from autocomplete`
|
|
147
|
+
: `${report.withheld} of ${report.withholdable} address(es) standing only on mail in Junk withheld from autocomplete`,
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
if (report.restorable > 0) {
|
|
151
|
+
lines.push(
|
|
152
|
+
report.mode === "check"
|
|
153
|
+
? `${report.restorable} withheld address(es) now stand on live mail, would be restored`
|
|
154
|
+
: `${report.restored} of ${report.restorable} withheld address(es) restored to autocomplete`,
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
return lines;
|
|
158
|
+
};
|