@remit/drizzle-service 0.0.45 → 0.0.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@remit/drizzle-service",
3
- "version": "0.0.45",
3
+ "version": "0.0.47",
4
4
  "description": "Drizzle ORM service over SQLite",
5
5
  "type": "module",
6
6
  "main": "src/index.ts",
@@ -0,0 +1,280 @@
1
+ /**
2
+ * The repair against the shape a deployment actually runs: the committed
3
+ * `CREATE TABLE` blocks, not a schema pushed from the drizzle table objects.
4
+ *
5
+ * What this has to hold, on a live database with no second copy of the data:
6
+ * every name the harvest guard would keep survives, a name that says something
7
+ * besides the address keeps that something, no row is removed, and a second run
8
+ * writes nothing.
9
+ */
10
+
11
+ import assert from "node:assert/strict";
12
+ import { after, before, describe, test } from "node:test";
13
+ import { storedDisplayName } from "@remit/data-ports/display-name";
14
+ import Database from "better-sqlite3";
15
+ import { shippedTableDdl } from "../test-shipped-sqlite-schema.js";
16
+ import {
17
+ type DisplayNameRepairClient,
18
+ EMBEDDED_ADDRESS_LIKE,
19
+ sweepDisplayNames,
20
+ } from "./address-display-name.js";
21
+
22
+ const SPOOF = "aramirez@secresaludguaviare.gov.co";
23
+
24
+ const clientOver = (sqlite: Database.Database): DisplayNameRepairClient => ({
25
+ all: async (sql, params) => sqlite.prepare(sql).all(...params),
26
+ run: async (sql, params) => sqlite.prepare(sql).run(...params).changes,
27
+ });
28
+
29
+ const seed = (sqlite: Database.Database): void => {
30
+ sqlite.exec(shippedTableDdl("0000_happy_roland_deschain", "address"));
31
+ sqlite.exec(
32
+ shippedTableDdl("0000_happy_roland_deschain", "envelope_address"),
33
+ );
34
+ sqlite.exec(shippedTableDdl("0000_happy_roland_deschain", "thread_message"));
35
+
36
+ const address = sqlite.prepare(
37
+ `INSERT INTO address (
38
+ address_id, account_config_id, display_name, local_part, domain,
39
+ normalized_email, normalized_compound, flags, inbound_count,
40
+ outbound_count, reply_count, last_inbound_at, last_outbound_at,
41
+ last_reply_at, created_at, updated_at
42
+ ) VALUES (?, 'cfg-1', ?, 'x', 'y', ?, ?, '{}', 0, 0, 0, 0, NULL, 0, 0, 0)`,
43
+ );
44
+ const addressRow = (id: string, name: string | null, email: string): void => {
45
+ address.run(
46
+ id,
47
+ name,
48
+ email,
49
+ `${(name ?? "").toLowerCase()} ${email}`.trim(),
50
+ );
51
+ };
52
+
53
+ addressRow("spoof", "matthijs@ischen.nl", SPOOF);
54
+ addressRow("embedded", "Matthijs <matthijs@ischen.nl>", SPOOF);
55
+ addressRow("tabbed", "Support\tmatthijs@ischen.nl", SPOOF);
56
+ addressRow("parenthesised", "Support (support@acme.com)", "noreply@acme.com");
57
+ addressRow("comma", "matthijs@ischen.nl, team", SPOOF);
58
+ addressRow("semicolon", "Team; matthijs@ischen.nl", SPOOF);
59
+ addressRow("self", "ing@ing-nl-mailing.nl", "ing@ing-nl-mailing.nl");
60
+ addressRow("self-cased", "Matthijs@Ischen.nl", "matthijs@ischen.nl");
61
+ addressRow("self-diacritic", "Özcan@example.com", "özcan@example.com");
62
+ addressRow("self-in-name", "Özcan <Özcan@example.com>", "özcan@example.com");
63
+ addressRow("human", "Matthijs van Henten", "matthijs@ischen.nl");
64
+ addressRow("absent", null, "quiet@example.com");
65
+ addressRow("blank", "", "blank@example.com");
66
+
67
+ const envelope = sqlite.prepare(
68
+ `INSERT INTO envelope_address (
69
+ envelope_address_id, message_id, address_id, display_name,
70
+ normalized_email, address_role, address_order, created_at, updated_at
71
+ ) VALUES (?, 'msg-1', 'addr-1', ?, ?, 'From', 0, 0, 0)`,
72
+ );
73
+ envelope.run("env-spoof", "matthijs@ischen.nl", SPOOF);
74
+ envelope.run("env-self", "Özcan@example.com", "özcan@example.com");
75
+ envelope.run("env-human", "Matthijs van Henten", "matthijs@ischen.nl");
76
+
77
+ const thread = sqlite.prepare(
78
+ `INSERT INTO thread_message (
79
+ thread_message_id, thread_id, message_id, account_config_id, mailbox_id,
80
+ uid, reference_order, from_email, from_name, subject, internal_date,
81
+ sent_date, is_read, has_attachment, has_stars, is_deleted,
82
+ created_at, updated_at
83
+ ) VALUES (?, 'thr-1', 'msg-1', 'cfg-1', 'mbx-1', 1, 0, ?, ?, 's', 0, 0, 0, 0, 0, 0, 0, 0)`,
84
+ );
85
+ thread.run("thr-spoof", SPOOF, "matthijs@ischen.nl");
86
+ thread.run("thr-named", SPOOF, "Support <matthijs@ischen.nl>");
87
+ thread.run("thr-unparseable", null, "matthijs@ischen.nl");
88
+ thread.run("thr-human", SPOOF, "Alejandro Ramirez");
89
+ };
90
+
91
+ describe("rewriting display names that claim another address", () => {
92
+ let sqlite: Database.Database;
93
+ let client: DisplayNameRepairClient;
94
+
95
+ before(() => {
96
+ sqlite = new Database(":memory:");
97
+ seed(sqlite);
98
+ client = clientOver(sqlite);
99
+ });
100
+
101
+ after(() => {
102
+ sqlite.close();
103
+ });
104
+
105
+ const nameOf = (id: string): string | null =>
106
+ (
107
+ sqlite
108
+ .prepare("SELECT display_name AS n FROM address WHERE address_id = ?")
109
+ .get(id) as { n: string | null }
110
+ ).n;
111
+
112
+ test("check writes nothing and counts what repair would rewrite", async () => {
113
+ const report = await sweepDisplayNames(client, "check");
114
+
115
+ assert.equal(report.claiming, 10);
116
+ assert.deepEqual(
117
+ report.sites.map((site) => [site.table, site.claiming, site.rewritten]),
118
+ [
119
+ ["address", 6, 0],
120
+ ["envelope_address", 1, 0],
121
+ ["thread_message", 3, 0],
122
+ ],
123
+ );
124
+ assert.equal(nameOf("spoof"), "matthijs@ischen.nl");
125
+ });
126
+
127
+ test("repair rewrites exactly those rows", async () => {
128
+ const report = await sweepDisplayNames(client, "repair");
129
+
130
+ assert.deepEqual(
131
+ report.sites.map((site) => [site.table, site.rewritten]),
132
+ [
133
+ ["address", 6],
134
+ ["envelope_address", 1],
135
+ ["thread_message", 3],
136
+ ],
137
+ );
138
+ });
139
+
140
+ test("empties a name that is nothing but another address", () => {
141
+ assert.equal(nameOf("spoof"), "");
142
+ });
143
+
144
+ test("keeps the text a name carries besides the address", () => {
145
+ const kept: ReadonlyArray<readonly [string, string]> = [
146
+ ["embedded", "Matthijs"],
147
+ ["tabbed", "Support"],
148
+ ["parenthesised", "Support"],
149
+ ["comma", "team"],
150
+ ["semicolon", "Team"],
151
+ ];
152
+ for (const [id, remainder] of kept) {
153
+ assert.equal(nameOf(id), remainder, id);
154
+ }
155
+ });
156
+
157
+ test("rebuilds the search compound the way the app writes it", () => {
158
+ const rows = sqlite
159
+ .prepare(
160
+ "SELECT address_id AS id, normalized_compound AS c FROM address WHERE address_id IN ('spoof', 'embedded') ORDER BY address_id",
161
+ )
162
+ .all() as Array<{ id: string; c: string }>;
163
+ assert.deepEqual(
164
+ rows.map((row) => [row.id, row.c]),
165
+ [
166
+ ["embedded", `matthijs ${SPOOF}`],
167
+ ["spoof", SPOOF],
168
+ ],
169
+ );
170
+ });
171
+
172
+ test("keeps every name the harvest guard keeps", () => {
173
+ const kept: ReadonlyArray<readonly [string, string | null]> = [
174
+ ["self", "ing@ing-nl-mailing.nl"],
175
+ ["self-cased", "Matthijs@Ischen.nl"],
176
+ ["self-diacritic", "Özcan@example.com"],
177
+ ["self-in-name", "Özcan <Özcan@example.com>"],
178
+ ["human", "Matthijs van Henten"],
179
+ ["absent", null],
180
+ ["blank", ""],
181
+ ];
182
+ for (const [id, name] of kept) {
183
+ assert.equal(nameOf(id), name, id);
184
+ }
185
+ });
186
+
187
+ test("clears the From line the message header renders", () => {
188
+ const rows = sqlite
189
+ .prepare(
190
+ "SELECT envelope_address_id AS id, display_name AS n FROM envelope_address ORDER BY envelope_address_id",
191
+ )
192
+ .all() as Array<{ id: string; n: string | null }>;
193
+ assert.deepEqual(rows, [
194
+ { id: "env-human", n: "Matthijs van Henten" },
195
+ { id: "env-self", n: "Özcan@example.com" },
196
+ { id: "env-spoof", n: "" },
197
+ ]);
198
+ });
199
+
200
+ test("clears the sender label in the message list", () => {
201
+ const rows = sqlite
202
+ .prepare(
203
+ "SELECT thread_message_id AS id, from_name AS n FROM thread_message ORDER BY thread_message_id",
204
+ )
205
+ .all() as Array<{ id: string; n: string | null }>;
206
+ assert.deepEqual(rows, [
207
+ { id: "thr-human", n: "Alejandro Ramirez" },
208
+ { id: "thr-named", n: "Support" },
209
+ { id: "thr-spoof", n: null },
210
+ { id: "thr-unparseable", n: null },
211
+ ]);
212
+ });
213
+
214
+ test("removes no row", () => {
215
+ for (const [table, count] of [
216
+ ["address", 13],
217
+ ["envelope_address", 3],
218
+ ["thread_message", 4],
219
+ ] as const) {
220
+ const row = sqlite
221
+ .prepare(`SELECT count(*) AS n FROM ${table}`)
222
+ .get() as {
223
+ n: number;
224
+ };
225
+ assert.equal(row.n, count, table);
226
+ }
227
+ });
228
+
229
+ test("a second run writes nothing", async () => {
230
+ const report = await sweepDisplayNames(client, "repair");
231
+ assert.equal(report.claiming, 0);
232
+ });
233
+ });
234
+
235
+ /**
236
+ * The SQL in this repair narrows; it never decides. That only holds if no name
237
+ * the rule rewrites can slip past the narrowing — the case where a planted name
238
+ * would survive the sweep unseen.
239
+ */
240
+ describe("the SQL narrowing is a superset of the rule", () => {
241
+ test("selects every name the rule rewrites", () => {
242
+ const sqlite = new Database(":memory:");
243
+ sqlite.exec("CREATE TABLE probe (name text)");
244
+ const insert = sqlite.prepare("INSERT INTO probe VALUES (?)");
245
+ const names = [
246
+ "matthijs@ischen.nl",
247
+ "Matthijs <matthijs@ischen.nl>",
248
+ "Support (support@acme.com)",
249
+ "Support\tmatthijs@ischen.nl",
250
+ "Support matthijs@ischen.nl",
251
+ "Support​matthijs@ischen.nl",
252
+ "prvs=0068b51f37=matthijs@ischen.nl",
253
+ '"matthijs@ischen.nl"',
254
+ "matthijs@ischen.nl, team",
255
+ "Team; matthijs@ischen.nl",
256
+ "Özcan@example.com",
257
+ "MATTHIJS@ISCHEN.NL",
258
+ "matthijs@mail.ischen.nl",
259
+ "Matthijs van Henten",
260
+ "me @ home",
261
+ "a@b.c",
262
+ "",
263
+ ];
264
+ for (const name of names) insert.run(name);
265
+
266
+ const selected = new Set(
267
+ (
268
+ sqlite
269
+ .prepare("SELECT name FROM probe WHERE name LIKE ?")
270
+ .all(EMBEDDED_ADDRESS_LIKE) as Array<{ name: string }>
271
+ ).map((row) => row.name),
272
+ );
273
+ sqlite.close();
274
+
275
+ for (const name of names) {
276
+ if (storedDisplayName(name, "nobody@example.org") === name) continue;
277
+ assert.equal(selected.has(name), true, name);
278
+ }
279
+ });
280
+ });
@@ -0,0 +1,200 @@
1
+ import { storedDisplayName } from "@remit/data-ports/display-name";
2
+
3
+ /**
4
+ * Rewriting the display names a spoofing sender already planted (issue #826).
5
+ *
6
+ * The harvest guard only decides what is stored from now on. Every name already
7
+ * written stays live, and none of the three write paths repairs one on a
8
+ * re-sync: `upsertAddress` refuses to overwrite a name with an empty one,
9
+ * `upsertEnvelopeAddress` is `onConflictDoNothing`, and a ThreadMessage is
10
+ * created once. On the instance that was hit there were 150 email-shaped
11
+ * display names in `address` alone, 30 of them naming a different address.
12
+ *
13
+ * The decision is `storedDisplayName` and nothing else. Expressing it a second
14
+ * time in SQL is what makes this dangerous: SQLite's `lower()` folds ASCII
15
+ * where JS folds all of Unicode, and its `trim()` and a literal space in a GLOB
16
+ * know only U+0020, so a SQL twin rewrites `Özcan@example.com` on
17
+ * `özcan@example.com` and `foo\tbar@baz.com` on anything — names the guard
18
+ * keeps, destroyed on a database holding the only copy. SQL narrows the scan
19
+ * and never decides.
20
+ *
21
+ * This is a repair rather than a migration because a migration is SQL, and SQL
22
+ * is exactly what must not hold the rule. It is convergent: a name the guard
23
+ * would keep is never rewritten twice, so re-running it, or resuming it after a
24
+ * crash part-way through, writes only what is left to write.
25
+ */
26
+
27
+ export interface DisplayNameRepairClient {
28
+ all(sql: string, params: readonly unknown[]): Promise<unknown[]>;
29
+ run(sql: string, params: readonly unknown[]): Promise<number>;
30
+ }
31
+
32
+ export type DisplayNameRepairMode = "check" | "repair";
33
+
34
+ /**
35
+ * The scan narrows to names that could carry an address at all: everything
36
+ * `storedDisplayName` rewrites contains `x@y.zz`, so a row this misses cannot
37
+ * be claiming anything. It is a filter, never the decision — the pairing is
38
+ * pinned by a test that runs both halves over the same strings.
39
+ */
40
+ export const EMBEDDED_ADDRESS_LIKE = "%_@_%.__%";
41
+
42
+ interface RepairSite {
43
+ readonly table: string;
44
+ readonly key: string;
45
+ readonly name: string;
46
+ readonly email: string;
47
+ /** The compound the search path reads, where the table keeps one. */
48
+ readonly compound?: string;
49
+ /** What the write path stores for an absent name on this table. */
50
+ readonly absent: "" | null;
51
+ }
52
+
53
+ /**
54
+ * Every column an attacker-chosen name lands in. `address.display_name` is the
55
+ * one autocomplete reads; `envelope_address.display_name` is the From line the
56
+ * message header renders; `thread_message.from_name` is the sender label in the
57
+ * message list and the text the search index tokenizes.
58
+ */
59
+ const SITES: readonly RepairSite[] = [
60
+ {
61
+ table: "address",
62
+ key: "address_id",
63
+ name: "display_name",
64
+ email: "normalized_email",
65
+ compound: "normalized_compound",
66
+ absent: "",
67
+ },
68
+ {
69
+ table: "envelope_address",
70
+ key: "envelope_address_id",
71
+ name: "display_name",
72
+ email: "normalized_email",
73
+ absent: "",
74
+ },
75
+ {
76
+ table: "thread_message",
77
+ key: "thread_message_id",
78
+ name: "from_name",
79
+ email: "from_email",
80
+ absent: null,
81
+ },
82
+ ];
83
+
84
+ export interface SiteResult {
85
+ readonly table: string;
86
+ readonly scanned: number;
87
+ readonly claiming: number;
88
+ readonly rewritten: number;
89
+ }
90
+
91
+ export interface DisplayNameReport {
92
+ readonly mode: DisplayNameRepairMode;
93
+ readonly sites: readonly SiteResult[];
94
+ readonly claiming: number;
95
+ }
96
+
97
+ interface CandidateRow {
98
+ id: string;
99
+ name: string | null;
100
+ email: string | null;
101
+ }
102
+
103
+ const isCandidateRow = (row: unknown): row is CandidateRow =>
104
+ typeof row === "object" &&
105
+ row !== null &&
106
+ typeof (row as { id: unknown }).id === "string";
107
+
108
+ const candidates = async (
109
+ client: DisplayNameRepairClient,
110
+ site: RepairSite,
111
+ ): Promise<CandidateRow[]> => {
112
+ const rows = await client.all(
113
+ `SELECT ${site.key} AS id, ${site.name} AS name, ${site.email} AS email
114
+ FROM ${site.table}
115
+ WHERE ${site.name} LIKE ?`,
116
+ [EMBEDDED_ADDRESS_LIKE],
117
+ );
118
+ return rows.filter(isCandidateRow);
119
+ };
120
+
121
+ /**
122
+ * One row at a time, because each row keeps a different remainder. The set is
123
+ * what a spoofing sender planted, not the table.
124
+ */
125
+ const rewrite = async (
126
+ client: DisplayNameRepairClient,
127
+ site: RepairSite,
128
+ row: CandidateRow,
129
+ stored: string,
130
+ ): Promise<number> => {
131
+ const columns = [`${site.name} = ?`];
132
+ const params: unknown[] = [stored === "" ? site.absent : stored];
133
+
134
+ if (site.compound) {
135
+ columns.push(`${site.compound} = ?`);
136
+ params.push(`${stored.toLowerCase()} ${row.email ?? ""}`.trim());
137
+ }
138
+ params.push(row.id);
139
+
140
+ return client.run(
141
+ `UPDATE ${site.table} SET ${columns.join(", ")} WHERE ${site.key} = ?`,
142
+ params,
143
+ );
144
+ };
145
+
146
+ /**
147
+ * `check` writes nothing, so it can be pointed at a live instance; `repair`
148
+ * runs the same scan and rewrites what it finds. One code path, so the report
149
+ * can never describe a decision the repair does not make.
150
+ */
151
+ export const sweepDisplayNames = async (
152
+ client: DisplayNameRepairClient,
153
+ mode: DisplayNameRepairMode,
154
+ ): Promise<DisplayNameReport> => {
155
+ const sites: SiteResult[] = [];
156
+
157
+ for (const site of SITES) {
158
+ const rows = await candidates(client, site);
159
+ let claiming = 0;
160
+ let rewritten = 0;
161
+
162
+ for (const row of rows) {
163
+ const name = row.name ?? "";
164
+ const stored = storedDisplayName(name, row.email ?? undefined);
165
+ if (stored === name) continue;
166
+ claiming += 1;
167
+ if (mode === "repair") {
168
+ rewritten += await rewrite(client, site, row, stored);
169
+ }
170
+ }
171
+
172
+ sites.push({
173
+ table: site.table,
174
+ scanned: rows.length,
175
+ claiming,
176
+ rewritten,
177
+ });
178
+ }
179
+
180
+ return {
181
+ mode,
182
+ sites,
183
+ claiming: sites.reduce((sum, site) => sum + site.claiming, 0),
184
+ };
185
+ };
186
+
187
+ export const formatDisplayNameReport = (
188
+ report: DisplayNameReport,
189
+ ): string[] => {
190
+ if (report.claiming === 0) {
191
+ return ["no display name claims another address"];
192
+ }
193
+ return report.sites
194
+ .filter((site) => site.claiming > 0)
195
+ .map(
196
+ (site) =>
197
+ `${site.table}: ${site.claiming} of ${site.scanned} scanned name(s) claim another address` +
198
+ (report.mode === "repair" ? `, ${site.rewritten} rewritten` : ""),
199
+ );
200
+ };
@@ -0,0 +1,117 @@
1
+ /**
2
+ * The repair against the shape a deployment actually runs: the committed
3
+ * `CREATE TABLE` block, not a schema pushed from the drizzle table objects.
4
+ *
5
+ * What it has to hold on a live database: a message whose filing is still in
6
+ * flight is left alone, a stranded one becomes visible, no row is removed, and
7
+ * a second run writes nothing.
8
+ */
9
+
10
+ import assert from "node:assert/strict";
11
+ import { after, before, describe, test } from "node:test";
12
+ import Database from "better-sqlite3";
13
+ import { shippedTableDdl } from "../test-shipped-sqlite-schema.js";
14
+ import {
15
+ STRANDED_AFTER_MILLIS,
16
+ type StrandedSentRepairClient,
17
+ sweepStrandedSentOutbox,
18
+ } from "./stranded-sent-outbox.js";
19
+
20
+ const clientOver = (sqlite: Database.Database): StrandedSentRepairClient => ({
21
+ all: async (sql, params) => sqlite.prepare(sql).all(...params),
22
+ run: async (sql, params) => sqlite.prepare(sql).run(...params).changes,
23
+ });
24
+
25
+ interface Row {
26
+ status: string;
27
+ last_error: string | null;
28
+ }
29
+
30
+ describe("outbox rows stranded at sent", () => {
31
+ let sqlite: Database.Database;
32
+
33
+ const insert = (
34
+ outboxMessageId: string,
35
+ status: string,
36
+ sentAt: number,
37
+ ): void => {
38
+ sqlite
39
+ .prepare(
40
+ `INSERT INTO outbox_message (
41
+ outbox_message_id, account_id, account_config_id, from_address,
42
+ to_addresses, cc_addresses, bcc_addresses, "references",
43
+ message_id_value, status, sent_at, created_at, updated_at
44
+ ) VALUES (?, 'acc', 'cfg', 'me@example.com', '["you@example.com"]',
45
+ '[]', '[]', '[]', 'mid@example.com', ?, ?, ?, ?)`,
46
+ )
47
+ .run(outboxMessageId, status, sentAt, sentAt, sentAt);
48
+ };
49
+
50
+ const read = (outboxMessageId: string): Row =>
51
+ sqlite
52
+ .prepare(
53
+ "SELECT status, last_error FROM outbox_message WHERE outbox_message_id = ?",
54
+ )
55
+ .get(outboxMessageId) as Row;
56
+
57
+ before(() => {
58
+ sqlite = new Database(":memory:");
59
+ sqlite.exec(
60
+ shippedTableDdl("0000_happy_roland_deschain", "outbox_message"),
61
+ );
62
+ });
63
+
64
+ after(() => {
65
+ sqlite.close();
66
+ });
67
+
68
+ test("makes a stranded sent message visible again", async () => {
69
+ const stranded = Date.now() - STRANDED_AFTER_MILLIS - 1000;
70
+ insert("stranded", "sent", stranded);
71
+
72
+ const report = await sweepStrandedSentOutbox(clientOver(sqlite), "repair");
73
+
74
+ assert.equal(report.stranded, 1);
75
+ assert.equal(report.settled, 1);
76
+ const row = read("stranded");
77
+ assert.equal(row.status, "unfiled");
78
+ assert.match(String(row.last_error), /not filed/);
79
+ });
80
+
81
+ test("a second run writes nothing", async () => {
82
+ const report = await sweepStrandedSentOutbox(clientOver(sqlite), "repair");
83
+
84
+ assert.equal(report.stranded, 0);
85
+ assert.equal(report.settled, 0);
86
+ });
87
+
88
+ test("leaves a filing that is still in flight alone", async () => {
89
+ insert("in-flight", "sent", Date.now());
90
+
91
+ const report = await sweepStrandedSentOutbox(clientOver(sqlite), "repair");
92
+
93
+ assert.equal(report.stranded, 0);
94
+ assert.equal(read("in-flight").status, "sent");
95
+ });
96
+
97
+ test("touches no other status", async () => {
98
+ const old = Date.now() - STRANDED_AFTER_MILLIS - 1000;
99
+ insert("a-draft", "draft", old);
100
+ insert("a-failure", "failed", old);
101
+
102
+ await sweepStrandedSentOutbox(clientOver(sqlite), "repair");
103
+
104
+ assert.equal(read("a-draft").status, "draft");
105
+ assert.equal(read("a-failure").status, "failed");
106
+ });
107
+
108
+ test("check mode reports what it would do and writes nothing", async () => {
109
+ insert("also-stranded", "sent", Date.now() - STRANDED_AFTER_MILLIS - 1000);
110
+
111
+ const report = await sweepStrandedSentOutbox(clientOver(sqlite), "check");
112
+
113
+ assert.equal(report.stranded, 1);
114
+ assert.equal(report.settled, 0);
115
+ assert.equal(read("also-stranded").status, "sent");
116
+ });
117
+ });
@@ -0,0 +1,109 @@
1
+ /**
2
+ * The repair for outbox rows stranded at `sent` (issue #824).
3
+ *
4
+ * A row holds `sent` only between SMTP accepting the message and the IMAP
5
+ * APPEND that files a copy in Sent, whose last act is to delete the row. Every
6
+ * way that APPEND could fail used to end in a silent return, so the row stayed
7
+ * at `sent` forever — and `sent` is the one non-draft status the Outbox list
8
+ * hides, on the assumption that the APPEND deletes it. The message was
9
+ * delivered, exists in no server folder, and appears in no view.
10
+ *
11
+ * The handler now settles those failures as `unfiled`. That reaches nothing
12
+ * already stranded: the events are long gone, and the rows are on instances
13
+ * that only an image update reaches. So the same settlement is applied here,
14
+ * once, at boot — the migrate one-shot is the only step every self-host
15
+ * instance runs before its app containers start (#281).
16
+ *
17
+ * The age bound is what separates a stranded row from an ordinary one. A
18
+ * message sent seconds before a restart has its APPEND event still queued, and
19
+ * that event settles the row itself once the workers come back; rewriting it
20
+ * here would take a filing that was about to succeed. An hour is far past any
21
+ * redrive budget, so a row older than that has no event coming.
22
+ *
23
+ * It is a status flip and an error string, on rows whose only other outcome is
24
+ * to stay invisible. Nothing is deleted and no message content is touched.
25
+ */
26
+
27
+ /**
28
+ * The smallest surface this needs, so the module imports no schema and no
29
+ * driver and can be driven by the migrator's `better-sqlite3` handle or a test.
30
+ */
31
+ export interface StrandedSentRepairClient {
32
+ all(sql: string, params: unknown[]): Promise<unknown[]>;
33
+ run(sql: string, params: unknown[]): Promise<number>;
34
+ }
35
+
36
+ export type StrandedSentRepairMode = "check" | "repair";
37
+
38
+ export type StrandedSentReport = {
39
+ readonly mode: StrandedSentRepairMode;
40
+ readonly stranded: number;
41
+ readonly settled: number;
42
+ };
43
+
44
+ /**
45
+ * One hour. A row younger than this may still have its APPEND event in the
46
+ * queue, and that event settles the row correctly on its own.
47
+ */
48
+ export const STRANDED_AFTER_MILLIS = 60 * 60 * 1000;
49
+
50
+ const STRANDED_SENT_REASON =
51
+ "Sent, but not filed: filing a copy in the Sent folder never completed. The message was delivered.";
52
+
53
+ const NOW_MILLIS = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
54
+
55
+ const AGE = `coalesce(sent_at, updated_at) < ${NOW_MILLIS} - ?`;
56
+
57
+ const COUNT_SQL = `SELECT count(*) AS row_count
58
+ FROM outbox_message
59
+ WHERE status = 'sent' AND ${AGE}`;
60
+
61
+ const SETTLE_SQL = `UPDATE outbox_message
62
+ SET status = 'unfiled', last_error = ?
63
+ WHERE status = 'sent' AND ${AGE}`;
64
+
65
+ const countStranded = async (
66
+ client: StrandedSentRepairClient,
67
+ ): Promise<number> => {
68
+ const [row] = (await client.all(COUNT_SQL, [STRANDED_AFTER_MILLIS])) as {
69
+ row_count: number;
70
+ }[];
71
+ return row?.row_count ?? 0;
72
+ };
73
+
74
+ /**
75
+ * The UPDATE runs only when the count found rows. SQLite takes its exclusive
76
+ * write lock the moment an UPDATE begins, before it can know the WHERE matches
77
+ * nothing, and a lock the migrator cannot get inside its `busy_timeout` fails
78
+ * the migration and holds every gated service down. Zero is the steady state.
79
+ */
80
+ export const sweepStrandedSentOutbox = async (
81
+ client: StrandedSentRepairClient,
82
+ mode: StrandedSentRepairMode,
83
+ ): Promise<StrandedSentReport> => {
84
+ const stranded = await countStranded(client);
85
+ if (mode === "check" || stranded === 0) {
86
+ return { mode, stranded, settled: 0 };
87
+ }
88
+ const settled = await client.run(SETTLE_SQL, [
89
+ STRANDED_SENT_REASON,
90
+ STRANDED_AFTER_MILLIS,
91
+ ]);
92
+ return { mode, stranded, settled };
93
+ };
94
+
95
+ export const formatStrandedSentReport = (
96
+ report: StrandedSentReport,
97
+ ): string[] => {
98
+ if (report.stranded === 0) {
99
+ return ["No sent message is stranded in the outbox"];
100
+ }
101
+ if (report.mode === "check") {
102
+ return [
103
+ `${report.stranded} sent message(s) stranded in the outbox, would be marked unfiled`,
104
+ ];
105
+ }
106
+ return [
107
+ `${report.settled} of ${report.stranded} stranded sent message(s) marked unfiled`,
108
+ ];
109
+ };