@remit/drizzle-service 0.0.53 → 0.0.54

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.53",
+  "version": "0.0.54",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/repair/stranded-sent-outbox.sqlite.test.ts

@@ -3,14 +3,17 @@
  * `CREATE TABLE` block, not a schema pushed from the drizzle table objects.
  *
  * What it has to hold on a live database: a message whose filing is still in
- * flight is left alone, a stranded one becomes visible, no row is removed, and
- * a second run writes nothing.
+ * flight is left alone, one that was never filed becomes visible, one that was
+ * filed loses the row that outlived its delete, and a second run writes nothing.
  */
 
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import Database from "better-sqlite3";
-import { shippedTableDdl } from "../test-shipped-sqlite-schema.js";
+import {
+	applyMigration,
+	shippedTableDdl,
+} from "../test-shipped-sqlite-schema.js";
 import {
 	STRANDED_AFTER_MILLIS,
 	type StrandedSentRepairClient,
@@ -34,31 +37,56 @@ describe("outbox rows stranded at sent", () => {
 		outboxMessageId: string,
 		status: string,
 		sentAt: number,
+		appendedUid = 0,
 	): void => {
 		sqlite
 			.prepare(
 				`INSERT INTO outbox_message (
 					outbox_message_id, account_id, account_config_id, from_address,
 					to_addresses, cc_addresses, bcc_addresses, "references",
-					message_id_value, status, sent_at, created_at, updated_at
+					message_id_value, status, sent_at, appended_uid, created_at, updated_at
 				) VALUES (?, 'acc', 'cfg', 'me@example.com', '["you@example.com"]',
-					'[]', '[]', '[]', 'mid@example.com', ?, ?, ?, ?)`,
+					'[]', '[]', '[]', 'mid@example.com', ?, ?, ?, ?, ?)`,
 			)
-			.run(outboxMessageId, status, sentAt, sentAt, sentAt);
+			.run(outboxMessageId, status, sentAt, appendedUid, sentAt, sentAt);
 	};
 
-	const read = (outboxMessageId: string): Row =>
+	const attach = (outboxMessageId: string): void => {
+		sqlite
+			.prepare(
+				`INSERT INTO outbox_attachment (
+					outbox_attachment_id, outbox_message_id, account_id,
+					account_config_id, filename, content_type, size_bytes, state,
+					storage_key, reservation_expires_at, created_at, updated_at
+				) VALUES (?, ?, 'acc', 'cfg', 'q.pdf', 'application/pdf', 12,
+					'Stored', 'k', 0, 0, 0)`,
+			)
+			.run(`att-${outboxMessageId}`, outboxMessageId);
+	};
+
+	const read = (outboxMessageId: string): Row | undefined =>
 		sqlite
 			.prepare(
 				"SELECT status, last_error FROM outbox_message WHERE outbox_message_id = ?",
 			)
-			.get(outboxMessageId) as Row;
+			.get(outboxMessageId) as Row | undefined;
+
+	const attachmentCount = (outboxMessageId: string): number =>
+		(
+			sqlite
+				.prepare(
+					"SELECT count(*) AS n FROM outbox_attachment WHERE outbox_message_id = ?",
+				)
+				.get(outboxMessageId) as { n: number }
+		).n;
 
 	before(() => {
 		sqlite = new Database(":memory:");
 		sqlite.exec(
 			shippedTableDdl("0000_happy_roland_deschain", "outbox_message"),
 		);
+		sqlite.exec(shippedTableDdl("0013_low_harpoon", "outbox_attachment"));
+		applyMigration(sqlite, "0015_true_meggan");
 	});
 
 	after(() => {
@@ -72,10 +100,11 @@ describe("outbox rows stranded at sent", () => {
 		const report = await sweepStrandedSentOutbox(clientOver(sqlite), "repair");
 
 		assert.equal(report.stranded, 1);
+		assert.equal(report.neverFiled, 1);
 		assert.equal(report.settled, 1);
 		const row = read("stranded");
-		assert.equal(row.status, "unfiled");
-		assert.match(String(row.last_error), /not filed/);
+		assert.equal(row?.status, "unfiled");
+		assert.match(String(row?.last_error), /not filed/);
 	});
 
 	test("a second run writes nothing", async () => {
@@ -83,6 +112,7 @@ describe("outbox rows stranded at sent", () => {
 
 		assert.equal(report.stranded, 0);
 		assert.equal(report.settled, 0);
+		assert.equal(report.dropped, 0);
 	});
 
 	test("leaves a filing that is still in flight alone", async () => {
@@ -91,7 +121,7 @@ describe("outbox rows stranded at sent", () => {
 		const report = await sweepStrandedSentOutbox(clientOver(sqlite), "repair");
 
 		assert.equal(report.stranded, 0);
-		assert.equal(read("in-flight").status, "sent");
+		assert.equal(read("in-flight")?.status, "sent");
 	});
 
 	test("touches no other status", async () => {
@@ -101,8 +131,8 @@ describe("outbox rows stranded at sent", () => {
 
 		await sweepStrandedSentOutbox(clientOver(sqlite), "repair");
 
-		assert.equal(read("a-draft").status, "draft");
-		assert.equal(read("a-failure").status, "failed");
+		assert.equal(read("a-draft")?.status, "draft");
+		assert.equal(read("a-failure")?.status, "failed");
 	});
 
 	test("check mode reports what it would do and writes nothing", async () => {
@@ -112,6 +142,53 @@ describe("outbox rows stranded at sent", () => {
 
 		assert.equal(report.stranded, 1);
 		assert.equal(report.settled, 0);
-		assert.equal(read("also-stranded").status, "sent");
+		assert.equal(read("also-stranded")?.status, "sent");
+	});
+
+	test("drops the leftover row of a message that was filed (#858)", async () => {
+		const old = Date.now() - STRANDED_AFTER_MILLIS - 1000;
+		insert("filed", "sent", old, 55);
+		attach("filed");
+		insert("never-filed", "sent", old);
+		attach("never-filed");
+
+		const report = await sweepStrandedSentOutbox(clientOver(sqlite), "repair");
+
+		// The copy is in the user's Sent folder. Marking it unfiled would say the
+		// message was never filed and show it in the Outbox alongside the copy.
+		assert.equal(report.filed, 1);
+		assert.equal(report.dropped, 1);
+		assert.equal(read("filed"), undefined);
+
+		// Its attachment rows go with it, so the objects stop being vouched for
+		// and the scheduler's attachment sweep can collect them.
+		assert.equal(attachmentCount("filed"), 0);
+
+		assert.equal(read("never-filed")?.status, "unfiled");
+		assert.equal(attachmentCount("never-filed"), 1);
+	});
+
+	test("check mode reports a filed leftover without dropping it", async () => {
+		const old = Date.now() - STRANDED_AFTER_MILLIS - 1000;
+		insert("filed-too", "sent", old, 55);
+
+		const report = await sweepStrandedSentOutbox(clientOver(sqlite), "check");
+
+		assert.equal(report.filed, 1);
+		assert.equal(report.dropped, 0);
+		assert.equal(read("filed-too")?.status, "sent");
+	});
+
+	test("counts a copy the server named no uid for as filed", async () => {
+		// A server without UIDPLUS files the copy and reports nothing. Settling
+		// that row unfiled would be as wrong as settling one with a real uid.
+		insert("no-uid", "sent", Date.now() - STRANDED_AFTER_MILLIS - 1000, -1);
+
+		const report = await sweepStrandedSentOutbox(clientOver(sqlite), "repair");
+
+		assert.equal(report.filed, 2);
+		assert.equal(report.dropped, 2);
+		assert.equal(read("no-uid"), undefined);
+		assert.equal(read("filed-too"), undefined);
 	});
 });

package/src/repair/stranded-sent-outbox.ts

@@ -14,14 +14,25 @@
  * once, at boot — the migrate one-shot is the only step every self-host
  * instance runs before its app containers start (#281).
  *
+ * Two different things strand a row, and `appended_uid` is what tells them
+ * apart (issue #858). At `0` the APPEND was never confirmed: the message is in
+ * no folder, and `unfiled` with an error the user can read is the honest
+ * outcome. At anything else the copy is in the user's Sent folder and only the
+ * delete that follows the APPEND failed, so the row is a leftover — marking it
+ * `unfiled` would tell the user a message that is sitting in Sent was never
+ * filed, and show it to them twice. Those rows are dropped instead, finishing
+ * the delete the handler owed. Their attachment objects are collected by the
+ * scheduler's attachment sweep, which is where every object that outlives its
+ * row already ends up.
+ *
  * The age bound is what separates a stranded row from an ordinary one. A
  * message sent seconds before a restart has its APPEND event still queued, and
  * that event settles the row itself once the workers come back; rewriting it
  * here would take a filing that was about to succeed. An hour is far past any
  * redrive budget, so a row older than that has no event coming.
  *
- * It is a status flip and an error string, on rows whose only other outcome is
- * to stay invisible. Nothing is deleted and no message content is touched.
+ * No message content is touched, and nothing is dropped that the mail server is
+ * not already holding a copy of.
  */
 
 /**
@@ -38,7 +49,12 @@ export type StrandedSentRepairMode = "check" | "repair";
 export type StrandedSentReport = {
 	readonly mode: StrandedSentRepairMode;
 	readonly stranded: number;
+	/** Stranded rows whose APPEND was never confirmed. */
+	readonly neverFiled: number;
+	/** Stranded rows whose copy reached Sent and whose delete did not. */
+	readonly filed: number;
 	readonly settled: number;
+	readonly dropped: number;
 };
 
 /**
@@ -50,46 +66,93 @@ export const STRANDED_AFTER_MILLIS = 60 * 60 * 1000;
 const STRANDED_SENT_REASON =
 	"Sent, but not filed: filing a copy in the Sent folder never completed. The message was delivered.";
 
-const NOW_MILLIS = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
+/**
+ * The cutoff is computed once and bound to all four statements, rather than
+ * each of them subtracting the age bound from its own `unixepoch()`. Four
+ * clocks give four different populations: a row that crosses the hour between
+ * the count and a write, or between the two deletes, is counted and not written
+ * or has its files taken and its row left.
+ */
+const STRANDED = "status = 'sent' AND coalesce(sent_at, updated_at) < ?";
+
+/**
+ * `isSentCopyFiled` in @remit/data-ports, in SQL. Only the two values that mean
+ * "filed" say so, so a value neither writes reads as unfiled — which settles a
+ * row rather than dropping it.
+ */
+const FILED_UID = "(appended_uid > 0 OR appended_uid = -1)";
+
+const NEVER_FILED = `${STRANDED} AND NOT ${FILED_UID}`;
 
-const AGE = `coalesce(sent_at, updated_at) < ${NOW_MILLIS} - ?`;
+const FILED = `${STRANDED} AND ${FILED_UID}`;
 
-const COUNT_SQL = `SELECT count(*) AS row_count
+const COUNT_SQL = `SELECT
+	coalesce(sum(CASE WHEN NOT ${FILED_UID} THEN 1 ELSE 0 END), 0) AS never_filed,
+	coalesce(sum(CASE WHEN ${FILED_UID} THEN 1 ELSE 0 END), 0) AS filed
 FROM outbox_message
-WHERE status = 'sent' AND ${AGE}`;
+WHERE ${STRANDED}`;
 
 const SETTLE_SQL = `UPDATE outbox_message
 SET status = 'unfiled', last_error = ?
-WHERE status = 'sent' AND ${AGE}`;
+WHERE ${NEVER_FILED}`;
+
+const DROP_ATTACHMENTS_SQL = `DELETE FROM outbox_attachment
+WHERE outbox_message_id IN (
+	SELECT outbox_message_id FROM outbox_message WHERE ${FILED}
+)`;
+
+const DROP_MESSAGES_SQL = `DELETE FROM outbox_message
+WHERE ${FILED}`;
 
 const countStranded = async (
 	client: StrandedSentRepairClient,
-): Promise<number> => {
-	const [row] = (await client.all(COUNT_SQL, [STRANDED_AFTER_MILLIS])) as {
-		row_count: number;
+	strandedBefore: number,
+): Promise<{ neverFiled: number; filed: number }> => {
+	const [row] = (await client.all(COUNT_SQL, [strandedBefore])) as {
+		never_filed: number;
+		filed: number;
 	}[];
-	return row?.row_count ?? 0;
+	return { neverFiled: row?.never_filed ?? 0, filed: row?.filed ?? 0 };
 };
 
 /**
- * The UPDATE runs only when the count found rows. SQLite takes its exclusive
- * write lock the moment an UPDATE begins, before it can know the WHERE matches
- * nothing, and a lock the migrator cannot get inside its `busy_timeout` fails
- * the migration and holds every gated service down. Zero is the steady state.
+ * Each write runs only when the count found rows for it. SQLite takes its
+ * exclusive write lock the moment a statement begins, before it can know the
+ * WHERE matches nothing, and a lock the migrator cannot get inside its
+ * `busy_timeout` fails the migration and holds every gated service down. Zero
+ * is the steady state.
+ *
+ * The attachment rows go before the message rows they hang off. Reversed, a
+ * crash between the two statements leaves attachment rows naming a message that
+ * no longer exists — nothing would ever find them again, and their objects would
+ * be vouched for forever.
  */
 export const sweepStrandedSentOutbox = async (
 	client: StrandedSentRepairClient,
 	mode: StrandedSentRepairMode,
 ): Promise<StrandedSentReport> => {
-	const stranded = await countStranded(client);
+	const strandedBefore = Date.now() - STRANDED_AFTER_MILLIS;
+	const { neverFiled, filed } = await countStranded(client, strandedBefore);
+	const stranded = neverFiled + filed;
+	const nothingWritten = { mode, stranded, neverFiled, filed } as const;
+
 	if (mode === "check" || stranded === 0) {
-		return { mode, stranded, settled: 0 };
+		return { ...nothingWritten, settled: 0, dropped: 0 };
 	}
-	const settled = await client.run(SETTLE_SQL, [
-		STRANDED_SENT_REASON,
-		STRANDED_AFTER_MILLIS,
-	]);
-	return { mode, stranded, settled };
+
+	const settled =
+		neverFiled === 0
+			? 0
+			: await client.run(SETTLE_SQL, [STRANDED_SENT_REASON, strandedBefore]);
+
+	if (filed === 0) {
+		return { ...nothingWritten, settled, dropped: 0 };
+	}
+
+	await client.run(DROP_ATTACHMENTS_SQL, [strandedBefore]);
+	const dropped = await client.run(DROP_MESSAGES_SQL, [strandedBefore]);
+
+	return { ...nothingWritten, settled, dropped };
 };
 
 export const formatStrandedSentReport = (
@@ -98,12 +161,31 @@ export const formatStrandedSentReport = (
 	if (report.stranded === 0) {
 		return ["No sent message is stranded in the outbox"];
 	}
+
+	const lines: string[] = [];
 	if (report.mode === "check") {
-		return [
-			`${report.stranded} sent message(s) stranded in the outbox, would be marked unfiled`,
-		];
+		if (report.neverFiled > 0) {
+			lines.push(
+				`${report.neverFiled} sent message(s) stranded in the outbox, would be marked unfiled`,
+			);
+		}
+		if (report.filed > 0) {
+			lines.push(
+				`${report.filed} sent message(s) already filed in Sent, their outbox row would be dropped`,
+			);
+		}
+		return lines;
+	}
+
+	if (report.neverFiled > 0) {
+		lines.push(
+			`${report.settled} of ${report.neverFiled} stranded sent message(s) marked unfiled`,
+		);
+	}
+	if (report.filed > 0) {
+		lines.push(
+			`${report.dropped} of ${report.filed} already-filed sent message(s) dropped from the outbox`,
+		);
 	}
-	return [
-		`${report.settled} of ${report.stranded} stranded sent message(s) marked unfiled`,
-	];
+	return lines;
 };

package/src/repos/i4-outbox-message.ts

@@ -37,6 +37,7 @@ function rowToOutboxMessage(
 		lastSmtpCode: row.lastSmtpCode ?? undefined,
 		sentAt: row.sentAt ?? undefined,
 		smtpMessageId: row.smtpMessageId ?? undefined,
+		appendedUid: row.appendedUid,
 		createdAt: row.createdAt,
 		updatedAt: row.updatedAt,
 	};
@@ -139,6 +140,8 @@ export class OutboxMessageRepo implements IOutboxMessageRepository {
 		if (input.sentAt !== undefined) updates.sentAt = input.sentAt;
 		if (input.smtpMessageId !== undefined)
 			updates.smtpMessageId = input.smtpMessageId;
+		if (input.appendedUid !== undefined)
+			updates.appendedUid = input.appendedUid;
 		if (input.toAddresses !== undefined)
 			updates.toAddresses = input.toAddresses;
 		if (input.ccAddresses !== undefined)