@remit/drizzle-service 0.0.76 → 0.0.78

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.76",
+  "version": "0.0.78",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/outbox-created-cleanup.sqlite.test.ts

@@ -0,0 +1,83 @@
+import assert from "node:assert/strict";
+import { describe, test } from "node:test";
+import Database from "better-sqlite3";
+import {
+	applyMigration,
+	migrationJournal,
+} from "./test-shipped-sqlite-schema.js";
+
+/**
+ * The data migration that clears the `message.created` backlog (reader#1063),
+ * against a database that already holds those rows.
+ *
+ * Every other outbox test starts from an empty table, so the case that matters
+ * — an instance upgrading with 30,906 undrained rows the drain filters out and
+ * the column's type no longer admits — is only reachable by running the shipped
+ * migrations over a table populated at the point the upgrade finds it.
+ */
+
+const CLEANUP_TAG = "0022_drop_message_created_outbox";
+
+const seed = (sqlite: Database.Database, event: string, id: string): void => {
+	sqlite
+		.prepare(
+			`INSERT INTO outbox (id, message_id, event, payload, created_at)
+			 VALUES (?, ?, ?, ?, ?)`,
+		)
+		.run(id, `message-${id}`, event, JSON.stringify({ messageId: id }), 1);
+};
+
+const unprocessed = (sqlite: Database.Database): string[] =>
+	(
+		sqlite
+			.prepare(
+				"SELECT event FROM outbox WHERE processed_at IS NULL ORDER BY event",
+			)
+			.all() as Array<{ event: string }>
+	).map((row) => row.event);
+
+const upgradedDatabase = (): Database.Database => {
+	const entries = [...migrationJournal()].sort(
+		(left, right) => left.idx - right.idx,
+	);
+	const cleanup = entries.findIndex((entry) => entry.tag === CLEANUP_TAG);
+	assert.notEqual(cleanup, -1, `${CLEANUP_TAG} is not in the journal`);
+
+	const sqlite = new Database(":memory:");
+	for (const entry of entries.slice(0, cleanup)) {
+		applyMigration(sqlite, entry.tag);
+	}
+
+	seed(sqlite, "message.created", "created-1");
+	seed(sqlite, "message.created", "created-2");
+	seed(sqlite, "message.body_synced", "synced-1");
+	seed(sqlite, "message.moved", "moved-1");
+
+	for (const entry of entries.slice(cleanup)) {
+		applyMigration(sqlite, entry.tag);
+	}
+	return sqlite;
+};
+
+describe("the message.created outbox cleanup", () => {
+	test("clears the rows an upgrading instance already has", () => {
+		const sqlite = upgradedDatabase();
+
+		const left = sqlite
+			.prepare("SELECT COUNT(*) AS n FROM outbox WHERE event = ?")
+			.get("message.created") as { n: number };
+
+		assert.equal(left.n, 0);
+		sqlite.close();
+	});
+
+	test("leaves the drained kinds alone", () => {
+		const sqlite = upgradedDatabase();
+
+		assert.deepEqual(unprocessed(sqlite), [
+			"message.body_synced",
+			"message.moved",
+		]);
+		sqlite.close();
+	});
+});

package/src/repair/thread-message-category-contract.ts

@@ -470,7 +470,7 @@ export const describeRepairContract = (
 				"ahead:",
 				"fan-out:",
 				"orphans:",
-				"not-yet-classified:",
+				"uncategorized:",
 				"divergent per mailbox:",
 				"category tally:",
 			]) {

package/src/repair/thread-message-category.ts

@@ -53,10 +53,9 @@
  *     `message.category` is still `uncategorized`, so the pending exclusion
  *     above refuses the row outright.
  *
- * Only the fourth is interesting, and it is not impossible. It is unreachable
- * through `backfillClassification`, which returns early once the category is
- * decided. It is reachable through the primary path, which is re-enterable: the
- * skip guard is `if (message.bodyStorageKey && !force)` (`body-sync.ts`), and
+ * Only the fourth is interesting, and it is not impossible. It is reachable
+ * through the one path that classifies, which is re-enterable: the skip guard
+ * is `if (hasStoredBody(message.bodyStorageKey) && !force)` (`body-sync.ts`), and
  * `force` is a live event flag — the read-miss re-arm cue, resolved in the
  * imap-worker's `sync-message-body.ts`. A forced re-fetch re-runs
  * `classifyByHeaders` over the same bytes, so it normally rewrites the value it
@@ -351,7 +350,7 @@ export const formatCheckReport = (
 	`  ahead: ${report.ahead} ${plural(report.ahead)} classified against a pending message — after #326 body-sync writes the row before the message, so this is a classification in flight. Not repaired: pushing it back to pending would undo a correct classification and serve Unclassified for mail that is already classified. Expected non-zero on a live instance mid-sync, zero on a quiescent one.`,
 	`fan-out: ${report.fanOutMessages} messages holding ${report.fanOutRows} thread_message rows — the multi-row shape #326 hardens against. Expected zero: deriveMessageId and deriveThreadMessageId are both mailbox-independent, so a message in two mailboxes collapses to one row, and the reachable case is thread-root drift.`,
 	`orphans: ${report.orphanRows} ${plural(report.orphanRows)} with no message row — not repaired, nothing to copy. Expected zero.`,
-	`not-yet-classified: ${report.notYetClassified} ${plural(report.notYetClassified)} pending against a pending message — not classified yet. Not a defect, not touched by the repair, and unchanged by it. Expected non-zero on a live instance.`,
+	`uncategorized: ${report.notYetClassified} ${plural(report.notYetClassified)} uncategorized against an uncategorized message — the row agrees with its message, so there is nothing to repair and this figure is untouched by the repair. It is NOT the same as "not classified yet": it mixes mail the classifier has not reached with mail it reached and had nothing to say about. Only \`message.classification_state\` separates them (\`SELECT classification_state, count(*) FROM message WHERE category = 'uncategorized' GROUP BY 1\`), and a large \`NotExamined\` count is the cohort worth acting on. Expected non-zero on a live instance.`,
 	`divergent per mailbox: ${
 		report.divergentByMailbox.length === 0
 			? "none"

package/src/repos/message.sqlite.test.ts

@@ -52,7 +52,7 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		await close();
 	});
 
-	test("create returns a MessageItem and writes one outbox row atomically", async () => {
+	test("create returns a MessageItem and writes no outbox row", async () => {
 		const item = await repo.create(BASE_INPUT);
 		assert.equal(item.messageId, MESSAGE_ID);
 		assert.equal(item.status, "active");
@@ -62,9 +62,11 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 			.select()
 			.from(outboxTable)
 			.where(eq(outboxTable.messageId, MESSAGE_ID));
-		assert.equal(rows.length, 1);
-		assert.equal(rows[0].event, "message.created");
-		assert.deepEqual(rows[0].payload, { messageId: MESSAGE_ID });
+		assert.equal(
+			rows.length,
+			0,
+			"a message with no body yet has nothing to index",
+		);
 	});
 
 	test("boolean and json columns round-trip", async () => {
@@ -120,7 +122,7 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		assert.equal(afterUpdate.placementVerdict?.action, "MoveToInbox");
 	});
 
-	test("duplicate messageId throws CreateFailedConflictError and rolls back the outbox row", async () => {
+	test("duplicate messageId throws CreateFailedConflictError and appends no outbox row", async () => {
 		const before = await db
 			.select()
 			.from(outboxTable)

package/src/repos/message.test.ts

@@ -44,7 +44,7 @@ describe("DrizzleMessageRepository", () => {
 		await stop();
 	});
 
-	describe("create — writes message + outbox row in one transaction", () => {
+	describe("create — writes the message row only", () => {
 		test("creates a message and returns MessageItem", async () => {
 			const item = await messageRepo.create(BASE_MESSAGE_INPUT);
 			assert.equal(item.messageId, MESSAGE_ID);
@@ -56,16 +56,14 @@ describe("DrizzleMessageRepository", () => {
 			assert.ok(typeof item.updatedAt === "number");
 		});
 
-		test("writes an outbox row in the same transaction", async () => {
+		test("writes no outbox row — there is nothing to index before the body", async () => {
 			const { outboxTable } = await import("../schema/message-data.js");
 			const { eq } = await import("drizzle-orm");
 			const rows = await db
 				.select()
 				.from(outboxTable)
 				.where(eq(outboxTable.messageId, MESSAGE_ID));
-			assert.ok(rows.length >= 1, "should have at least 1 outbox row");
-			assert.equal(rows[0].event, "message.created");
-			assert.deepStrictEqual(rows[0].payload, { messageId: MESSAGE_ID });
+			assert.equal(rows.length, 0);
 		});
 
 		test("duplicate messageId throws CreateFailedConflictError and writes no extra outbox row", async () => {

package/src/repos/message.ts

@@ -59,6 +59,7 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
 		status: row.status,
 		syncStatus: row.syncStatus,
 		category: row.category,
+		classificationState: row.classificationState,
 		authenticityVerdict: row.authenticityVerdict,
 		hasListUnsubscribe: row.hasListUnsubscribe,
 		movedByRemit: row.movedByRemit,
@@ -106,7 +107,7 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
  * the search-index worker relays a search-index REMOVE and the vectors are
  * dropped.
  */
-export const MESSAGE_REMOVED_EVENT = "message.removed";
+export const MESSAGE_REMOVED_EVENT = "message.removed" as const;
 
 export type SubtreeDb = Pick<Db<Record<string, unknown>>, "delete" | "insert">;
 
@@ -185,6 +186,8 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			status: input.status ?? ("active" as const),
 			syncStatus: input.syncStatus ?? ("pending" as const),
 			category: input.category ?? ("uncategorized" as const),
+			classificationState:
+				input.classificationState ?? ("NotExamined" as const),
 			authenticityVerdict:
 				input.authenticityVerdict ?? ("NotEvaluated" as const),
 			hasListUnsubscribe: input.hasListUnsubscribe ?? false,
@@ -204,22 +207,15 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			updatedAt: now,
 		};
 
-		// Faithful to ElectroDB message.create: a duplicate messageId throws
-		// CreateFailedConflictError. The plain insert raises a unique-constraint
-		// violation, which rolls back the transaction so NO outbox row is
-		// written; we surface it as the domain conflict error.
+		// Faithful to ElectroDB message.create: a duplicate messageId raises a
+		// unique-constraint violation, surfaced as the domain conflict error.
+		//
+		// No outbox event. A freshly created message has neither a body nor a
+		// threadMessage yet, so there is nothing to index — the search-index
+		// relay never drained a creation event, and the rows accumulated forever
+		// (reader#1063). Body-sync appends the event once there is content.
 		try {
-			await runInTransaction(this.db, async (tx) => {
-				await tx.insert(messageTable).values(row);
-
-				await tx.insert(outboxTable).values({
-					id: randomUUID(),
-					messageId: input.messageId,
-					event: "message.created",
-					payload: { messageId: input.messageId },
-					createdAt: new Date(),
-				});
-			});
+			await this.db.insert(messageTable).values(row);
 		} catch (error) {
 			if (isUniqueViolation(error)) {
 				throw new CreateFailedConflictError("Message", input);
@@ -368,6 +364,9 @@ export class DrizzleMessageRepository implements IMessageRepository {
 				? { syncStatus: input.syncStatus }
 				: {}),
 			...(input.category !== undefined ? { category: input.category } : {}),
+			...(input.classificationState !== undefined
+				? { classificationState: input.classificationState }
+				: {}),
 			...(input.authenticityVerdict !== undefined
 				? { authenticityVerdict: input.authenticityVerdict }
 				: {}),

package/src/repos/serialized-writes.sqlite.test.ts

@@ -21,7 +21,7 @@ import { runInTransaction, serializeSqliteWrites } from "../tx.js";
 const row = (id: string) => ({
 	id,
 	messageId: `msg-${id}`,
-	event: "message.body_synced",
+	event: "message.body_synced" as const,
 	payload: { messageId: `msg-${id}` },
 	createdAt: new Date(),
 });

package/src/repos/unit-of-work.test.ts

@@ -22,7 +22,7 @@ describe("DrizzleUnitOfWork", () => {
 	const MAILBOX_ID = "00000000-0000-0000-2222-000000000002";
 	const NOW = 1700000000000;
 
-	const writeEnvelopeAndMessage = async (
+	const writeEnvelopeMessageAndBody = async (
 		repos: Parameters<Parameters<DrizzleUnitOfWork["transaction"]>[0]>[0],
 	) => {
 		await repos.envelope.upsertEnvelope({
@@ -43,6 +43,10 @@ describe("DrizzleUnitOfWork", () => {
 			envelopeId: deriveEnvelopeId(MESSAGE_ID),
 			rootBodyPartId: deriveRootBodyPartId(MESSAGE_ID),
 		});
+		// Body-sync appends the transactional-outbox row; the create emits none.
+		await repos.message.update(MESSAGE_ID, {
+			bodyStorageKey: "body/hello.json",
+		});
 	};
 
 	const rows = async () => {
@@ -74,7 +78,7 @@ describe("DrizzleUnitOfWork", () => {
 		await assert.rejects(
 			() =>
 				unitOfWork.transaction(async (repos) => {
-					await writeEnvelopeAndMessage(repos);
+					await writeEnvelopeMessageAndBody(repos);
 					// A later write in the set fails after the envelope, message and its
 					// transactional-outbox row have already been written.
 					throw new Error("thread write failed");
@@ -93,13 +97,13 @@ describe("DrizzleUnitOfWork", () => {
 	});
 
 	test("a successful transaction commits the data rows and the outbox row", async () => {
-		await unitOfWork.transaction(writeEnvelopeAndMessage);
+		await unitOfWork.transaction(writeEnvelopeMessageAndBody);
 
 		const { envelopes, messages, outbox } = await rows();
 		assert.equal(envelopes.length, 1);
 		assert.equal(messages.length, 1);
 		assert.equal(outbox.length, 1);
-		assert.equal(outbox[0].event, "message.created");
+		assert.equal(outbox[0].event, "message.body_synced");
 		assert.deepStrictEqual(outbox[0].payload, { messageId: MESSAGE_ID });
 	});
 });

package/src/schema/message-data.ts

@@ -1,7 +1,7 @@
 import * as entities from "@remit/drizzle-sqlite-schema";
-import { outboxTable } from "./outbox.js";
+import { OUTBOX_EVENTS, type OutboxEvent, outboxTable } from "./outbox.js";
 
-export { outboxTable };
+export { OUTBOX_EVENTS, type OutboxEvent, outboxTable };
 
 const bodyPartContentTable = entities.bodyPartContents;
 const bodyPartParameterTable = entities.bodyPartParameters;

package/src/schema/outbox.ts

@@ -10,12 +10,27 @@ import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
  * Exported raw for committed-migration generation (schema-full-sqlite) as well
  * as for the repos and the drain logic.
  */
+/**
+ * The event vocabulary a producer may append. Every kind here is drained by the
+ * search-index relay (`DRAIN_EVENTS`); a kind nothing drains accumulates
+ * undrained rows forever, which is what `message.created` did (reader#1063).
+ * The column carries the union so a new producer cannot invent a kind without
+ * adding it here, where the drain-coverage test picks it up.
+ */
+export const OUTBOX_EVENTS = [
+	"message.body_synced",
+	"message.moved",
+	"message.removed",
+] as const;
+
+export type OutboxEvent = (typeof OUTBOX_EVENTS)[number];
+
 export const outboxTable = sqliteTable(
 	"outbox",
 	{
 		id: text("id").primaryKey(),
 		messageId: text("message_id").notNull(),
-		event: text("event").notNull(),
+		event: text("event", { enum: OUTBOX_EVENTS }).notNull(),
 		payload: text("payload", { mode: "json" }).notNull(),
 		createdAt: integer("created_at", { mode: "timestamp_ms" })
 			.$defaultFn(() => new Date())