@remit/drizzle-service 0.0.1

package/src/repos/envelope.sqlite.test.ts

@@ -0,0 +1,94 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import {
+	type MessageDataSchema,
+	messageDataSchema,
+} from "../schema/message-data.js";
+import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
+import { DrizzleEnvelopeRepository } from "./envelope.js";
+
+// Envelope repo on sqlite (RFC 036 D1): the onConflictDoUpdate upsert path and
+// the transactional body-part upsert both run on better-sqlite3 unchanged.
+
+const MESSAGE_ID = "00000000-0000-0000-4444-000000000001";
+
+describe("DrizzleEnvelopeRepository (sqlite)", () => {
+	let db: SqliteTestDb<MessageDataSchema>;
+	let close: () => Promise<void>;
+	let repo: DrizzleEnvelopeRepository;
+
+	before(async () => {
+		({ db, close } = await createSqliteTestDb(messageDataSchema));
+		repo = new DrizzleEnvelopeRepository(db);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	test("upsertEnvelope inserts then updates on conflict", async () => {
+		const created = await repo.upsertEnvelope({
+			envelopeId: "ignored-derived",
+			messageId: MESSAGE_ID,
+			dateValue: 1700000000000,
+			dateRaw: "Mon, 14 Nov 2023 08:00:00 +0000",
+			subject: "First",
+		});
+		assert.equal(created.subject, "First");
+
+		const updated = await repo.upsertEnvelope({
+			envelopeId: "ignored-derived",
+			messageId: MESSAGE_ID,
+			dateValue: 1700000000000,
+			dateRaw: "Mon, 14 Nov 2023 08:00:00 +0000",
+			subject: "Second",
+		});
+		assert.equal(updated.envelopeId, created.envelopeId);
+		assert.equal(updated.subject, "Second");
+	});
+
+	test("upsertBodyParts writes parts and parameters in one transaction", async () => {
+		await repo.upsertBodyParts(MESSAGE_ID, [
+			{
+				partPath: "1",
+				parentPartPath: null,
+				mediaType: "TEXT",
+				mediaSubtype: "plain",
+				transferEncoding: "7BIT",
+				sizeOctets: 128,
+				isMultipart: false,
+				parameters: [{ parameterName: "charset", parameterValue: "utf-8" }],
+			},
+		]);
+
+		const data = await repo.getMessageData(MESSAGE_ID);
+		assert.ok(data.bodyPart.length >= 1);
+		assert.ok(
+			data.bodyPartParameter.some(
+				(p) => p.parameterName === "charset" && p.parameterValue === "utf-8",
+			),
+		);
+	});
+
+	test("re-upserting the same partPath does not duplicate the row", async () => {
+		await repo.upsertBodyParts(MESSAGE_ID, [
+			{
+				partPath: "1",
+				parentPartPath: null,
+				mediaType: "TEXT",
+				mediaSubtype: "plain",
+				transferEncoding: "7BIT",
+				sizeOctets: 256,
+				isMultipart: false,
+				parameters: [],
+			},
+		]);
+		const data = await repo.getMessageData(MESSAGE_ID);
+		const parts = data.bodyPart.filter((p) => p.partPath === "1");
+		assert.equal(
+			parts.length,
+			1,
+			"conflict update must not insert a second row",
+		);
+	});
+});

package/src/repos/envelope.test.ts

@@ -0,0 +1,225 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import type { BodyPartUpsertInput } from "@remit/data-ports";
+import { envelopeId as deriveEnvelopeId } from "../id.js";
+import { DrizzleEnvelopeRepository } from "./envelope.js";
+import { createTestDb, type TestDb } from "./test-helpers.js";
+
+describe("DrizzleEnvelopeRepository", () => {
+	let db: TestDb;
+	let stop: () => Promise<void>;
+	let repo: DrizzleEnvelopeRepository;
+
+	const MESSAGE_ID = "00000000-0000-0000-0000-000000000001";
+	const ENVELOPE_ID = deriveEnvelopeId(MESSAGE_ID);
+	const ADDRESS_ID = "00000000-0000-0000-0000-000000000003";
+	const ENVELOPE_ADDR_ID = "00000000-0000-0000-0000-000000000004";
+	const NOW = 1700000000000;
+
+	before(async () => {
+		({ db, stop } = await createTestDb());
+		repo = new DrizzleEnvelopeRepository(db);
+
+		await repo.upsertEnvelope({
+			envelopeId: ENVELOPE_ID,
+			messageId: MESSAGE_ID,
+			dateValue: NOW,
+			dateRaw: "Mon, 14 Nov 2023 08:00:00 +0000",
+			subject: "Hello World",
+			messageIdValue: "<test@example.com>",
+		});
+	});
+
+	after(async () => {
+		await stop();
+	});
+
+	describe("getMessageData — MessageData shape (8 members)", () => {
+		test("returns all 8 collection members", async () => {
+			const data = await repo.getMessageData(MESSAGE_ID);
+			const keys = Object.keys(data).sort();
+			assert.deepStrictEqual(keys, [
+				"bodyPart",
+				"bodyPartContent",
+				"bodyPartParameter",
+				"bodyPartStorage",
+				"envelope",
+				"envelopeAddress",
+				"messageReference",
+				"rawMessageStorage",
+			]);
+		});
+
+		test("envelope has 1 row with correct fields", async () => {
+			const data = await repo.getMessageData(MESSAGE_ID);
+			assert.equal(data.envelope.length, 1);
+			const env = data.envelope[0];
+			assert.equal(env.envelopeId, ENVELOPE_ID);
+			assert.equal(env.messageId, MESSAGE_ID);
+			assert.equal(env.dateValue, NOW);
+			assert.equal(env.dateRaw, "Mon, 14 Nov 2023 08:00:00 +0000");
+			assert.equal(env.subject, "Hello World");
+			assert.equal(env.messageIdValue, "<test@example.com>");
+		});
+
+		test("empty collections return empty arrays", async () => {
+			const data = await repo.getMessageData(MESSAGE_ID);
+			assert.equal(data.messageReference.length, 0);
+			assert.equal(data.envelopeAddress.length, 0);
+			assert.equal(data.bodyPart.length, 0);
+			assert.equal(data.bodyPartParameter.length, 0);
+			assert.equal(data.rawMessageStorage.length, 0);
+			assert.equal(data.bodyPartStorage.length, 0);
+			assert.equal(data.bodyPartContent.length, 0);
+		});
+
+		test("throws NotFoundError for unknown messageId", async () => {
+			await assert.rejects(
+				() => repo.getMessageData("00000000-0000-0000-0000-999999999999"),
+				(err: Error) => {
+					assert.equal(err.name, "NotFoundError");
+					return true;
+				},
+			);
+		});
+	});
+
+	describe("upsertBodyParts — idempotent single-transaction write", () => {
+		const PART: BodyPartUpsertInput = {
+			partPath: "1",
+			parentPartPath: null,
+			mediaType: "TEXT",
+			mediaSubtype: "plain",
+			transferEncoding: "7BIT",
+			sizeOctets: 42,
+			isMultipart: false,
+			parameters: [{ parameterName: "charset", parameterValue: "utf-8" }],
+		};
+
+		test("upserts body parts and parameters", async () => {
+			await repo.upsertBodyParts(MESSAGE_ID, [PART]);
+			const parts = await repo.listBodyParts(MESSAGE_ID);
+			assert.equal(parts.length, 1);
+			assert.equal(parts[0].partPath, "1");
+			assert.equal(parts[0].mediaType, "TEXT");
+		});
+
+		test("idempotent — second upsert does not duplicate", async () => {
+			await repo.upsertBodyParts(MESSAGE_ID, [PART]);
+			await repo.upsertBodyParts(MESSAGE_ID, [PART]);
+			const parts = await repo.listBodyParts(MESSAGE_ID);
+			assert.equal(
+				parts.length,
+				1,
+				"should still have exactly 1 body part after re-upsert",
+			);
+		});
+
+		test("duplicate partPath in one call does not throw and writes one row", async () => {
+			// The MIME walker can hand back two nodes with the same synthetic
+			// partPath (e.g. the inner body of a message/rfc822 attachment),
+			// yielding duplicate bodyPartIds in a single insert. A single
+			// INSERT ... ON CONFLICT DO UPDATE would raise SQLSTATE 21000 unless
+			// we dedupe first.
+			const DUP_MSG_ID = "00000000-0000-0000-0000-000000000077";
+			const duped: BodyPartUpsertInput[] = [
+				{
+					partPath: "2",
+					parentPartPath: null,
+					mediaType: "MESSAGE",
+					mediaSubtype: "rfc822",
+					transferEncoding: "7BIT",
+					sizeOctets: 10,
+					isMultipart: false,
+					parameters: [{ parameterName: "charset", parameterValue: "utf-8" }],
+				},
+				{
+					partPath: "2",
+					parentPartPath: null,
+					mediaType: "TEXT",
+					mediaSubtype: "plain",
+					transferEncoding: "7BIT",
+					sizeOctets: 20,
+					isMultipart: false,
+					parameters: [{ parameterName: "charset", parameterValue: "latin1" }],
+				},
+			];
+
+			await assert.doesNotReject(() => repo.upsertBodyParts(DUP_MSG_ID, duped));
+
+			const parts = await repo.listBodyParts(DUP_MSG_ID);
+			const matching = parts.filter((p) => p.partPath === "2");
+			assert.equal(
+				matching.length,
+				1,
+				"duplicate partPath must collapse to exactly one row",
+			);
+		});
+	});
+
+	describe("deleteManyEnvelopes — single-transaction delete", () => {
+		const DEL_MSG_ID = "00000000-0000-0000-0000-000000000099";
+		const DEL_ENV_ID = deriveEnvelopeId(DEL_MSG_ID);
+
+		before(async () => {
+			await repo.upsertEnvelope({
+				envelopeId: DEL_ENV_ID,
+				messageId: DEL_MSG_ID,
+				dateValue: NOW,
+				dateRaw: "Mon, 14 Nov 2023 08:00:00 +0000",
+			});
+		});
+
+		test("deletes the specified envelopes", async () => {
+			await repo.deleteManyEnvelopes([DEL_ENV_ID]);
+			await assert.rejects(
+				() => repo.getMessageData(DEL_MSG_ID),
+				(err: Error) => {
+					assert.equal(err.name, "NotFoundError");
+					return true;
+				},
+			);
+		});
+
+		test("no-op on empty array", async () => {
+			await assert.doesNotReject(() => repo.deleteManyEnvelopes([]));
+		});
+	});
+
+	describe("envelopeAddress — verifies envelopeAddress in getMessageData", () => {
+		before(async () => {
+			const { envelopeAddressTable } = await import(
+				"../schema/message-data.js"
+			);
+			await db
+				.insert(envelopeAddressTable)
+				.values({
+					envelopeAddressId: ENVELOPE_ADDR_ID,
+					messageId: MESSAGE_ID,
+					addressId: ADDRESS_ID,
+					normalizedEmail: "alice@example.com",
+					displayName: "Alice",
+					addressRole: "from",
+					addressOrder: 0,
+					createdAt: NOW,
+					updatedAt: NOW,
+				})
+				.onConflictDoNothing();
+		});
+
+		test("getMessageData includes envelopeAddress rows", async () => {
+			const data = await repo.getMessageData(MESSAGE_ID);
+			assert.ok(
+				data.envelopeAddress.length >= 1,
+				"should have at least one address",
+			);
+			const addr = data.envelopeAddress.find(
+				(a) => a.envelopeAddressId === ENVELOPE_ADDR_ID,
+			);
+			assert.ok(addr, "should find the inserted address");
+			assert.equal(addr.normalizedEmail, "alice@example.com");
+			assert.equal(addr.displayName, "Alice");
+			assert.equal(addr.addressRole, "from");
+		});
+	});
+});

package/src/repos/envelope.ts

@@ -0,0 +1,342 @@
+import type {
+	BodyPartContentUpsertInput,
+	BodyPartItem,
+	BodyPartUpsertInput,
+	CreateEnvelopeInput,
+	EnvelopeItem,
+	IEnvelopeRepository,
+	MessageData,
+	UpdateEnvelopeInput,
+} from "@remit/data-ports";
+import { eq, inArray } from "drizzle-orm";
+import type { Db } from "../db.js";
+import {
+	CreateFailedConflictError,
+	isUniqueViolation,
+	NotFoundError,
+} from "../error.js";
+import {
+	bodyPartContentId as deriveBodyPartContentId,
+	bodyPartId as deriveBodyPartId,
+	bodyPartParameterId as deriveBodyPartParameterId,
+	envelopeId as deriveEnvelopeId,
+} from "../id.js";
+import {
+	bodyPartContentTable,
+	bodyPartParameterTable,
+	bodyPartStorageTable,
+	bodyPartTable,
+	envelopeAddressTable,
+	envelopeTable,
+	type MessageDataSchema,
+	messageReferenceTable,
+	rawMessageStorageTable,
+} from "../schema/message-data.js";
+import { runInTransaction } from "../tx.js";
+import {
+	toBodyPartContentItem,
+	toBodyPartItem,
+	toBodyPartParameterItem,
+	toBodyPartStorageItem,
+	toEnvelopeAddressItem,
+	toEnvelopeItem,
+	toMessageReferenceItem,
+	toRawMessageStorageItem,
+} from "./mappers.js";
+
+type DB = Db<MessageDataSchema>;
+
+function dedupeById<T>(rows: T[], getId: (row: T) => string): T[] {
+	const byId = new Map<string, T>();
+	for (const row of rows) {
+		byId.set(getId(row), row);
+	}
+	return [...byId.values()];
+}
+
+export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
+	constructor(private db: DB) {}
+
+	async getMessageData(messageId: string): Promise<MessageData> {
+		const [
+			envelopes,
+			messageReferences,
+			envelopeAddresses,
+			bodyParts,
+			bodyPartParameters,
+			rawMessageStorages,
+			bodyPartStorages,
+			bodyPartContents,
+		] = await Promise.all([
+			this.db
+				.select()
+				.from(envelopeTable)
+				.where(eq(envelopeTable.messageId, messageId)),
+			this.db
+				.select()
+				.from(messageReferenceTable)
+				.where(eq(messageReferenceTable.messageId, messageId)),
+			this.db
+				.select()
+				.from(envelopeAddressTable)
+				.where(eq(envelopeAddressTable.messageId, messageId)),
+			this.db
+				.select()
+				.from(bodyPartTable)
+				.where(eq(bodyPartTable.messageId, messageId)),
+			this.db
+				.select()
+				.from(bodyPartParameterTable)
+				.where(eq(bodyPartParameterTable.messageId, messageId)),
+			this.db
+				.select()
+				.from(rawMessageStorageTable)
+				.where(eq(rawMessageStorageTable.messageId, messageId)),
+			this.db
+				.select()
+				.from(bodyPartStorageTable)
+				.where(eq(bodyPartStorageTable.messageId, messageId)),
+			this.db
+				.select()
+				.from(bodyPartContentTable)
+				.where(eq(bodyPartContentTable.messageId, messageId)),
+		]);
+
+		if (envelopes.length === 0) {
+			throw new NotFoundError(`Message data not found: ${messageId}`);
+		}
+
+		return {
+			envelope: envelopes.map(toEnvelopeItem),
+			messageReference: messageReferences.map(toMessageReferenceItem),
+			envelopeAddress: envelopeAddresses.map(toEnvelopeAddressItem),
+			bodyPart: bodyParts.map(toBodyPartItem),
+			bodyPartParameter: bodyPartParameters.map(toBodyPartParameterItem),
+			rawMessageStorage: rawMessageStorages.map(toRawMessageStorageItem),
+			bodyPartStorage: bodyPartStorages.map(toBodyPartStorageItem),
+			bodyPartContent: bodyPartContents.map(toBodyPartContentItem),
+		};
+	}
+
+	async upsertBodyParts(
+		messageId: string,
+		parts: BodyPartUpsertInput[],
+	): Promise<void> {
+		if (parts.length === 0) return;
+
+		const now = Date.now();
+
+		const bodyPartRows = parts.map((part) => ({
+			bodyPartId: deriveBodyPartId(messageId, part.partPath),
+			messageId,
+			partPath: part.partPath,
+			mediaType: part.mediaType,
+			mediaSubtype: part.mediaSubtype,
+			transferEncoding: part.transferEncoding,
+			sizeOctets: part.sizeOctets,
+			isMultipart: part.isMultipart,
+			parentBodyPartId:
+				part.parentPartPath !== null
+					? deriveBodyPartId(messageId, part.parentPartPath)
+					: null,
+			contentId: part.contentId ?? null,
+			contentDescription: part.contentDescription ?? null,
+			lineCount: part.lineCount ?? null,
+			md5Hash: part.md5Hash ?? null,
+			disposition: part.disposition ?? null,
+			dispositionFilename: part.dispositionFilename ?? null,
+			language: part.language ?? null,
+			location: part.location ?? null,
+			multipartSubtype: part.multipartSubtype ?? null,
+			createdAt: now,
+			updatedAt: now,
+		}));
+
+		const parameterRows = parts.flatMap((part) =>
+			part.parameters.map((param) => ({
+				bodyPartParameterId: deriveBodyPartParameterId(
+					messageId,
+					part.partPath,
+					param.parameterName,
+				),
+				messageId,
+				bodyPartId: deriveBodyPartId(messageId, part.partPath),
+				parameterName: param.parameterName,
+				parameterValue: param.parameterValue,
+				createdAt: now,
+				updatedAt: now,
+			})),
+		);
+
+		// The MIME walker assigns synthetic partPaths for nodes imapflow returns
+		// without a `part` field (e.g. the inner body of a message/rfc822
+		// attachment), which can produce duplicate ids. A single
+		// INSERT ... ON CONFLICT DO UPDATE fails (SQLSTATE 21000) if the same
+		// conflict-target id appears twice, so dedupe first (last-write-wins),
+		// matching the ElectroDB `seen` guard.
+		const dedupedBodyPartRows = dedupeById(bodyPartRows, (r) => r.bodyPartId);
+		const dedupedParameterRows = dedupeById(
+			parameterRows,
+			(r) => r.bodyPartParameterId,
+		);
+
+		await runInTransaction(this.db, async (tx) => {
+			await tx
+				.insert(bodyPartTable)
+				.values(dedupedBodyPartRows)
+				.onConflictDoUpdate({
+					target: bodyPartTable.bodyPartId,
+					set: { updatedAt: now },
+				});
+
+			if (dedupedParameterRows.length > 0) {
+				await tx
+					.insert(bodyPartParameterTable)
+					.values(dedupedParameterRows)
+					.onConflictDoUpdate({
+						target: bodyPartParameterTable.bodyPartParameterId,
+						set: {
+							parameterValue: bodyPartParameterTable.parameterValue,
+							updatedAt: now,
+						},
+					});
+			}
+		});
+	}
+
+	async deleteManyEnvelopes(envelopeIds: string[]): Promise<void> {
+		if (envelopeIds.length === 0) return;
+
+		await runInTransaction(this.db, async (tx) => {
+			await tx
+				.delete(envelopeTable)
+				.where(inArray(envelopeTable.envelopeId, envelopeIds));
+		});
+	}
+
+	async createEnvelope(input: CreateEnvelopeInput): Promise<EnvelopeItem> {
+		const now = Date.now();
+		const row = {
+			...input,
+			envelopeId: deriveEnvelopeId(input.messageId),
+			subject: input.subject ?? null,
+			messageIdValue: input.messageIdValue ?? null,
+			createdAt: now,
+			updatedAt: now,
+		};
+		try {
+			await this.db.insert(envelopeTable).values(row);
+		} catch (error) {
+			if (isUniqueViolation(error)) {
+				throw new CreateFailedConflictError("Envelope", input);
+			}
+			throw error;
+		}
+		return toEnvelopeItem(row as typeof envelopeTable.$inferSelect);
+	}
+
+	async upsertEnvelope(input: CreateEnvelopeInput): Promise<EnvelopeItem> {
+		const now = Date.now();
+		const row = {
+			...input,
+			envelopeId: deriveEnvelopeId(input.messageId),
+			subject: input.subject ?? null,
+			messageIdValue: input.messageIdValue ?? null,
+			createdAt: now,
+			updatedAt: now,
+		};
+		await this.db
+			.insert(envelopeTable)
+			.values(row)
+			.onConflictDoUpdate({
+				target: envelopeTable.envelopeId,
+				set: {
+					subject: row.subject,
+					messageIdValue: row.messageIdValue,
+					updatedAt: now,
+				},
+			});
+		return toEnvelopeItem(row as typeof envelopeTable.$inferSelect);
+	}
+
+	async getEnvelope(envelopeId: string): Promise<EnvelopeItem>;
+	async getEnvelope(envelopeIds: string[]): Promise<EnvelopeItem[]>;
+	async getEnvelope(
+		envelopeId: string | string[],
+	): Promise<EnvelopeItem | EnvelopeItem[]> {
+		if (Array.isArray(envelopeId)) {
+			if (envelopeId.length === 0) return [];
+			const rows = await this.db
+				.select()
+				.from(envelopeTable)
+				.where(inArray(envelopeTable.envelopeId, envelopeId));
+			return rows.map(toEnvelopeItem);
+		}
+
+		const rows = await this.db
+			.select()
+			.from(envelopeTable)
+			.where(eq(envelopeTable.envelopeId, envelopeId));
+		if (rows.length === 0) {
+			throw new NotFoundError(`Envelope not found: ${envelopeId}`);
+		}
+		return toEnvelopeItem(rows[0]);
+	}
+
+	async updateEnvelope(
+		envelopeId: string,
+		input: UpdateEnvelopeInput,
+	): Promise<EnvelopeItem> {
+		const now = Date.now();
+		await this.db
+			.update(envelopeTable)
+			.set({ ...input, updatedAt: now })
+			.where(eq(envelopeTable.envelopeId, envelopeId));
+		return this.getEnvelope(envelopeId);
+	}
+
+	async deleteEnvelope(envelopeId: string): Promise<void> {
+		await this.db
+			.delete(envelopeTable)
+			.where(eq(envelopeTable.envelopeId, envelopeId));
+	}
+
+	async upsertBodyPartContents(
+		messageId: string,
+		contents: BodyPartContentUpsertInput[],
+	): Promise<void> {
+		if (contents.length === 0) return;
+
+		const now = Date.now();
+
+		const rows = contents.map((c) => ({
+			bodyPartContentId: deriveBodyPartContentId(messageId, c.bodyPartId),
+			messageId,
+			bodyPartId: c.bodyPartId,
+			content: c.content,
+			contentLength: c.content.length,
+			createdAt: now,
+			updatedAt: now,
+		}));
+
+		await this.db
+			.insert(bodyPartContentTable)
+			.values(rows)
+			.onConflictDoUpdate({
+				target: bodyPartContentTable.bodyPartContentId,
+				set: {
+					content: bodyPartContentTable.content,
+					contentLength: bodyPartContentTable.contentLength,
+					updatedAt: now,
+				},
+			});
+	}
+
+	async listBodyParts(messageId: string): Promise<BodyPartItem[]> {
+		const rows = await this.db
+			.select()
+			.from(bodyPartTable)
+			.where(eq(bodyPartTable.messageId, messageId));
+		return rows.map(toBodyPartItem);
+	}
+}