@remit/drizzle-service 0.0.103 → 0.0.105

package/src/repos/calendar-put.sqlite.test.ts

@@ -1,210 +1,4 @@
-import assert from "node:assert/strict";
-import { after, before, describe, test } from "node:test";
-import {
-	prepareCalendarObject,
-	provisionDefaultCalendar,
-	putCalendarObject,
-} from "@remit/calendar-service";
-import type {
-	CalendarRepositories,
-	CalendarWrite,
-	ICalendarStore,
-} from "@remit/data-ports";
-import type { Db } from "../db.js";
-import {
-	calendarEventIndexTable,
-	calendarObjectTable,
-	calendarTable,
-} from "../schema.js";
-import { createSqliteRepositories } from "../sqlite-repositories.js";
-import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { openCalendarSqliteTestStores } from "../test-stores.js";
+import { calendarPutSuite } from "./calendar-put.suite.js";
 
-const ACCOUNT_CONFIG_ID = "account-config-1";
-
-const RESOURCE = [
-	"BEGIN:VCALENDAR",
-	"VERSION:2.0",
-	"BEGIN:VEVENT",
-	"UID:atomic@example.com",
-	"DTSTART:20260826T090000Z",
-	"DTEND:20260826T100000Z",
-	"SUMMARY:Quarterly review",
-	"RRULE:FREQ=WEEKLY;COUNT=3",
-	"END:VEVENT",
-	"END:VCALENDAR",
-	"",
-].join("\r\n");
-
-/**
- * Appends a write that fails to every commit: a resource put into a collection
- * that does not exist, which the store refuses at its last statement. Every
- * statement of the real write set has run by then, so a failure there is the
- * case that proves the boundary.
- */
-class FailingLastWriteStore implements ICalendarStore {
-	constructor(private inner: ICalendarStore) {}
-
-	get repos(): CalendarRepositories {
-		return this.inner.repos;
-	}
-
-	async commit(writes: readonly CalendarWrite[]): Promise<void> {
-		const orphan = await prepareCalendarObject(
-			{ timezone: "" },
-			{
-				accountConfigId: ACCOUNT_CONFIG_ID,
-				calendarId: "no-such-calendar",
-				resourceName: "orphan.ics",
-				icalData: RESOURCE,
-			},
-		);
-		assert.ok(orphan.ok);
-		await this.inner.commit([...writes, orphan.value]);
-	}
-}
-
-describe("the calendar write path against sqlite", () => {
-	let db: Db<Record<string, unknown>>;
-	let close: () => Promise<void>;
-	let store: ICalendarStore;
-	let calendarId: string;
-
-	before(async () => {
-		const created = await createSqliteTestDb({
-			calendars: calendarTable,
-			calendarObjects: calendarObjectTable,
-			calendarEventIndexes: calendarEventIndexTable,
-		});
-		db = created.db as unknown as Db<Record<string, unknown>>;
-		close = created.close;
-		store = createSqliteRepositories({
-			app: db,
-			mail: db,
-			search: db,
-		}).calendarStore;
-		const collection = await provisionDefaultCalendar(store, ACCOUNT_CONFIG_ID);
-		calendarId = collection.calendarId;
-	});
-
-	after(() => close());
-
-	const objects = () => db.select().from(calendarObjectTable);
-	const occurrences = () => db.select().from(calendarEventIndexTable);
-
-	test("writes the object, its occurrences and the sequence bump together", async () => {
-		const result = await putCalendarObject(store, {
-			accountConfigId: ACCOUNT_CONFIG_ID,
-			calendarId,
-			resourceName: "review.ics",
-			icalData: RESOURCE,
-		});
-
-		assert.ok(result.ok);
-		assert.equal(result.value.icalData, RESOURCE);
-		assert.equal(
-			(await objects())[0]?.icalData,
-			RESOURCE,
-			"the stored row holds the input bytes, CRLF and all",
-		);
-		assert.equal(result.value.syncSequence, 1);
-		assert.equal((await objects()).length, 1);
-		assert.equal(
-			(await occurrences()).filter(
-				(row) => row.calendarObjectId === result.value.calendarObjectId,
-			).length,
-			3,
-		);
-	});
-
-	test("a failing write in the batch leaves no object behind", async () => {
-		const before = await objects();
-
-		await assert.rejects(
-			putCalendarObject(new FailingLastWriteStore(store), {
-				accountConfigId: ACCOUNT_CONFIG_ID,
-				calendarId,
-				resourceName: "doomed.ics",
-				icalData: RESOURCE,
-			}),
-			/NOT NULL constraint failed/,
-		);
-
-		const after = await objects();
-		assert.equal(after.length, before.length);
-		assert.equal(
-			after.some((row) => row.resourceName === "doomed.ics"),
-			false,
-		);
-	});
-
-	test("a failing write in the batch leaves the collection's sequence where it was", async () => {
-		const [before] = await db.select().from(calendarTable);
-
-		await assert.rejects(
-			putCalendarObject(new FailingLastWriteStore(store), {
-				accountConfigId: ACCOUNT_CONFIG_ID,
-				calendarId,
-				resourceName: "doomed-again.ics",
-				icalData: RESOURCE,
-			}),
-			/NOT NULL constraint failed/,
-		);
-
-		const [after] = await db.select().from(calendarTable);
-		assert.equal(after?.syncSequence, before?.syncSequence);
-	});
-
-	test("a refused resource writes nothing at all", async () => {
-		const before = await objects();
-		const [collectionBefore] = await db.select().from(calendarTable);
-
-		const result = await putCalendarObject(store, {
-			accountConfigId: ACCOUNT_CONFIG_ID,
-			calendarId,
-			resourceName: "backwards.ics",
-			icalData: [
-				"BEGIN:VCALENDAR",
-				"VERSION:2.0",
-				"BEGIN:VEVENT",
-				"UID:backwards@example.com",
-				"DTSTART:20260826T100000Z",
-				"DTEND:20260826T090000Z",
-				"END:VEVENT",
-				"END:VCALENDAR",
-				"",
-			].join("\r\n"),
-		});
-
-		assert.ok(!result.ok);
-		assert.equal(result.error.code, "BackwardsEnd");
-		assert.equal((await objects()).length, before.length);
-		const [collectionAfter] = await db.select().from(calendarTable);
-		assert.equal(collectionAfter?.syncSequence, collectionBefore?.syncSequence);
-	});
-
-	test("rewriting a resource replaces its occurrences rather than adding to them", async () => {
-		const first = await putCalendarObject(store, {
-			accountConfigId: ACCOUNT_CONFIG_ID,
-			calendarId,
-			resourceName: "shrinking.ics",
-			icalData: RESOURCE,
-		});
-		assert.ok(first.ok);
-
-		const second = await putCalendarObject(store, {
-			accountConfigId: ACCOUNT_CONFIG_ID,
-			calendarId,
-			resourceName: "shrinking.ics",
-			icalData: RESOURCE.replace("COUNT=3", "COUNT=1"),
-		});
-		assert.ok(second.ok);
-
-		assert.equal(second.value.calendarObjectId, first.value.calendarObjectId);
-		assert.equal(
-			(await occurrences()).filter(
-				(row) => row.calendarObjectId === second.value.calendarObjectId,
-			).length,
-			1,
-		);
-	});
-});
+calendarPutSuite(openCalendarSqliteTestStores);

package/src/repos/calendar-put.suite.ts

@@ -0,0 +1,210 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import {
+	prepareCalendarObject,
+	provisionDefaultCalendar,
+	putCalendarObject,
+} from "@remit/calendar-service";
+import type {
+	CalendarRepositories,
+	CalendarWrite,
+	ICalendarStore,
+} from "@remit/data-ports";
+import type { RepositoryDb } from "../db.js";
+import {
+	calendarEventIndexTable,
+	calendarObjectTable,
+	calendarTable,
+} from "../schema.js";
+import { composeSqliteRepositories } from "../sqlite-repositories.js";
+import type { OpenTestStores } from "../test-stores.js";
+
+const ACCOUNT_CONFIG_ID = "account-config-1";
+
+const RESOURCE = [
+	"BEGIN:VCALENDAR",
+	"VERSION:2.0",
+	"BEGIN:VEVENT",
+	"UID:atomic@example.com",
+	"DTSTART:20260826T090000Z",
+	"DTEND:20260826T100000Z",
+	"SUMMARY:Quarterly review",
+	"RRULE:FREQ=WEEKLY;COUNT=3",
+	"END:VEVENT",
+	"END:VCALENDAR",
+	"",
+].join("\r\n");
+
+/**
+ * Appends a write that fails to every commit: a resource put into a collection
+ * that does not exist, which the store refuses at its last statement. Every
+ * statement of the real write set has run by then, so a failure there is the
+ * case that proves the boundary.
+ */
+class FailingLastWriteStore implements ICalendarStore {
+	constructor(private inner: ICalendarStore) {}
+
+	get repos(): CalendarRepositories {
+		return this.inner.repos;
+	}
+
+	async commit(writes: readonly CalendarWrite[]): Promise<void> {
+		const orphan = await prepareCalendarObject(
+			{ timezone: "" },
+			{
+				accountConfigId: ACCOUNT_CONFIG_ID,
+				calendarId: "no-such-calendar",
+				resourceName: "orphan.ics",
+				icalData: RESOURCE,
+			},
+		);
+		assert.ok(orphan.ok);
+		await this.inner.commit([...writes, orphan.value]);
+	}
+}
+
+export const calendarPutSuite = (open: OpenTestStores): void => {
+	describe("the calendar write path against sqlite", () => {
+		let db: RepositoryDb<Record<string, unknown>>;
+		let close: () => Promise<void>;
+		let store: ICalendarStore;
+		let calendarId: string;
+
+		before(async () => {
+			const opened = await open();
+			db = opened.stores.app.db;
+			close = opened.close;
+			store = composeSqliteRepositories(opened.stores).calendarStore;
+			const collection = await provisionDefaultCalendar(
+				store,
+				ACCOUNT_CONFIG_ID,
+			);
+			calendarId = collection.calendarId;
+		});
+
+		after(() => close());
+
+		const objects = () => db.select().from(calendarObjectTable);
+		const occurrences = () => db.select().from(calendarEventIndexTable);
+
+		test("writes the object, its occurrences and the sequence bump together", async () => {
+			const result = await putCalendarObject(store, {
+				accountConfigId: ACCOUNT_CONFIG_ID,
+				calendarId,
+				resourceName: "review.ics",
+				icalData: RESOURCE,
+			});
+
+			assert.ok(result.ok);
+			assert.equal(result.value.icalData, RESOURCE);
+			assert.equal(
+				(await objects())[0]?.icalData,
+				RESOURCE,
+				"the stored row holds the input bytes, CRLF and all",
+			);
+			assert.equal(result.value.syncSequence, 1);
+			assert.equal((await objects()).length, 1);
+			assert.equal(
+				(await occurrences()).filter(
+					(row) => row.calendarObjectId === result.value.calendarObjectId,
+				).length,
+				3,
+			);
+		});
+
+		test("a failing write in the batch leaves no object behind", async () => {
+			const before = await objects();
+
+			await assert.rejects(
+				putCalendarObject(new FailingLastWriteStore(store), {
+					accountConfigId: ACCOUNT_CONFIG_ID,
+					calendarId,
+					resourceName: "doomed.ics",
+					icalData: RESOURCE,
+				}),
+				/NOT NULL constraint failed/,
+			);
+
+			const after = await objects();
+			assert.equal(after.length, before.length);
+			assert.equal(
+				after.some((row) => row.resourceName === "doomed.ics"),
+				false,
+			);
+		});
+
+		test("a failing write in the batch leaves the collection's sequence where it was", async () => {
+			const [before] = await db.select().from(calendarTable);
+
+			await assert.rejects(
+				putCalendarObject(new FailingLastWriteStore(store), {
+					accountConfigId: ACCOUNT_CONFIG_ID,
+					calendarId,
+					resourceName: "doomed-again.ics",
+					icalData: RESOURCE,
+				}),
+				/NOT NULL constraint failed/,
+			);
+
+			const [after] = await db.select().from(calendarTable);
+			assert.equal(after?.syncSequence, before?.syncSequence);
+		});
+
+		test("a refused resource writes nothing at all", async () => {
+			const before = await objects();
+			const [collectionBefore] = await db.select().from(calendarTable);
+
+			const result = await putCalendarObject(store, {
+				accountConfigId: ACCOUNT_CONFIG_ID,
+				calendarId,
+				resourceName: "backwards.ics",
+				icalData: [
+					"BEGIN:VCALENDAR",
+					"VERSION:2.0",
+					"BEGIN:VEVENT",
+					"UID:backwards@example.com",
+					"DTSTART:20260826T100000Z",
+					"DTEND:20260826T090000Z",
+					"END:VEVENT",
+					"END:VCALENDAR",
+					"",
+				].join("\r\n"),
+			});
+
+			assert.ok(!result.ok);
+			assert.equal(result.error.code, "BackwardsEnd");
+			assert.equal((await objects()).length, before.length);
+			const [collectionAfter] = await db.select().from(calendarTable);
+			assert.equal(
+				collectionAfter?.syncSequence,
+				collectionBefore?.syncSequence,
+			);
+		});
+
+		test("rewriting a resource replaces its occurrences rather than adding to them", async () => {
+			const first = await putCalendarObject(store, {
+				accountConfigId: ACCOUNT_CONFIG_ID,
+				calendarId,
+				resourceName: "shrinking.ics",
+				icalData: RESOURCE,
+			});
+			assert.ok(first.ok);
+
+			const second = await putCalendarObject(store, {
+				accountConfigId: ACCOUNT_CONFIG_ID,
+				calendarId,
+				resourceName: "shrinking.ics",
+				icalData: RESOURCE.replace("COUNT=3", "COUNT=1"),
+			});
+			assert.ok(second.ok);
+
+			assert.equal(second.value.calendarObjectId, first.value.calendarObjectId);
+			assert.equal(
+				(await occurrences()).filter(
+					(row) => row.calendarObjectId === second.value.calendarObjectId,
+				).length,
+				1,
+			);
+		});
+	});
+};

package/src/repos/calendar-store.ts

@@ -8,6 +8,7 @@ import type {
 import { deriveCalendarObjectId } from "@remit/data-ports/id";
 import { and, eq, exists, sql } from "drizzle-orm";
 import type { SqliteBatchItem, WriteBatch } from "../batch.js";
+import { chunkRows } from "../bound-parameters.js";
 import type { RepositoryDb, SqliteStore } from "../db.js";
 import {
 	calendarEventIndexTable,
@@ -36,6 +37,7 @@ export class DrizzleCalendarStore implements ICalendarStore {
 
 	private readonly db: DB;
 	private readonly batch: WriteBatch;
+	private readonly maxBoundParameters: number;
 
 	constructor(
 		store: SqliteStore<Record<string, unknown>>,
@@ -43,10 +45,11 @@ export class DrizzleCalendarStore implements ICalendarStore {
 	) {
 		this.db = store.db;
 		this.batch = store.batch;
+		this.maxBoundParameters = store.maxBoundParameters;
 		this.repos = {
 			calendarCollection: new CalendarCollectionRepo(store.db),
 			calendarObject: new CalendarObjectRepo(store.db),
-			calendarEventIndex: new CalendarEventIndexRepo(store.db),
+			calendarEventIndex: new CalendarEventIndexRepo(store),
 			calendarSuggestion,
 			calendarFeedToken: new CalendarFeedTokenRepo(store.db),
 		};
@@ -138,17 +141,21 @@ export class DrizzleCalendarStore implements ICalendarStore {
 						}),
 					this.clearOccurrences(calendarId, calendarObjectId),
 				];
-				if (write.occurrences.length === 0) return items;
+				const occurrences = write.occurrences.map((occurrence) => ({
+					...occurrence,
+					calendarId,
+					calendarObjectId,
+					createdAt: now,
+					updatedAt: now,
+				}));
 				return [
 					...items,
-					this.db.insert(calendarEventIndexTable).values(
-						write.occurrences.map((occurrence) => ({
-							...occurrence,
-							calendarId,
-							calendarObjectId,
-							createdAt: now,
-							updatedAt: now,
-						})),
+					...chunkRows(
+						calendarEventIndexTable,
+						occurrences,
+						this.maxBoundParameters,
+					).map((slice) =>
+						this.db.insert(calendarEventIndexTable).values(slice),
 					),
 				];
 			}

package/src/repos/cascade-delete.test.ts

@@ -2,6 +2,7 @@ import assert from "node:assert";
 import { after, before, describe, test } from "node:test";
 import { MailboxCursorState } from "@remit/domain-enums";
 import { eq } from "drizzle-orm";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { syncStores } from "../databases.js";
 import {
 	accountSettingTable,
@@ -300,7 +301,10 @@ describe("runDrizzleCascadeDelete", () => {
 
 	test("deletes every enumerated row across all tables", async () => {
 		await runDrizzleCascadeDelete(
-			syncStores({ app: db, mail: db, search: db } as never),
+			syncStores(
+				{ app: db, mail: db, search: db } as never,
+				BETTER_SQLITE3_LIMITS,
+			),
 			entities,
 			log,
 		);
@@ -439,7 +443,10 @@ describe("runDrizzleCascadeDelete", () => {
 		await assert.rejects(
 			() =>
 				runDrizzleCascadeDelete(
-					syncStores({ app: db, mail: db, search: db } as never),
+					syncStores(
+						{ app: db, mail: db, search: db } as never,
+						BETTER_SQLITE3_LIMITS,
+					),
 					[{ entityType: "Nonsense", key: { id: "x" } }],
 					log,
 				),

package/src/repos/cascade-delete.ts

@@ -1,4 +1,5 @@
 import type { SqliteBatchItem } from "../batch.js";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import {
 	openSqliteDatabases,
 	type SqliteDatabasePaths,
@@ -77,7 +78,7 @@ const groupByType = (
  * first, then the app rows, then the search documents of the deleted thread
  * messages. Every statement is keyed, so a rerun after a failure between two
  * batches deletes what is left. Message subtrees
- * (message + nine child tables) are deleted set-wise by message id; the remaining
+ * (message + ten child tables) are deleted set-wise by message id; the remaining
  * entity types are deleted by their enumerated primary keys. AccountConfig is
  * never deleted here — the caller removes it last through its repository, the
  * cascade-in-progress marker.
@@ -241,6 +242,6 @@ export const createSqliteCascadeDeleter = async (
 	paths: SqliteDatabasePaths,
 ): Promise<CascadeDeleter> => {
 	const databases = await openSqliteDatabases(paths);
-	const stores = syncStores(databases);
+	const stores = syncStores(databases, BETTER_SQLITE3_LIMITS);
 	return (entities, log) => runDrizzleCascadeDelete(stores, entities, log);
 };

package/src/repos/envelope.d1.test.ts

@@ -0,0 +1,66 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import type { BodyPartUpsertInput } from "@remit/data-ports";
+import { ReferenceType } from "@remit/domain-enums";
+import { openD1TestStore } from "../test-d1.js";
+import { DrizzleEnvelopeRepository } from "./envelope.js";
+
+const MESSAGE_ID = "00000000-0000-0000-4444-000000000d01";
+
+const bodyPart = (index: number): BodyPartUpsertInput => ({
+	partPath: `${index + 1}`,
+	parentPartPath: null,
+	mediaType: "TEXT",
+	mediaSubtype: "plain",
+	transferEncoding: "7BIT",
+	sizeOctets: 128,
+	isMultipart: false,
+	parameters: [{ parameterName: "charset", parameterValue: "utf-8" }],
+});
+
+describe("DrizzleEnvelopeRepository on D1's 100 bound parameter limit", () => {
+	let repo: DrizzleEnvelopeRepository;
+	let close: () => Promise<void>;
+
+	before(async () => {
+		const opened = await openD1TestStore("mail");
+		close = opened.close;
+		repo = new DrizzleEnvelopeRepository(opened.store);
+		await repo.upsertEnvelope({
+			envelopeId: "ignored-derived",
+			messageId: MESSAGE_ID,
+			dateValue: 1700000000000,
+			dateRaw: "Mon, 14 Nov 2023 08:00:00 +0000",
+			subject: "Thirty parts",
+		});
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	test("upsertBodyParts writes thirty parts and their parameters", async () => {
+		await repo.upsertBodyParts(
+			MESSAGE_ID,
+			Array.from({ length: 30 }, (_, index) => bodyPart(index)),
+		);
+
+		const data = await repo.getMessageData(MESSAGE_ID);
+		assert.equal(data.bodyPart.length, 30);
+		assert.equal(data.bodyPartParameter.length, 30);
+	});
+
+	test("replaceMessageReferences writes thirty references", async () => {
+		await repo.replaceMessageReferences(
+			MESSAGE_ID,
+			Array.from({ length: 30 }, (_, index) => ({
+				messageIdValue: `<ref-${index}@example.com>`,
+				referenceType: ReferenceType.References,
+				referenceOrder: index,
+			})),
+		);
+
+		const data = await repo.getMessageData(MESSAGE_ID);
+		assert.equal(data.messageReference.length, 30);
+	});
+});

package/src/repos/envelope.sqlite.test.ts

@@ -1,5 +1,6 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import {
 	type MessageDataSchema,
 	messageDataSchema,
@@ -20,7 +21,10 @@ describe("DrizzleEnvelopeRepository (sqlite)", () => {
 
 	before(async () => {
 		({ db, close } = await createSqliteTestDb(messageDataSchema));
-		repo = createSqliteRepositories({ app: db, mail: db, search: db }).envelope;
+		repo = createSqliteRepositories(
+			{ app: db, mail: db, search: db },
+			BETTER_SQLITE3_LIMITS,
+		).envelope;
 	});
 
 	after(async () => {

package/src/repos/envelope.test.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
 import type { BodyPartUpsertInput } from "@remit/data-ports";
+import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
 import { envelopeId as deriveEnvelopeId } from "../id.js";
 import { createSqliteRepositories } from "../sqlite-repositories.js";
 import type { DrizzleEnvelopeRepository } from "./envelope.js";
@@ -19,7 +20,10 @@ describe("DrizzleEnvelopeRepository", () => {
 
 	before(async () => {
 		({ db, stop } = await createTestDb());
-		repo = createSqliteRepositories({ app: db, mail: db, search: db }).envelope;
+		repo = createSqliteRepositories(
+			{ app: db, mail: db, search: db },
+			BETTER_SQLITE3_LIMITS,
+		).envelope;
 
 		await repo.upsertEnvelope({
 			envelopeId: ENVELOPE_ID,