@remit/drizzle-service 0.0.70 → 0.0.72

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.70",
+  "version": "0.0.72",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/index.ts

@@ -1,6 +1,7 @@
 export { CreateFailedConflictError, NotFoundError } from "./error.js";
 export { CalendarCollectionRepo } from "./repos/calendar-collection.js";
 export { CalendarEventIndexRepo } from "./repos/calendar-event-index.js";
+export { CalendarFeedTokenRepo } from "./repos/calendar-feed-token.js";
 export { CalendarObjectRepo } from "./repos/calendar-object.js";
 export { CalendarSuggestionRepo } from "./repos/calendar-suggestion.js";
 export { DrizzleCalendarUnitOfWork } from "./repos/calendar-unit-of-work.js";

package/src/repos/calendar-feed-token.conformance.sqlite.test.ts

@@ -0,0 +1,23 @@
+import { calendarFeedTokenRepositoryConformance } from "@remit/data-ports/conformance";
+import { NotFoundError } from "../error.js";
+import { randomId } from "../id.js";
+import { calendarFeedTokenTable } from "../schema.js";
+import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { CalendarFeedTokenRepo } from "./calendar-feed-token.js";
+
+let close: (() => Promise<void>) | undefined;
+
+calendarFeedTokenRepositoryConformance({
+	async createRepository() {
+		const { db, close: closeDb } = await createSqliteTestDb({
+			calendarFeedTokens: calendarFeedTokenTable,
+		});
+		close = closeDb;
+		return new CalendarFeedTokenRepo(db as never);
+	},
+	teardown: async () => {
+		await close?.();
+	},
+	makeId: () => randomId(),
+	isNotFoundError: (error) => error instanceof NotFoundError,
+});

package/src/repos/calendar-feed-token.ts

@@ -0,0 +1,97 @@
+import type {
+	CalendarFeedTokenItem,
+	ICalendarFeedTokenRepository,
+	PutCalendarFeedTokenInput,
+} from "@remit/data-ports";
+import { deriveCalendarFeedTokenId } from "@remit/data-ports/id";
+import { and, eq } from "drizzle-orm";
+import type { Db } from "../db.js";
+import { calendarFeedTokenTable } from "../schema.js";
+
+type DB = Db<Record<string, unknown>>;
+
+function rowToFeedToken(
+	row: typeof calendarFeedTokenTable.$inferSelect,
+): CalendarFeedTokenItem {
+	return {
+		feedTokenId: row.feedTokenId,
+		accountConfigId: row.accountConfigId,
+		calendarId: row.calendarId,
+		tokenHash: row.tokenHash,
+		createdAt: row.createdAt,
+		rotatedAt: row.rotatedAt,
+		updatedAt: row.updatedAt,
+	};
+}
+
+export class CalendarFeedTokenRepo implements ICalendarFeedTokenRepository {
+	constructor(private db: DB) {}
+
+	/**
+	 * One statement, so a rotation can never be observed as two live addresses.
+	 * The conflict target is the derived primary key, which is what makes "one
+	 * active token per calendar" a property of the schema rather than of this
+	 * method: `createdAt` is left where it was and `rotatedAt` takes the clock,
+	 * so the row itself says whether this address is the calendar's first.
+	 */
+	async put(input: PutCalendarFeedTokenInput): Promise<CalendarFeedTokenItem> {
+		const now = Date.now();
+		const [row] = await this.db
+			.insert(calendarFeedTokenTable)
+			.values({
+				feedTokenId: deriveCalendarFeedTokenId(input.calendarId),
+				accountConfigId: input.accountConfigId,
+				calendarId: input.calendarId,
+				tokenHash: input.tokenHash,
+				createdAt: now,
+				rotatedAt: 0,
+				updatedAt: now,
+			})
+			.onConflictDoUpdate({
+				target: [
+					calendarFeedTokenTable.accountConfigId,
+					calendarFeedTokenTable.feedTokenId,
+				],
+				set: { tokenHash: input.tokenHash, rotatedAt: now, updatedAt: now },
+			})
+			.returning();
+		return rowToFeedToken(row);
+	}
+
+	async findByCalendar(
+		accountConfigId: string,
+		calendarId: string,
+	): Promise<CalendarFeedTokenItem | null> {
+		const [row] = await this.db
+			.select()
+			.from(calendarFeedTokenTable)
+			.where(
+				and(
+					eq(calendarFeedTokenTable.accountConfigId, accountConfigId),
+					eq(calendarFeedTokenTable.calendarId, calendarId),
+				),
+			);
+		return row ? rowToFeedToken(row) : null;
+	}
+
+	async findByTokenHash(
+		tokenHash: string,
+	): Promise<CalendarFeedTokenItem | null> {
+		const [row] = await this.db
+			.select()
+			.from(calendarFeedTokenTable)
+			.where(eq(calendarFeedTokenTable.tokenHash, tokenHash));
+		return row ? rowToFeedToken(row) : null;
+	}
+
+	async delete(accountConfigId: string, calendarId: string): Promise<void> {
+		await this.db
+			.delete(calendarFeedTokenTable)
+			.where(
+				and(
+					eq(calendarFeedTokenTable.accountConfigId, accountConfigId),
+					eq(calendarFeedTokenTable.calendarId, calendarId),
+				),
+			);
+	}
+}

package/src/repos/calendar-unit-of-work.ts

@@ -6,6 +6,7 @@ import type { Db } from "../db.js";
 import { runInTransaction } from "../tx.js";
 import { CalendarCollectionRepo } from "./calendar-collection.js";
 import { CalendarEventIndexRepo } from "./calendar-event-index.js";
+import { CalendarFeedTokenRepo } from "./calendar-feed-token.js";
 import { CalendarObjectRepo } from "./calendar-object.js";
 import { CalendarSuggestionRepo } from "./calendar-suggestion.js";
 
@@ -23,6 +24,7 @@ export class DrizzleCalendarUnitOfWork implements ICalendarUnitOfWork {
 				calendarObject: new CalendarObjectRepo(tx as never),
 				calendarEventIndex: new CalendarEventIndexRepo(tx as never),
 				calendarSuggestion: new CalendarSuggestionRepo(tx as never),
+				calendarFeedToken: new CalendarFeedTokenRepo(tx as never),
 			}),
 		);
 	}

package/src/repos/i4-address.ts

@@ -18,6 +18,7 @@ import {
 	desc,
 	eq,
 	getTableColumns,
+	gt,
 	inArray,
 	type SQL,
 	sql,
@@ -748,6 +749,40 @@ export class AddressRepo implements IAddressRepository {
 		);
 	}
 
+	async listAllByAccountConfigPage(input: {
+		accountConfigId: string;
+		cursor?: string;
+		limit?: number;
+	}): Promise<ResultList<AddressItem>> {
+		const { accountConfigId, cursor, limit = 100 } = input;
+		const afterAddressId = cursor
+			? (decodeToken(cursor).addressId as string)
+			: undefined;
+
+		const rows = await this.db
+			.select()
+			.from(addressTable)
+			.where(
+				and(
+					eq(addressTable.accountConfigId, accountConfigId),
+					afterAddressId
+						? gt(addressTable.addressId, afterAddressId)
+						: undefined,
+				),
+			)
+			.orderBy(asc(addressTable.addressId))
+			.limit(limit + 1);
+
+		const hasMore = rows.length > limit;
+		const items = rows.slice(0, limit).map(rowToAddress);
+		const lastItem = items[items.length - 1];
+		return resultList(
+			items,
+			limit,
+			hasMore && lastItem ? { addressId: lastItem.addressId } : undefined,
+		);
+	}
+
 	async createEnvelopeAddress(
 		input: CreateEnvelopeAddressInput,
 	): Promise<EnvelopeAddressItem> {

package/src/repos/i4-outbox-message.test.ts

@@ -224,6 +224,59 @@ describe("OutboxMessageRepo", () => {
 		await repo.deleteMany(accountConfigId, created);
 	});
 
+	test("listByAccounts pages one ordering over every account (#1013)", async () => {
+		const accountConfigId = randomId();
+		const first = randomId();
+		const second = randomId();
+		const created: string[] = [];
+		const byAccount = new Map<string, string[]>([
+			[first, []],
+			[second, []],
+		]);
+		for (let i = 0; i < 6; i++) {
+			const accountId = i % 2 === 0 ? first : second;
+			const msg = await repo.create(
+				makeOutboxInput(accountId, accountConfigId),
+			);
+			created.push(msg.outboxMessageId);
+			byAccount.get(accountId)?.push(msg.outboxMessageId);
+		}
+
+		const wholeList = await repo.listByAccounts([first, second]);
+		const wholeIds = wholeList.items.map((m) => m.outboxMessageId);
+		for (const ids of byAccount.values()) {
+			for (const id of ids) {
+				assert.ok(wholeIds.includes(id), "every account's rows come back");
+			}
+		}
+
+		const seen: string[] = [];
+		let continuationToken: string | undefined;
+		let pages = 0;
+		do {
+			const page = await repo.listByAccounts([first, second], {
+				limit: 2,
+				continuationToken,
+			});
+			seen.push(...page.items.map((m) => m.outboxMessageId));
+			continuationToken = page.continuationToken;
+			pages++;
+			assert.ok(pages < 10, "pagination must terminate");
+		} while (continuationToken);
+
+		assert.equal(seen.length, 6, "every row returned exactly once");
+		assert.equal(new Set(seen).size, 6, "no duplicates across pages");
+		assert.deepEqual([...seen].sort(), [...created].sort(), "no gaps");
+
+		await repo.deleteMany(accountConfigId, created);
+	});
+
+	test("listByAccounts answers no accounts with no rows", async () => {
+		const list = await repo.listByAccounts([]);
+		assert.deepEqual(list.items, []);
+		assert.equal(list.continuationToken, undefined);
+	});
+
 	test("listQueued returns only queued messages", async () => {
 		const accountId = randomId();
 		const accountConfigId = randomId();

package/src/repos/i4-outbox-message.ts

@@ -272,8 +272,17 @@ export class OutboxMessageRepo implements IOutboxMessageRepository {
 	async listByAccount(
 		accountId: string,
 		options?: { limit?: number; continuationToken?: string },
+	): Promise<ResultList<OutboxMessageItem>> {
+		return this.listByAccounts([accountId], options);
+	}
+
+	async listByAccounts(
+		accountIds: string[],
+		options?: { limit?: number; continuationToken?: string },
 	): Promise<ResultList<OutboxMessageItem>> {
 		const limit = options?.limit ?? 100;
+		if (accountIds.length === 0) return resultList([], limit);
+
 		const cursor = options?.continuationToken
 			? decodeToken(options.continuationToken)
 			: undefined;
@@ -289,7 +298,7 @@ export class OutboxMessageRepo implements IOutboxMessageRepository {
 			.from(outboxMessageTable)
 			.where(
 				and(
-					eq(outboxMessageTable.accountId, accountId),
+					inArray(outboxMessageTable.accountId, accountIds),
 					after
 						? or(
 								lt(outboxMessageTable.createdAt, after.createdAt),

package/src/schema.ts

@@ -11,6 +11,7 @@ import * as entities from "@remit/drizzle-sqlite-schema";
 export const calendarTable = entities.calendarCollections;
 export const calendarObjectTable = entities.calendarObjects;
 export const calendarEventIndexTable = entities.calendarEventIndexes;
+export const calendarFeedTokenTable = entities.calendarFeedTokens;
 export const calendarSuggestionTable = entities.calendarSuggestions;
 export const filterAnchorTable = entities.filterAnchors;
 export const filterTable = entities.filters;

package/src/tx.sqlite.test.ts

@@ -0,0 +1,83 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, before, describe, test } from "node:test";
+import Database from "better-sqlite3";
+import { sql } from "drizzle-orm";
+import { drizzle } from "drizzle-orm/better-sqlite3";
+import type { Db } from "./db.js";
+import { runInTransaction } from "./tx.js";
+
+/**
+ * A write set that reads before it writes, against the file four processes
+ * share (RFC 036 D3).
+ *
+ * SQLite gives a deferred transaction its write lock at its first write, and by
+ * then a unit that read first is holding a snapshot. Another process committing
+ * in that gap makes the upgrade impossible, and SQLite refuses it at once —
+ * `busy_timeout` waits for a lock, not for a snapshot that is already stale. The
+ * unit then dies with "database is locked" however long that timeout is, which
+ * is what answered a calendar write 500 while the imap-worker was syncing.
+ */
+
+const PRAGMAS = [
+	"journal_mode = WAL",
+	"busy_timeout = 5000",
+	"synchronous = NORMAL",
+	"foreign_keys = ON",
+];
+
+let directory: string;
+let sqlite: Database.Database;
+let other: Database.Database;
+let db: Db<Record<string, never>>;
+
+before(() => {
+	directory = mkdtempSync(join(tmpdir(), "remit-tx-"));
+	const path = join(directory, "remit.db");
+
+	sqlite = new Database(path);
+	for (const pragma of PRAGMAS) sqlite.pragma(pragma);
+	sqlite.exec("CREATE TABLE row (id INTEGER PRIMARY KEY, v TEXT NOT NULL)");
+	sqlite.exec("INSERT INTO row (id, v) VALUES (1, 'read'), (2, 'other')");
+
+	// The other writer, as a second connection. It refuses instead of waiting,
+	// so the run finishes in its own time whichever of the two holds the lock.
+	other = new Database(path);
+	other.pragma("busy_timeout = 0");
+
+	db = drizzle(sqlite) as unknown as Db<Record<string, never>>;
+});
+
+after(() => {
+	other.close();
+	sqlite.close();
+	rmSync(directory, { recursive: true, force: true });
+});
+
+describe("a top-level sqlite unit that reads before it writes", () => {
+	test("commits through another connection's write landing between the two", async () => {
+		const outcome = await runInTransaction(db, async (tx) => {
+			await tx.get(sql`SELECT v FROM row WHERE id = 1`);
+
+			// The other process, arriving in the gap an async write set leaves
+			// open. Whether its own write lands is not the claim here — only that
+			// this unit can still finish the one it came to make.
+			try {
+				other.prepare("UPDATE row SET v = 'moved' WHERE id = 2").run();
+			} catch {
+				// Refused because this unit holds the write lock, which is the
+				// outcome that keeps the unit below alive.
+			}
+
+			await tx.run(sql`UPDATE row SET v = 'written' WHERE id = 1`);
+			return "committed";
+		});
+
+		assert.equal(outcome, "committed");
+		assert.deepEqual(sqlite.prepare("SELECT v FROM row WHERE id = 1").get(), {
+			v: "written",
+		});
+	});
+});

package/src/tx.ts

@@ -7,9 +7,19 @@ import type { Db } from "./db.js";
 //
 // better-sqlite3's native transaction runner rejects a callback that returns a
 // promise, and the repos' write sets are async, so drizzle's own
-// `db.transaction()` cannot be used. The callback is bracketed with a SAVEPOINT
-// instead — a savepoint opens a transaction when none is active and commits
-// when the outermost one is released.
+// `db.transaction()` cannot be used. A top-level unit is bracketed with
+// `BEGIN IMMEDIATE` / `COMMIT` instead, and a nested one with a SAVEPOINT.
+//
+// IMMEDIATE, not the deferred transaction a bare `BEGIN` or `SAVEPOINT` opens.
+// A deferred transaction takes its write lock at its first write, and a unit
+// that reads before it writes (every unit that checks a row before updating it)
+// has by then taken a read snapshot. If another process commits in between,
+// SQLite refuses the upgrade with SQLITE_BUSY_SNAPSHOT — and that refusal is
+// immediate, because `busy_timeout` cannot wait out a snapshot that is already
+// stale. Four processes write this file (RFC 036 D3), so the gap is real: it
+// answered a calendar write 500 while the imap-worker was syncing. Taking the
+// write lock at BEGIN closes it — an IMMEDIATE that meets another writer waits
+// out the `busy_timeout` and then proceeds, which is what that timeout is for.
 //
 // All writers on this backend share one better-sqlite3 connection (RFC 036 D3),
 // and every query runs synchronously, but an async callback still yields the
@@ -61,6 +71,27 @@ async function runSqliteSavepoint<TSchema extends Record<string, unknown>, T>(
 	}
 }
 
+async function runSqliteImmediate<TSchema extends Record<string, unknown>, T>(
+	db: Db<TSchema>,
+	fn: (tx: Db<TSchema>) => Promise<T>,
+): Promise<T> {
+	const runner = db as unknown as { run: (query: SQL) => unknown };
+	runner.run(sql.raw("BEGIN IMMEDIATE"));
+	try {
+		const result = await fn(db);
+		runner.run(sql.raw("COMMIT"));
+		return result;
+	} catch (error) {
+		try {
+			runner.run(sql.raw("ROLLBACK"));
+		} catch {
+			// A failed rollback must not mask the error that caused it; surface
+			// the original below.
+		}
+		throw error;
+	}
+}
+
 export async function runInTransaction<
 	TSchema extends Record<string, unknown>,
 	T,
@@ -72,7 +103,7 @@ export async function runInTransaction<
 	}
 
 	return serializeSqlite(() =>
-		inSqliteTx.run(true, () => runSqliteSavepoint(db, fn)),
+		inSqliteTx.run(true, () => runSqliteImmediate(db, fn)),
 	);
 }