@remit/backend 0.0.143 → 0.0.145

package/dev-server/upload-handler.ts

@@ -1,132 +0,0 @@
-import type { Readable } from "node:stream";
-import {
-	authorizeUploadRequest,
-	parseOutboxAttachmentKey,
-	type StorageService,
-} from "@remit/storage-service";
-
-/**
- * The self-hosted receiver for an attachment upload.
- *
- * A hosted deployment mints a presigned PUT and none of this runs: the browser
- * writes to block storage and the size is bound by the storage service itself.
- * Where there is no storage service to presign against, this route is the
- * equivalent, and it has to bind the same things — one attachment, one size,
- * for a while — on its own.
- */
-
-export type CollectedBody =
-	| { readonly outcome: "Collected"; readonly content: Buffer }
-	| { readonly outcome: "TooLarge" };
-
-/**
- * Read a request body, refusing it the moment it passes what was reserved.
- *
- * The count is checked per chunk and the stream destroyed on the one that
- * crosses the line, so a client that reserved a megabyte and started sending a
- * hundred is cut off after the first megabyte rather than after all of it.
- */
-export const collectLimitedBody = async (
-	stream: Readable,
-	maxBytes: number,
-): Promise<CollectedBody> => {
-	const chunks: Buffer[] = [];
-	let received = 0;
-
-	for await (const chunk of stream) {
-		const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-		received += buffer.length;
-		if (received > maxBytes) {
-			// Stop reading, but leave the socket alone: destroying it here races the
-			// 413 the caller is about to write, and the client sees a connection
-			// reset instead of the reason it was refused.
-			stream.pause();
-			return { outcome: "TooLarge" };
-		}
-		chunks.push(buffer);
-	}
-
-	return { outcome: "Collected", content: Buffer.concat(chunks) };
-};
-
-export interface ReceiveUploadInput {
-	/** Decoded storage key, the `/outbox-upload/` prefix already stripped. */
-	storageKey: string;
-	exp: string | undefined;
-	max: string | undefined;
-	sig: string | undefined;
-	body: Readable;
-	nowSeconds: number;
-	secret: string | undefined;
-	/**
-	 * Whether the database still holds a live reservation for this attachment.
-	 * Injected rather than reached for, so the receiver stays a function of its
-	 * request and the storage it writes to.
-	 */
-	findLiveReservation: (
-		accountConfigId: string,
-		outboxAttachmentId: string,
-		nowSeconds: number,
-	) => Promise<boolean>;
-}
-
-export interface UploadResult {
-	status: number;
-	reason: string;
-}
-
-export const receiveUpload = async (
-	storage: StorageService,
-	input: ReceiveUploadInput,
-): Promise<UploadResult> => {
-	const target = parseOutboxAttachmentKey(input.storageKey);
-	if (!target) return { status: 404, reason: "not-an-attachment-key" };
-
-	const auth = authorizeUploadRequest({
-		secret: input.secret,
-		relativePath: input.storageKey,
-		exp: input.exp,
-		max: input.max,
-		sig: input.sig,
-		nowSeconds: input.nowSeconds,
-	});
-	if (!auth.authorized) {
-		return { status: auth.status, reason: auth.reason };
-	}
-
-	// A signed URL outlives the draft it was minted for, so possession of one is
-	// not enough: the database has to still hold a live reservation for this
-	// attachment. Without this a URL minted before a discard writes bytes back
-	// under a draft that no longer exists. The hosted shape cannot make this
-	// check — block storage receives the PUT directly — which is what the sweep
-	// exists for.
-	const reserved = await input.findLiveReservation(
-		target.accountConfigId,
-		target.outboxAttachmentId,
-		input.nowSeconds,
-	);
-	if (!reserved) return { status: 409, reason: "no-live-reservation" };
-
-	const collected = await collectLimitedBody(input.body, auth.maxBytes);
-	if (collected.outcome === "TooLarge") {
-		return { status: 413, reason: "over-reserved-size" };
-	}
-
-	// Exactly the reserved size, not merely under it — the same thing S3 binds
-	// when it recomputes a presigned PUT's signature over the content-length it
-	// was sent. A short body is a failed upload, and the completion would refuse
-	// it anyway; refusing here means no partial object is left to refuse.
-	if (collected.content.length !== auth.maxBytes) {
-		return { status: 400, reason: "size-mismatch" };
-	}
-
-	await storage.storeOutboxAttachment({
-		accountConfigId: target.accountConfigId,
-		accountId: target.accountId,
-		outboxMessageId: target.outboxMessageId,
-		outboxAttachmentId: target.outboxAttachmentId,
-		content: collected.content,
-	});
-
-	return { status: 204, reason: "stored" };
-};

package/dev-server/upload-route.test.ts

@@ -1,196 +0,0 @@
-/**
- * Issue #679: the upload route through the real dev-server app.
- *
- * Not a copy of its middleware order — the app object itself, imported from
- * `server.ts`. A copy is what let a JSON-typed upload silently read zero bytes:
- * `express.json` drains the stream before any route below it sees the body, and
- * a test that rebuilds the stack stays green when the real one is reordered.
- */
-
-import assert from "node:assert/strict";
-import { mkdtemp, rm } from "node:fs/promises";
-import type { Server } from "node:http";
-import { join } from "node:path";
-import { after, before, describe, it } from "node:test";
-import type {
-	IOutboxAttachmentRepository,
-	OutboxAttachmentItem,
-} from "@remit/data-ports";
-import { NotFoundError } from "@remit/data-ports/errors";
-import { OutboxAttachmentService } from "@remit/mailbox-service";
-import {
-	type StorageService,
-	UPLOAD_ROUTE_PREFIX,
-} from "@remit/storage-service";
-import { createFilesystemStorageService } from "@remit/storage-service/filesystem";
-import {
-	_resetForTest,
-	type RemitClient,
-	setClient,
-} from "../src/service/data-client.js";
-
-const SECRET = "a-signing-secret-of-at-least-32-characters";
-const CFG = "cfg-upload";
-const ACC = "acc-upload";
-const DRAFT = "draft-upload";
-
-let storage: StorageService;
-let basePath: string;
-let rows: Map<string, OutboxAttachmentItem>;
-let server: Server;
-let port: number;
-
-const repository = (): IOutboxAttachmentRepository =>
-	({
-		get: async (accountConfigId: string, outboxAttachmentId: string) => {
-			const row = rows.get(outboxAttachmentId);
-			if (!row || row.accountConfigId !== accountConfigId) {
-				throw new NotFoundError("gone");
-			}
-			return row;
-		},
-		listByOutboxMessage: async () => [...rows.values()],
-	}) as unknown as IOutboxAttachmentRepository;
-
-const reserve = async (
-	outboxAttachmentId: string,
-	sizeBytes: number,
-): Promise<string> => {
-	const expiresAt = Math.floor(Date.now() / 1000) + 900;
-	rows.set(outboxAttachmentId, {
-		outboxAttachmentId,
-		outboxMessageId: DRAFT,
-		accountId: ACC,
-		accountConfigId: CFG,
-		filename: `${outboxAttachmentId}.bin`,
-		contentType: "application/octet-stream",
-		sizeBytes,
-		state: "Pending",
-		storageKey: "",
-		reservationExpiresAt: expiresAt,
-		createdAt: 0,
-		updatedAt: 0,
-	});
-	const { uploadUrl } = await storage.createOutboxAttachmentUploadUrl({
-		accountConfigId: CFG,
-		accountId: ACC,
-		outboxMessageId: DRAFT,
-		outboxAttachmentId,
-		sizeBytes,
-		expiresAt,
-	});
-	const url = new URL(uploadUrl);
-	return `http://127.0.0.1:${port}${url.pathname}${url.search}`;
-};
-
-before(async () => {
-	// Repo-local scratch, not the machine's shared temp directory.
-	basePath = await mkdtemp(join(process.cwd(), ".tmp-upload-route-"));
-	storage = createFilesystemStorageService(basePath, {
-		origin: "http://127.0.0.1",
-		signingSecret: SECRET,
-	});
-	rows = new Map();
-
-	setClient({
-		storage,
-		outboxAttachment: new OutboxAttachmentService({
-			outboxMessageService: {} as never,
-			outboxAttachmentService: repository(),
-			storage,
-		}),
-	} as unknown as RemitClient);
-
-	process.env.BETTER_AUTH_SECRET = SECRET;
-	// Port 0: the app binds whatever is free and tells us which.
-	process.env.SERVER_PORT = "0";
-	process.env.CORS_ALLOWED_ORIGINS = "*";
-
-	const imported = await import("./server.js");
-	server = imported.listener;
-	if (!server.listening) {
-		await new Promise<void>((resolve) => server.once("listening", resolve));
-	}
-	const address = server.address();
-	port = typeof address === "object" && address ? address.port : 0;
-});
-
-after(async () => {
-	await new Promise<void>((resolve) => server.close(() => resolve()));
-	await rm(basePath, { recursive: true, force: true });
-	_resetForTest();
-});
-
-describe("PUT /outbox-upload through the dev-server's own app", () => {
-	it("stores every byte of a file the browser labelled application/json", async () => {
-		const content = Buffer.from(JSON.stringify({ note: "x".repeat(100) }));
-		const url = await reserve("attjson", content.length);
-
-		const response = await fetch(url, {
-			method: "PUT",
-			body: content,
-			// What `fetch(url, { body: file })` sends for a .json file: File.type.
-			headers: { "content-type": "application/json" },
-		});
-
-		assert.equal(response.status, 204);
-		assert.equal(
-			(await storage.statOutboxAttachment(CFG, ACC, DRAFT, "attjson"))
-				?.sizeBytes,
-			content.length,
-		);
-	});
-
-	it("stores every byte of a form-urlencoded-looking upload too", async () => {
-		const content = Buffer.from(`a=1&b=2&c=${"z".repeat(200)}`);
-		const url = await reserve("attform", content.length);
-
-		const response = await fetch(url, {
-			method: "PUT",
-			body: content,
-			headers: { "content-type": "application/x-www-form-urlencoded" },
-		});
-
-		assert.equal(response.status, 204);
-		assert.equal(
-			(await storage.statOutboxAttachment(CFG, ACC, DRAFT, "attform"))
-				?.sizeBytes,
-			content.length,
-		);
-	});
-
-	it("stores binary unchanged", async () => {
-		const content = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0xff, 0x10]);
-		const url = await reserve("attpng", content.length);
-
-		const response = await fetch(url, {
-			method: "PUT",
-			body: content,
-			headers: { "content-type": "image/png" },
-		});
-
-		assert.equal(response.status, 204);
-		assert.equal(
-			(await storage.statOutboxAttachment(CFG, ACC, DRAFT, "attpng"))
-				?.sizeBytes,
-			content.length,
-		);
-	});
-
-	it("refuses a PUT once the draft no longer has a live reservation", async () => {
-		const url = await reserve("attgone", 8);
-		rows.delete("attgone");
-
-		const response = await fetch(url, {
-			method: "PUT",
-			body: Buffer.alloc(8),
-			headers: { "content-type": "application/octet-stream" },
-		});
-
-		assert.equal(response.status, 409);
-		assert.equal(
-			await storage.statOutboxAttachment(CFG, ACC, DRAFT, "attgone"),
-			null,
-		);
-	});
-});

package/scripts/backfill-classification.ts

@@ -1,91 +0,0 @@
-import { existsSync } from "node:fs";
-import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
-import { dirname } from "node:path";
-import { logger } from "@remit/logger-lambda";
-import {
-	backfillClassifications,
-	type ClassificationBackfillCheckpoint,
-	type ClassificationBackfillCheckpointStore,
-} from "@remit/mailbox-service";
-import { getClient } from "../src/service/data-client.js";
-
-/**
- * One-time, full-corpus classification backfill over the stored-body cohort
- * the classifier never examined (issue #1197): rows whose bodies were synced
- * before the classifier reached them, which carry `NotExamined` on
- * `Message.classificationState` and so stay `uncategorized` in the list. The
- * same alternate-entrypoint shape `backfill-list-id.mjs` uses: baked into the
- * backend image, never wired into a compose one-shot, run by hand once per
- * install:
- *
- *   docker compose -f docker-compose.sqlite.yml run --rm backend \
- *     node backfill-classification.mjs
- *
- * Safe to interrupt: progress is checkpointed to disk after every page and
- * picked back up on the next run, every already-examined row reads as
- * `alreadyExamined` on a rerun, and a row whose category was already decided
- * is recorded without re-deriving it, so a partial pass resumes rather than
- * repeats.
- */
-
-const CHECKPOINT_PATH =
-	process.env.CLASSIFICATION_BACKFILL_CHECKPOINT_PATH ??
-	"/data/sqlite/classification-backfill-checkpoint.json";
-
-const isCheckpoint = (
-	value: unknown,
-): value is ClassificationBackfillCheckpoint => {
-	if (typeof value !== "object" || value === null) return false;
-	if (!("accountConfigId" in value)) return false;
-	const candidate = value as Record<string, unknown>;
-	return (
-		typeof candidate.accountConfigId === "string" &&
-		(candidate.continuationToken === undefined ||
-			typeof candidate.continuationToken === "string")
-	);
-};
-
-const fileCheckpointStore: ClassificationBackfillCheckpointStore = {
-	load: async () => {
-		if (!existsSync(CHECKPOINT_PATH)) return undefined;
-		const raw = await readFile(CHECKPOINT_PATH, "utf8");
-		const parsed: unknown = JSON.parse(raw);
-		return isCheckpoint(parsed) ? parsed : undefined;
-	},
-	save: async (checkpoint) => {
-		await mkdir(dirname(CHECKPOINT_PATH), { recursive: true });
-		const staging = `${CHECKPOINT_PATH}.writing`;
-		await writeFile(staging, JSON.stringify(checkpoint));
-		await rename(staging, CHECKPOINT_PATH);
-	},
-	clear: async () => {
-		await rm(CHECKPOINT_PATH, { force: true });
-	},
-};
-
-const run = async (): Promise<void> => {
-	const client = await getClient();
-
-	const result = await backfillClassifications(
-		{
-			accountConfigService: client.accountConfig,
-			accountService: client.account,
-			mailboxSpecialUseService: client.mailboxSpecialUse,
-			addressService: client.address,
-			threadMessageService: client.threadMessage,
-			messageService: client.message,
-			storageService: client.storage,
-		},
-		{ checkpointStore: fileCheckpointStore, logger },
-	);
-
-	// biome-ignore lint/plugin/no-logger-info: a completed full-corpus backfill is an audit-grade signal
-	logger.info({ result }, "backfill done");
-};
-
-run()
-	.then(() => process.exit(0))
-	.catch((error: unknown) => {
-		logger.error({ error }, "backfill failed");
-		process.exit(1);
-	});

package/scripts/backfill-list-id.ts

@@ -1,83 +0,0 @@
-import { existsSync } from "node:fs";
-import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
-import { dirname } from "node:path";
-import { logger } from "@remit/logger-lambda";
-import {
-	backfillListIds,
-	type ListIdBackfillCheckpoint,
-	type ListIdBackfillCheckpointStore,
-} from "@remit/mailbox-service";
-import { getClient } from "../src/service/data-client.js";
-
-/**
- * One-time, full-corpus `ThreadMessage.listId` backfill (issue #263). Ships as
- * an alternate entrypoint baked into the backend image — "the backend image
- * with a command", the same shape `migrate.mjs` uses — rather than a service
- * of its own. Unlike `migrate`, it is never wired into a compose one-shot: a
- * full pass runs once per install, not on every restart. Invoke it by hand:
- *
- *   docker compose -f docker-compose.sqlite.yml run --rm backend \
- *     node backfill-list-id.mjs
- *
- * Safe to interrupt: progress is checkpointed to disk after every page and
- * picked back up on the next run, and every row it touches is read-only
- * against the stored message, so a rerun after a partial pass is a no-op for
- * everything already done.
- */
-
-const CHECKPOINT_PATH =
-	process.env.LIST_ID_BACKFILL_CHECKPOINT_PATH ??
-	"/data/sqlite/list-id-backfill-checkpoint.json";
-
-const isCheckpoint = (value: unknown): value is ListIdBackfillCheckpoint => {
-	if (typeof value !== "object" || value === null) return false;
-	if (!("accountConfigId" in value)) return false;
-	const candidate = value as Record<string, unknown>;
-	return (
-		typeof candidate.accountConfigId === "string" &&
-		(candidate.continuationToken === undefined ||
-			typeof candidate.continuationToken === "string")
-	);
-};
-
-const fileCheckpointStore: ListIdBackfillCheckpointStore = {
-	load: async () => {
-		if (!existsSync(CHECKPOINT_PATH)) return undefined;
-		const raw = await readFile(CHECKPOINT_PATH, "utf8");
-		const parsed: unknown = JSON.parse(raw);
-		return isCheckpoint(parsed) ? parsed : undefined;
-	},
-	save: async (checkpoint) => {
-		await mkdir(dirname(CHECKPOINT_PATH), { recursive: true });
-		const staging = `${CHECKPOINT_PATH}.writing`;
-		await writeFile(staging, JSON.stringify(checkpoint));
-		await rename(staging, CHECKPOINT_PATH);
-	},
-	clear: async () => {
-		await rm(CHECKPOINT_PATH, { force: true });
-	},
-};
-
-const run = async (): Promise<void> => {
-	const client = await getClient();
-
-	const result = await backfillListIds(
-		{
-			accountConfigService: client.accountConfig,
-			threadMessageService: client.threadMessage,
-			messageService: client.message,
-			storageService: client.storage,
-		},
-		{ checkpointStore: fileCheckpointStore, logger },
-	);
-
-	// biome-ignore lint/plugin/no-logger-info: a completed full-corpus backfill is an audit-grade signal
-	logger.info({ result }, "backfill done");
-};
-
-run()
-	.then(() => process.exit(0))
-	.catch((error: unknown) => {
-		logger.error({ error }, "backfill failed");
-		process.exit(1);
-	});

package/scripts/config-save.ts

@@ -1,128 +0,0 @@
-import { readConfigForExport } from "@remit/config-transfer";
-import { env } from "expect-env";
-// Reached by module path rather than through either package's entry point, the
-// same way the migrate entrypoint reaches its repairs: this file is bundled by
-// esbuild, and a barrel import would drag every repository and the native
-// driver behind it into a script that reads one table.
-import { auth_user } from "../../auth-service/src/schema/auth-schema-sqlite.js";
-import { createSqliteDatabase } from "../../drizzle-service/src/sqlite-client.js";
-import { deriveAccountConfigId } from "../src/auth.js";
-import { getClient } from "../src/service/data-client.js";
-import { exportIdentity } from "../src/service/export-identity.js";
-
-/**
- * `remit config save` (issue #1021). Writes one configuration out as a
- * versioned JSON document on stdout, over the same reader the export endpoint
- * uses. Ships as an alternate entrypoint in the backend image — "the backend
- * image with a command", the shape `migrate.mjs` and `backfill-list-id.mjs`
- * already use — because the operator runs it before a migration drops the
- * database, with no browser and no session to authenticate.
- *
- * The document goes to stdout and never to a path inside the container: the
- * wrapper redirects it to a host file, so the file lands where the operator can
- * see it and with their ownership, rather than root-owned inside a volume.
- */
-
-const USAGE = `Usage: node config-save.mjs [--user <email>]
-
-Writes the configuration as a versioned JSON document to stdout. Contains no
-credential: each account records which one it will need back instead.
-
-  --user <email>  Which sign-in to export. Optional on an instance that holds
-                  exactly one configuration.
-`;
-
-interface Options {
-	user: string | undefined;
-}
-
-const parseArguments = (argv: readonly string[]): Options => {
-	const options: Options = { user: undefined };
-	for (let index = 0; index < argv.length; index += 1) {
-		const argument = argv[index];
-		if (argument === "--user") {
-			const value = argv[index + 1];
-			if (value === undefined || value.startsWith("--")) {
-				throw new Error("--user needs an email address");
-			}
-			options.user = value;
-			index += 1;
-			continue;
-		}
-		throw new Error(`unknown option '${argument}'`);
-	}
-	return options;
-};
-
-/**
- * The configuration behind a sign-in. Identity lives in better-auth's own
- * tables in the same database file, and the configuration id derives from the
- * user id, so the email on the sign-in screen is the one thing an operator can
- * be expected to know.
- */
-const accountConfigIdForUser = async (email: string): Promise<string> => {
-	const { db, close } = await createSqliteDatabase(
-		{ auth_user },
-		{ filename: env.SQLITE_DB_PATH },
-	);
-	try {
-		const users = await db
-			.select({ id: auth_user.id, email: auth_user.email })
-			.from(auth_user);
-		// Matched in JS rather than in the query: an address is
-		// case-insensitive on the part that matters, and which collation the
-		// column happens to carry is not something an operator should have to
-		// know before their own email matches.
-		const wanted = email.toLowerCase();
-		const user = users.find((row) => row.email.toLowerCase() === wanted);
-		if (!user) throw new Error(`no user signs in as ${email}`);
-		return deriveAccountConfigId(user.id);
-	} finally {
-		await close();
-	}
-};
-
-const soleAccountConfigId = async (
-	listAll: () => Promise<Array<{ accountConfigId: string }>>,
-): Promise<string> => {
-	const configs = await listAll();
-	const only = configs[0];
-	if (!only) throw new Error("this instance holds no configuration to export");
-	if (configs.length > 1) {
-		throw new Error(
-			`this instance holds ${configs.length} configurations — name one with --user <email>`,
-		);
-	}
-	return only.accountConfigId;
-};
-
-const run = async (): Promise<void> => {
-	const argv = process.argv.slice(2);
-	if (argv.includes("--help") || argv.includes("-h")) {
-		process.stdout.write(USAGE);
-		return;
-	}
-
-	const options = parseArguments(argv);
-	const client = await getClient();
-	const accountConfigId = options.user
-		? await accountConfigIdForUser(options.user)
-		: await soleAccountConfigId(() => client.accountConfig.listAll());
-
-	const document = await readConfigForExport(
-		client,
-		accountConfigId,
-		exportIdentity(),
-	);
-	process.stdout.write(`${JSON.stringify(document, null, "\t")}\n`);
-};
-
-// No `process.exit` on the way out: the document is written to stdout, and
-// stdout is a pipe under `compose run`, so exiting drops whatever write is
-// still pending. Setting the code and letting the process end delivers it.
-await run().catch((error: unknown) => {
-	process.stderr.write(
-		`config save: ${error instanceof Error ? error.message : String(error)}\n`,
-	);
-	process.exitCode = 1;
-});

package/src/auth.test.ts

@@ -1,12 +0,0 @@
-import assert from "node:assert/strict";
-import { describe, test } from "node:test";
-import { deriveAccountConfigId } from "./auth.js";
-
-describe("derived ids are stored primary keys: a changed value orphans every row already written and needs a migration, never a new expectation here", () => {
-	test("deriveAccountConfigId pins the account a Cognito subject resolves to", () => {
-		assert.equal(
-			deriveAccountConfigId("golden-cognito-sub"),
-			"97h2fhd0c32ocenk6ju12oxto",
-		);
-	});
-});