@remit/backend 0.0.1

package/src/handlers/search.ts

@@ -0,0 +1,105 @@
+import type {
+	MessageCategory,
+	SemanticSearchResult,
+} from "@remit/api-openapi-types";
+import type { SearchResult } from "@remit/search-service";
+import type { APIGatewayProxyEvent } from "aws-lambda";
+import type { Context } from "openapi-backend";
+import { getAccountConfigIdFromEvent } from "../auth.js";
+import { getClient } from "../service/dynamodb.js";
+import {
+	isSemanticSearchUnavailable,
+	noteSemanticCapabilityAbsence,
+} from "../service/semantic-capability.js";
+import type { OperationHandler, SemanticSearchOperationIds } from "../types.js";
+
+const DEFAULT_LIMIT = 25;
+
+const toResponse = (item: SearchResult): SemanticSearchResult => {
+	const result: SemanticSearchResult = {
+		messageId: item.messageId,
+		threadId: item.threadId,
+		score: item.score,
+		matchedChunkType: item.matchedChunkType,
+		mailboxIds: item.mailboxIds,
+		sentDate: item.sentDate,
+	};
+	// fromName is null for messages with no sender name; exclude undefined (absent from old vectors)
+	if (item.fromName !== undefined) {
+		result.fromName = item.fromName ?? undefined;
+	}
+	if (item.subject !== undefined) {
+		result.subject = item.subject;
+	}
+	if (item.category !== undefined) {
+		result.category = item.category;
+	}
+	return result;
+};
+
+export const SemanticSearchOperations: Record<
+	SemanticSearchOperationIds,
+	OperationHandler<SemanticSearchOperationIds>
+> = {
+	SemanticSearchOperations_semanticSearch: async (
+		context: Context,
+		...args: unknown[]
+	) => {
+		const event = args[0] as APIGatewayProxyEvent;
+		const accountConfigId = getAccountConfigIdFromEvent(event);
+		const {
+			query,
+			mailboxId,
+			sentDateFrom,
+			sentDateTo,
+			hasAttachment,
+			hasStars,
+			isRead,
+			category,
+			limit,
+		} = context.request.query as {
+			query: string;
+			mailboxId?: string;
+			sentDateFrom?: number;
+			sentDateTo?: number;
+			hasAttachment?: boolean;
+			hasStars?: boolean;
+			isRead?: boolean;
+			category?: MessageCategory;
+			limit?: number;
+		};
+
+		const sentDateRange =
+			sentDateFrom !== undefined || sentDateTo !== undefined
+				? { from: sentDateFrom, to: sentDateTo }
+				: undefined;
+
+		// Capability gate for the self-host compose profiles, whose backend
+		// image cannot run the semantic pipeline — see semantic-capability.ts.
+		if (isSemanticSearchUnavailable()) {
+			return { items: [] };
+		}
+
+		let results: SearchResult[];
+		try {
+			results = await (await getClient()).search.search({
+				query,
+				accountConfigId,
+				mailboxId,
+				sentDateRange,
+				hasAttachment,
+				hasStars,
+				isRead,
+				category,
+				limit: limit ?? DEFAULT_LIMIT,
+			});
+		} catch (error) {
+			if (!noteSemanticCapabilityAbsence(error)) throw error;
+			return { items: [] };
+		}
+
+		return {
+			items: results.map(toResponse),
+		};
+	},
+};

package/src/handlers/sync-progress.test.ts

@@ -0,0 +1,158 @@
+/**
+ * Unit tests for sync-status aggregation math and phase derivation.
+ *
+ * Imports the pure helpers from sync-progress.ts directly so the tests pin
+ * the shipped implementation (no DynamoDB or AWS infrastructure required).
+ */
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import {
+	computeMessagesSynced,
+	deriveMailboxPhase,
+	type MailboxSyncFields,
+} from "./sync-progress.js";
+
+// ─── deriveMailboxPhase tests ───────────────────────────────────────────────
+
+describe("deriveMailboxPhase", () => {
+	it("returns pending when lastMessageSyncAt is 0", () => {
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: 0,
+			messageCount: 100,
+			lastSyncUid: 0,
+			highWaterMarkUid: 0,
+		};
+		assert.equal(deriveMailboxPhase(mailbox), "pending");
+	});
+
+	it("returns pending even when watermarks are set but lastMessageSyncAt is 0", () => {
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: 0,
+			messageCount: 50,
+			lastSyncUid: 50,
+			highWaterMarkUid: 100,
+		};
+		assert.equal(deriveMailboxPhase(mailbox), "pending");
+	});
+
+	it("returns complete when the worker stamped initialSyncCompletedAt after the last batch", () => {
+		const now = Date.now();
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: now - 1000,
+			initialSyncCompletedAt: now,
+			messageCount: 200,
+			// Backfill settles at the smallest real UID (sparse UIDs) — the
+			// marker, not the watermark, decides completion.
+			lastSyncUid: 42,
+			highWaterMarkUid: 200,
+		};
+		assert.equal(deriveMailboxPhase(mailbox), "complete");
+	});
+
+	it("returns complete when marker equals lastMessageSyncAt (same-millisecond writes)", () => {
+		const now = Date.now();
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: now,
+			initialSyncCompletedAt: now,
+			messageCount: 10,
+			lastSyncUid: 5,
+			highWaterMarkUid: 20,
+		};
+		assert.equal(deriveMailboxPhase(mailbox), "complete");
+	});
+
+	it("returns complete for empty mailbox (messageCount 0) without a marker", () => {
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: Date.now(),
+			messageCount: 0,
+			lastSyncUid: 0,
+			highWaterMarkUid: 0,
+		};
+		assert.equal(deriveMailboxPhase(mailbox), "complete");
+	});
+
+	it("returns syncing when a batch was written after the completion marker", () => {
+		const now = Date.now();
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: now,
+			initialSyncCompletedAt: now - 60_000,
+			messageCount: 500,
+			lastSyncUid: 150,
+			highWaterMarkUid: 500,
+		};
+		assert.equal(deriveMailboxPhase(mailbox), "syncing");
+	});
+
+	it("returns syncing when backfill is in progress and no marker is set", () => {
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: Date.now(),
+			messageCount: 500,
+			lastSyncUid: 150,
+			highWaterMarkUid: 500,
+		};
+		assert.equal(deriveMailboxPhase(mailbox), "syncing");
+	});
+});
+
+// ─── computeMessagesSynced tests ───────────────────────────────────────────
+
+describe("computeMessagesSynced", () => {
+	it("returns 0 when no sync has started (highWaterMarkUid = 0)", () => {
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: 0,
+			messageCount: 100,
+			lastSyncUid: 0,
+			highWaterMarkUid: 0,
+		};
+		assert.equal(computeMessagesSynced(mailbox), 0);
+	});
+
+	it("computes UID range approximation", () => {
+		// highWaterMarkUid - lastSyncUid + 1 = 200 - 150 + 1 = 51
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: Date.now(),
+			messageCount: 200,
+			lastSyncUid: 150,
+			highWaterMarkUid: 200,
+		};
+		assert.equal(computeMessagesSynced(mailbox), 51);
+	});
+
+	it("clamps result to messagesTotal", () => {
+		// UID range would be 200 - 1 + 1 = 200; total is 180 (sparse UIDs)
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: Date.now(),
+			messageCount: 180,
+			lastSyncUid: 1,
+			highWaterMarkUid: 200,
+		};
+		assert.equal(
+			computeMessagesSynced(mailbox),
+			180,
+			"should clamp to messagesTotal",
+		);
+	});
+
+	it("never returns negative value", () => {
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: Date.now(),
+			messageCount: 0,
+			lastSyncUid: 100,
+			highWaterMarkUid: 50,
+		};
+		assert.ok(computeMessagesSynced(mailbox) >= 0, "should never be negative");
+	});
+
+	it("returns messagesTotal when fully synced (backfill reached the oldest UID)", () => {
+		// Fully synced: lastSyncUid settled at the smallest real UID (42).
+		// Range 100 - 42 + 1 = 59, clamped to the 50 actual messages.
+		const mailbox: MailboxSyncFields = {
+			lastMessageSyncAt: Date.now(),
+			initialSyncCompletedAt: Date.now(),
+			messageCount: 50,
+			lastSyncUid: 42,
+			highWaterMarkUid: 100,
+		};
+		assert.equal(computeMessagesSynced(mailbox), 50);
+	});
+});

package/src/handlers/sync-progress.ts

@@ -0,0 +1,64 @@
+import type { MailboxItem } from "@remit/data-ports";
+
+/** The subset of mailbox fields needed to derive sync progress. */
+export type MailboxSyncFields = Pick<
+	MailboxItem,
+	| "lastMessageSyncAt"
+	| "initialSyncCompletedAt"
+	| "messageCount"
+	| "lastSyncUid"
+	| "highWaterMarkUid"
+>;
+
+/**
+ * Derive a per-mailbox sync phase from worker-written sync state.
+ *
+ * - pending:  never synced (lastMessageSyncAt is 0 / unset)
+ * - complete: the imap-worker drained the mailbox and stamped
+ *             initialSyncCompletedAt after the last batch write
+ *             (initialSyncCompletedAt >= lastMessageSyncAt), or the
+ *             mailbox is empty
+ * - syncing:  sync has started but is not yet complete
+ *
+ * Note: UID watermarks alone cannot express completion — backfill settles
+ * at the mailbox's smallest real UID (UIDs are sparse), so the completion
+ * marker written by the worker is the source of truth.
+ */
+export const deriveMailboxPhase = (mailbox: MailboxSyncFields): string => {
+	if (!mailbox.lastMessageSyncAt) return "pending";
+	const messageCount = mailbox.messageCount ?? 0;
+	const initialSyncCompletedAt = mailbox.initialSyncCompletedAt ?? 0;
+
+	// Completion marker is fresh: no batch was written after the worker
+	// recorded the mailbox as drained.
+	if (
+		initialSyncCompletedAt > 0 &&
+		initialSyncCompletedAt >= mailbox.lastMessageSyncAt
+	) {
+		return "complete";
+	}
+
+	// Also complete if messageCount is 0 (empty mailbox)
+	if (messageCount === 0) return "complete";
+
+	return "syncing";
+};
+
+/**
+ * Compute approximate messagesSynced from UID watermarks.
+ *
+ * UIDs are sparse (not necessarily contiguous), so this is an approximation.
+ * We clamp the result to [0, messagesTotal].
+ */
+export const computeMessagesSynced = (mailbox: MailboxSyncFields): number => {
+	const messagesTotal = mailbox.messageCount ?? 0;
+	const lastSyncUid = mailbox.lastSyncUid ?? 0;
+	const highWaterMarkUid = mailbox.highWaterMarkUid ?? 0;
+
+	if (highWaterMarkUid === 0) return 0;
+
+	// UID range approximation: number of UIDs in the synced range
+	const synced = highWaterMarkUid - lastSyncUid + 1;
+
+	return Math.min(Math.max(synced, 0), messagesTotal);
+};

package/src/handlers/sync.test.ts

@@ -0,0 +1,146 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import { triggerSyncSafe } from "./sync.js";
+
+const QUEUE_URL = "http://localhost:9324/000000000000/remit-dev-mailboxes";
+
+interface LogCall {
+	fields: Record<string, unknown>;
+	message: string;
+}
+
+const createLoggerSpy = () => {
+	const info: LogCall[] = [];
+	const error: LogCall[] = [];
+	const logger = {
+		info: (fields: Record<string, unknown>, message: string): void => {
+			info.push({ fields, message });
+		},
+		error: (fields: Record<string, unknown>, message: string): void => {
+			error.push({ fields, message });
+		},
+	};
+	return { info, error, logger };
+};
+
+const okSqsClient = (sent: SendMessageCommand[]): SQSClient =>
+	({
+		send: async (cmd: SendMessageCommand) => {
+			sent.push(cmd);
+			return {};
+		},
+	}) as unknown as SQSClient;
+
+/**
+ * Reproduces the real smoke failure shape: the SQS send does NOT reject inline —
+ * it rejects on a later microtask/timer, the way a socket ECONNREFUSED settles
+ * after the AWS SDK's connect + retries. A try/catch that only guards the
+ * synchronous call would miss it; only awaiting the actual promise contains it.
+ */
+const deferredFailingSqsClient = (error: Error): SQSClient =>
+	({
+		send: () =>
+			new Promise((_resolve, reject) => {
+				setImmediate(() => reject(error));
+			}),
+	}) as unknown as SQSClient;
+
+describe("triggerSyncSafe", () => {
+	it("enqueues a SYNC_MAILBOXES event and logs success", async () => {
+		const sent: SendMessageCommand[] = [];
+		const { logger, info, error } = createLoggerSpy();
+
+		await triggerSyncSafe("acc-a", "config-1", {
+			sqsClient: okSqsClient(sent),
+			queueUrl: QUEUE_URL,
+			logger,
+		});
+
+		assert.equal(sent.length, 1);
+		assert.equal(info.length, 1);
+		assert.equal(error.length, 0);
+	});
+
+	it("does NOT reject when the SQS enqueue fails (POST /sync stays resilient)", async () => {
+		const econnrefused = Object.assign(new Error(""), {
+			name: "AggregateError",
+			code: "ECONNREFUSED",
+		});
+		const { logger } = createLoggerSpy();
+
+		await assert.doesNotReject(
+			triggerSyncSafe("acc-a", "config-1", {
+				sqsClient: deferredFailingSqsClient(econnrefused),
+				queueUrl: QUEUE_URL,
+				logger,
+			}),
+		);
+	});
+
+	it("logs the failure loudly with the alertable structured fields", async () => {
+		const econnrefused = Object.assign(new Error(""), {
+			name: "AggregateError",
+			code: "ECONNREFUSED",
+		});
+		const { logger, error } = createLoggerSpy();
+
+		await triggerSyncSafe("acc-a", "config-1", {
+			sqsClient: deferredFailingSqsClient(econnrefused),
+			queueUrl: QUEUE_URL,
+			logger,
+		});
+
+		assert.equal(error.length, 1);
+		const call = error[0];
+		if (!call) throw new Error("expected an error log");
+		assert.equal(call.fields.alert, "sync_trigger_failed");
+		assert.equal(call.fields.source, "trigger_sync");
+		assert.equal(call.fields.accountId, "acc-a");
+		assert.equal(call.fields.accountConfigId, "config-1");
+		assert.equal(call.fields.errorCode, "ECONNREFUSED");
+	});
+
+	// The actual smoke-storm failure mode #746/#753 missed: POST /sync void-fires
+	// this trigger, the queue is unreachable, and a concurrent read is in flight on
+	// the same event loop. Before containment the escaped rejection landed on that
+	// read and 500'd it. The SQS send here rejects on a LATER tick (a real socket
+	// error never settles inline), which is exactly why a synchronous-only guard
+	// did not reproduce the failure. Assert nothing leaks onto the concurrent read.
+	it("void-fired with an unreachable queue leaks NO unhandled rejection onto a concurrent read", async () => {
+		const leaked: unknown[] = [];
+		const onUnhandled = (reason: unknown): void => {
+			leaked.push(reason);
+		};
+		process.on("unhandledRejection", onUnhandled);
+
+		const econnrefused = Object.assign(new Error(""), {
+			name: "AggregateError",
+			code: "ECONNREFUSED",
+		});
+		const { logger } = createLoggerSpy();
+
+		let readResolved = false;
+		try {
+			// Fire-and-forget exactly as SyncOperations_triggerSync does.
+			void triggerSyncSafe("acc-a", "config-1", {
+				sqsClient: deferredFailingSqsClient(econnrefused),
+				queueUrl: QUEUE_URL,
+				logger,
+			});
+
+			// A concurrent "read" sharing the event loop, like /mailboxes or /outbox.
+			await new Promise((resolve) => setImmediate(resolve)).then(() => {
+				readResolved = true;
+			});
+			// Drain remaining microtasks so any escaped rejection would surface.
+			await new Promise((resolve) => setImmediate(resolve));
+			await new Promise((resolve) => setImmediate(resolve));
+		} finally {
+			process.off("unhandledRejection", onUnhandled);
+		}
+
+		assert.equal(readResolved, true);
+		assert.equal(leaked.length, 0);
+	});
+});

package/src/handlers/sync.ts

@@ -0,0 +1,119 @@
+import type { SQSClient } from "@aws-sdk/client-sqs";
+import { logger } from "@remit/logger-lambda";
+import type { APIGatewayProxyEvent } from "aws-lambda";
+import { env } from "expect-env";
+import { getAccountConfigIdFromEvent } from "../auth.js";
+import { getClient } from "../service/dynamodb.js";
+import {
+	type FireAndForgetLogger,
+	fireAndForget,
+} from "../service/fire-and-forget.js";
+import { sqsClient } from "../service/sqs.js";
+import { triggerAccountSync } from "../service/trigger-sync.js";
+import type { OperationHandler, SyncOperationIds } from "../types.js";
+import { assertAccountOwnership } from "./account-ownership.js";
+import { computeMessagesSynced, deriveMailboxPhase } from "./sync-progress.js";
+
+interface SyncTriggerDeps {
+	sqsClient: SQSClient;
+	queueUrl: string;
+	logger: FireAndForgetLogger & {
+		info: (fields: Record<string, unknown>, message: string) => void;
+	};
+}
+
+const defaultSyncTriggerDeps = (): SyncTriggerDeps => ({
+	sqsClient,
+	queueUrl: env.SQS_QUEUE_URL,
+	logger,
+});
+
+/**
+ * Fire-and-forget enqueue for the POST /sync kick.
+ *
+ * POST /sync is a best-effort nudge: the web client polls it to wake a sync and
+ * acts on no enqueue result in the response. So the enqueue must never fail —
+ * nor leak onto — this request or any concurrent read when the queue is
+ * unreachable. Previously this handler `await`ed the SQS send directly; with the
+ * queue down (smoke/e2e, or an SQS outage in prod) the awaited rejection escaped
+ * the handler's promise and landed on whatever read was in flight on the shared
+ * event loop, 500-ing unrelated `/mailboxes`, `/threads` and `/outbox` reads.
+ *
+ * Routing it through fireAndForget catches the rejection at the source, logs it
+ * loudly with the alertable structured fields, and resolves to void. A genuine
+ * account-lookup / ownership failure still propagates and 500s as normal.
+ */
+export const triggerSyncSafe = async (
+	accountId: string,
+	accountConfigId: string,
+	deps: SyncTriggerDeps = defaultSyncTriggerDeps(),
+): Promise<void> => {
+	await fireAndForget(
+		async () => {
+			const { eventId } = await triggerAccountSync({
+				sqsClient: deps.sqsClient,
+				queueUrl: deps.queueUrl,
+				accountId,
+			});
+			deps.logger.info(
+				{ accountId, eventId },
+				"Sync triggered - enqueued SYNC_MAILBOXES event",
+			);
+		},
+		{
+			source: "trigger_sync",
+			message: "Failed to enqueue SYNC_MAILBOXES on POST /sync (best-effort)",
+			ids: { accountId, accountConfigId },
+			logger: deps.logger,
+		},
+	);
+};
+
+export const SyncOperations: Record<
+	SyncOperationIds,
+	OperationHandler<SyncOperationIds>
+> = {
+	SyncOperations_triggerSync: async (context, ...args: unknown[]) => {
+		const event = args[0] as APIGatewayProxyEvent;
+		const accountConfigId = getAccountConfigIdFromEvent(event);
+		const { accountId } = context.request.params as { accountId: string };
+
+		const account = await (await getClient()).account.get(accountId);
+		assertAccountOwnership(account, accountConfigId, "act");
+
+		void triggerSyncSafe(account.accountId, accountConfigId);
+
+		return {
+			triggered: true,
+			message: `Sync triggered for account ${accountId}`,
+		};
+	},
+
+	SyncOperations_getSyncStatus: async (context, ...args: unknown[]) => {
+		const event = args[0] as APIGatewayProxyEvent;
+		const accountConfigId = getAccountConfigIdFromEvent(event);
+		const { accountId } = context.request.params as { accountId: string };
+
+		const client = await getClient();
+		const account = await client.account.get(accountId);
+		assertAccountOwnership(account, accountConfigId, "read");
+
+		// Collect all mailboxes (read-only — does not write or contend with sync lock)
+		const mailboxes = await client.mailbox.listAllByAccount(accountId);
+
+		return {
+			accountId: account.accountId,
+			syncPhase: account.syncPhase,
+			mailboxCountTotal: account.mailboxCountTotal,
+			mailboxCountSynced: account.mailboxCountSynced,
+			mailboxes: mailboxes.map((mailbox) => ({
+				mailboxId: mailbox.mailboxId,
+				fullPath: mailbox.fullPath,
+				phase: deriveMailboxPhase(mailbox),
+				messagesTotal: mailbox.messageCount ?? 0,
+				messagesSynced: computeMessagesSynced(mailbox),
+				lastSyncedAt: mailbox.lastMessageSyncAt || undefined,
+			})),
+		};
+	},
+};