@remit/backend 0.0.1

package/src/service/filter.ts

@@ -0,0 +1,55 @@
+import { type AnchorPayload, buildMessageAnchor } from "@remit/search-service";
+import {
+	buildEmbeddingServiceFromEnv,
+	buildVectorStoreFromEnv,
+} from "@remit/search-service/from-env";
+
+/**
+ * Build a filter's persisted semantic anchor from an already-indexed message's
+ * chunk vectors (RFC 034 Decision 2.1), or `null` when the message has no
+ * indexed chunks. Never embeds anything new — the vectors already exist from
+ * index time.
+ *
+ * This is the only piece of filter CRUD not on the `RemitClient` port bundle:
+ * the vector store and embedder are selected by env, independent of
+ * `DATA_BACKEND`, so it works unchanged on either backend. The filter and
+ * anchor rows themselves are written through
+ * `client.filter`/`client.filterAnchor` so they land in the same backend as
+ * everything else.
+ */
+export type BuildFilterAnchor = (
+	accountConfigId: string,
+	anchorMessageId: string,
+) => Promise<AnchorPayload | null>;
+
+let cached: BuildFilterAnchor | null = null;
+
+const build = (): BuildFilterAnchor => {
+	const embedder = buildEmbeddingServiceFromEnv();
+	const store = buildVectorStoreFromEnv(embedder.dimensions);
+	return (accountConfigId, anchorMessageId) =>
+		buildMessageAnchor(
+			{ store, embedder },
+			{ accountConfigId, anchorMessageId },
+		);
+};
+
+export const buildFilterAnchor: BuildFilterAnchor = (
+	accountConfigId,
+	anchorMessageId,
+) => {
+	if (!cached) cached = build();
+	return cached(accountConfigId, anchorMessageId);
+};
+
+/** Inject the anchor builder — test use only. */
+export const _setBuildFilterAnchorForTest = (
+	override: BuildFilterAnchor,
+): void => {
+	cached = override;
+};
+
+/** Reset the singleton — test use only. */
+export const _resetBuildFilterAnchorForTest = (): void => {
+	cached = null;
+};

package/src/service/fire-and-forget.test.ts

@@ -0,0 +1,126 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { type FireAndForgetLogger, fireAndForget } from "./fire-and-forget.js";
+
+interface LogCall {
+	fields: Record<string, unknown>;
+	message: string;
+}
+
+const createLoggerSpy = () => {
+	const error: LogCall[] = [];
+	const logger: FireAndForgetLogger = {
+		error: (fields: Record<string, unknown>, message: string): void => {
+			error.push({ fields, message });
+		},
+	};
+	return { error, logger };
+};
+
+describe("fireAndForget", () => {
+	it("runs the work and stays quiet on success", async () => {
+		const { error, logger } = createLoggerSpy();
+		let ran = false;
+
+		await fireAndForget(
+			async () => {
+				ran = true;
+			},
+			{ source: "test", message: "should not log", logger },
+		);
+
+		assert.equal(ran, true);
+		assert.equal(error.length, 0);
+	});
+
+	it("does NOT reject when the work throws", async () => {
+		const { logger } = createLoggerSpy();
+
+		await assert.doesNotReject(
+			fireAndForget(
+				async () => {
+					throw new Error("boom");
+				},
+				{ source: "test", message: "contained", logger },
+			),
+		);
+	});
+
+	it("logs the failure loudly with the alertable structured fields and ids", async () => {
+		const { error, logger } = createLoggerSpy();
+		const econnrefused = Object.assign(new Error(""), {
+			name: "AggregateError",
+			code: "ECONNREFUSED",
+		});
+
+		await fireAndForget(
+			async () => {
+				throw econnrefused;
+			},
+			{
+				source: "config_load",
+				message: "Failed to trigger account sync on config load",
+				ids: { accountId: "acc-a", accountConfigId: "config-1" },
+				logger,
+			},
+		);
+
+		assert.equal(error.length, 1);
+		const call = error[0];
+		if (!call) throw new Error("expected an error log");
+		assert.equal(call.fields.alert, "sync_trigger_failed");
+		assert.equal(call.fields.source, "config_load");
+		assert.equal(call.fields.accountId, "acc-a");
+		assert.equal(call.fields.accountConfigId, "config-1");
+		assert.equal(call.fields.errorName, "AggregateError");
+		assert.equal(call.fields.errorCode, "ECONNREFUSED");
+		assert.equal(call.message, "Failed to trigger account sync on config load");
+	});
+
+	it("reads the SDK error code from the .Code field when present", async () => {
+		const { error, logger } = createLoggerSpy();
+		const sdkError = Object.assign(new Error("boom"), {
+			name: "QueueDoesNotExist",
+			Code: "AWS.SimpleQueueService.NonExistentQueue",
+		});
+
+		await fireAndForget(
+			async () => {
+				throw sdkError;
+			},
+			{ source: "test", message: "contained", logger },
+		);
+
+		const call = error[0];
+		if (!call) throw new Error("expected an error log");
+		assert.equal(
+			call.fields.errorCode,
+			"AWS.SimpleQueueService.NonExistentQueue",
+		);
+	});
+
+	it("never leaks an unhandled rejection even when void-fired", async () => {
+		const leaked: unknown[] = [];
+		const onUnhandled = (reason: unknown): void => {
+			leaked.push(reason);
+		};
+		process.on("unhandledRejection", onUnhandled);
+
+		try {
+			// Fire-and-forget exactly as a request handler does: no await.
+			void fireAndForget(
+				async () => {
+					throw Object.assign(new Error(""), { code: "ECONNREFUSED" });
+				},
+				{ source: "test", message: "contained", logger: { error: () => {} } },
+			);
+			// Let the microtask queue drain so any escaped rejection would surface.
+			await new Promise((resolve) => setImmediate(resolve));
+			await new Promise((resolve) => setImmediate(resolve));
+		} finally {
+			process.off("unhandledRejection", onUnhandled);
+		}
+
+		assert.equal(leaked.length, 0);
+	});
+});

package/src/service/fire-and-forget.ts

@@ -0,0 +1,79 @@
+import { inspect } from "node:util";
+import { logger } from "@remit/logger-lambda";
+
+type StructuredLog = (fields: Record<string, unknown>, message: string) => void;
+
+export interface FireAndForgetLogger {
+	error: StructuredLog;
+}
+
+export interface FireAndForgetContext {
+	/**
+	 * Stable discriminator for the originating call site (e.g. `"config_load"`,
+	 * `"account_create"`). Surfaces on the log line as `source` so a CloudWatch
+	 * metric filter / alarm can attribute a leak to its origin.
+	 */
+	source: string;
+	/**
+	 * Message logged when the background work rejects. Should describe the work
+	 * and reassure that the failure is contained (e.g. "...best-effort").
+	 */
+	message: string;
+	/**
+	 * Identifiers carried onto the structured error line (accountId,
+	 * accountConfigId, …) so a failure is traceable without a stack dive.
+	 */
+	ids?: Record<string, unknown>;
+	/**
+	 * Alertable discriminator on the structured error line, the field CloudWatch
+	 * metric filters key on. Defaults to `"sync_trigger_failed"` for the original
+	 * sync-kick sites; a different fire-and-forget surface (e.g. a flag enqueue)
+	 * must pass its own so it is not mis-attributed to the sync-trigger alarm.
+	 */
+	alert?: string;
+	logger?: FireAndForgetLogger;
+}
+
+/**
+ * Run unawaited background work that MUST NOT be able to fail the request that
+ * spawned it.
+ *
+ * The dev/Lambda runtime shares one event loop across concurrent requests, so a
+ * rejection escaping an unawaited background promise (a `void trigger()`) lands
+ * on whatever request happens to be in flight when the deferred work finally
+ * fails — turning an unrelated READ into a spurious 500. That is exactly how an
+ * unreachable SQS queue 500'd `/mailboxes`, `/threads`, `/outbox` and `/config`.
+ *
+ * This helper is the single containment point: it awaits the work inside a
+ * try/catch, logs every rejection LOUDLY with the alertable structured fields
+ * (`alert` — defaulting to `"sync_trigger_failed"`, `source`, the SDK error
+ * name/code, the caller ids), and ALWAYS resolves to void. Callers `void
+ * fireAndForget(...)` with no
+ * chance of an unhandled rejection escaping. Routing all fire-and-forget sites
+ * through here keeps the containment from regressing one handler at a time.
+ *
+ * Use this ONLY for genuine side effects whose failure must not change the
+ * response. A failure the caller is responsible for (the real DynamoDB read, an
+ * enqueue that IS the request's purpose) must still propagate and 500.
+ */
+export const fireAndForget = async (
+	work: () => Promise<unknown>,
+	context: FireAndForgetContext,
+): Promise<void> => {
+	await work().catch((error: unknown) => {
+		const log = context.logger ?? logger;
+		log.error(
+			{
+				alert: context.alert ?? "sync_trigger_failed",
+				source: context.source,
+				...context.ids,
+				errorName: (error as { name?: string })?.name,
+				errorCode:
+					(error as { Code?: string })?.Code ??
+					(error as { code?: string })?.code,
+				error: inspect(error),
+			},
+			context.message,
+		);
+	});
+};

package/src/service/localhost-env-config.test.ts

@@ -0,0 +1,40 @@
+import assert from "node:assert/strict";
+import { existsSync, readFileSync } from "node:fs";
+import { describe, it } from "node:test";
+import { fileURLToPath } from "node:url";
+
+const repoRoot = fileURLToPath(new URL("../../../../", import.meta.url));
+
+const parseEnvFile = (relativePath: string): Record<string, string> => {
+	const contents = readFileSync(`${repoRoot}${relativePath}`, "utf8");
+	const entries: Record<string, string> = {};
+	for (const line of contents.split("\n")) {
+		const trimmed = line.trim();
+		if (!trimmed || trimmed.startsWith("#")) continue;
+		const eq = trimmed.indexOf("=");
+		if (eq === -1) continue;
+		entries[trimmed.slice(0, eq)] = trimmed.slice(eq + 1);
+	}
+	return entries;
+};
+
+describe(
+	"local dev SQS routing (localhost-dev-aws.env)",
+	{
+		// localhost-dev-aws.env is AWS-local-dev config, stripped from the open-core
+		// tree; skip where it is absent.
+		skip: !existsSync(`${repoRoot}localhost-dev-aws.env`),
+	},
+	() => {
+		const env = parseEnvFile("localhost-dev-aws.env");
+
+		it("publishes backend writes to the FIFO mailboxes queue the sync consumer drains", () => {
+			// Prod wires the API's SQS_QUEUE_URL to the mailboxes FIFO queue
+			// (infra remit-api-stack.ts). Onboarding publishes SYNC_MAILBOXES there;
+			// the imap worker consumes it. Local dev must mirror that or onboarding
+			// events land in a queue nothing drains and accounts never sync.
+			assert.equal(env.SQS_QUEUE_URL, env.SQS_QUEUE_URL_MAILBOXES);
+			assert.ok(env.SQS_QUEUE_URL?.endsWith(".fifo"));
+		});
+	},
+);

package/src/service/organize.test.ts

@@ -0,0 +1,384 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { FilterMatchOperator } from "@remit/domain-enums";
+import type {
+	AnchorPayload,
+	ChunkMetadata,
+	VectorRecord,
+} from "@remit/search-service";
+import { createMemoryVectorStore } from "@remit/search-service";
+import type { RemitClient } from "./dynamodb.js";
+import {
+	applyOrganize,
+	matchOrganize,
+	type OrganizeMatchDeps,
+	type OrganizePredicate,
+} from "./organize.js";
+
+const ACCOUNT_CONFIG_ID = "cfg-1";
+const ANCHOR_VECTOR = [1, 0, 0, 0];
+const ORTHOGONAL_VECTOR = [0, 1, 0, 0];
+
+const anchorPayload: AnchorPayload = {
+	anchorEmbedding: ANCHOR_VECTOR,
+	anchorEmbeddingId: "test-model@4",
+	anchorSourceText: "book me a table",
+};
+
+const metadata = (over: Partial<ChunkMetadata>): ChunkMetadata => ({
+	messageId: "msg-x",
+	threadId: "thread-x",
+	accountConfigId: ACCOUNT_CONFIG_ID,
+	mailboxIds: ["mbox-1"],
+	chunkType: "body",
+	sentDate: 1_700_000_000,
+	isRead: false,
+	hasAttachment: false,
+	hasStars: false,
+	...over,
+});
+
+const bodyChunk = (
+	messageId: string,
+	vector: number[],
+	over: Partial<ChunkMetadata> = {},
+): VectorRecord => ({
+	chunkId: `${messageId}#body`,
+	vector,
+	metadata: metadata({ messageId, ...over }),
+});
+
+const predicate = (
+	over: Partial<OrganizePredicate> = {},
+): OrganizePredicate => ({
+	anchorMessageId: "msg-anchor",
+	matchOperator: FilterMatchOperator.And,
+	literalClauses: [],
+	similarityThreshold: 0.75,
+	actionLabelId: "None",
+	actionMailboxId: "None",
+	...over,
+});
+
+/**
+ * A client that records MessageLabel writes and blows up if the back-apply path
+ * ever touches Filter/FilterAnchor — the RFC 034 guardrail: this scope never
+ * persists a standing rule.
+ */
+const trackingClient = () => {
+	const labeled: Array<{ messageId: string; labelId: string }> = [];
+	let filterWrites = 0;
+	let filterAnchorWrites = 0;
+	const client = {
+		messageLabel: {
+			apply: async (input: {
+				messageId: string;
+				labelId: string;
+				accountConfigId: string;
+				appliedByFilterId?: string;
+			}) => {
+				assert.equal(
+					input.appliedByFilterId,
+					undefined,
+					"back-apply must never attribute a filter",
+				);
+				labeled.push({ messageId: input.messageId, labelId: input.labelId });
+				return {} as never;
+			},
+		},
+		message: {
+			get: async (messageId: string) => ({ messageId, mailboxId: "mbox-src" }),
+		},
+		mailbox: {
+			resolveAccountId: async () => "acct-1",
+		},
+		filter: {
+			create: async () => {
+				filterWrites += 1;
+				return {} as never;
+			},
+		},
+		filterAnchor: {
+			put: async () => {
+				filterAnchorWrites += 1;
+				return {} as never;
+			},
+		},
+	} as unknown as RemitClient;
+	return {
+		client,
+		labeled,
+		filterWrites: () => filterWrites,
+		filterAnchorWrites: () => filterAnchorWrites,
+	};
+};
+
+/**
+ * A stand-in for the local-first placement mover. `moveMessage` is idempotent
+ * on the (messageId, destination) pair — mirroring the real
+ * `PlacementMoveService` marker engine (#1297) — so a redelivered back-apply
+ * re-issues the same move without double-enqueuing. Records every call and the
+ * final resting mailbox per message.
+ */
+const trackingMoveService = () => {
+	const moves: Array<{
+		messageId: string;
+		destinationMailboxId: string;
+		accountId: string;
+	}> = [];
+	const destinationOf = new Map<string, string>();
+	let enqueues = 0;
+	return {
+		moveService: {
+			moveMessage: async (
+				_accountConfigId: string,
+				messageId: string,
+				destinationMailboxId: string,
+				accountId: string,
+			): Promise<void> => {
+				moves.push({ messageId, destinationMailboxId, accountId });
+				if (destinationOf.get(messageId) === destinationMailboxId) return;
+				destinationOf.set(messageId, destinationMailboxId);
+				enqueues += 1;
+			},
+		} as unknown as import("@remit/mailbox-service").PlacementMoveService,
+		moves,
+		enqueues: () => enqueues,
+		destinations: () => destinationOf,
+	};
+};
+
+const matchDeps = (
+	store: ReturnType<typeof createMemoryVectorStore>,
+): OrganizeMatchDeps => ({
+	buildAnchor: async () => anchorPayload,
+	vectorStore: store,
+	listAccountMessageIds: async () => [],
+});
+
+describe("matchOrganize", () => {
+	it("returns every semantically matching message and excludes the misses", async () => {
+		const store = createMemoryVectorStore();
+		const matching = ["msg-1", "msg-2", "msg-3", "msg-4", "msg-5"];
+		await store.upsert([
+			...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
+			bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
+		]);
+
+		const matched = await matchOrganize(
+			matchDeps(store),
+			ACCOUNT_CONFIG_ID,
+			predicate({ actionLabelId: "lbl-1" }),
+		);
+
+		assert.deepEqual([...matched].sort(), matching);
+	});
+
+	it("matches nothing when the predicate has neither an anchor nor a clause", async () => {
+		const store = createMemoryVectorStore();
+		await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
+
+		const matched = await matchOrganize(matchDeps(store), ACCOUNT_CONFIG_ID, {
+			...predicate(),
+			anchorMessageId: "None",
+		});
+
+		assert.deepEqual(matched, []);
+	});
+
+	it("refines the semantic set by literal clauses", async () => {
+		const store = createMemoryVectorStore();
+		await store.upsert([
+			bodyChunk("msg-1", ANCHOR_VECTOR, {
+				subject: "Dinner reservation",
+				textPreview: "your table is booked",
+			}),
+			bodyChunk("msg-2", ANCHOR_VECTOR, {
+				subject: "Newsletter",
+				textPreview: "weekly digest",
+			}),
+		]);
+
+		const matched = await matchOrganize(matchDeps(store), ACCOUNT_CONFIG_ID, {
+			...predicate(),
+			literalClauses: [{ field: "Subject", value: "reservation" }],
+		});
+
+		assert.deepEqual(matched, ["msg-1"]);
+	});
+});
+
+describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
+	it("applies the label to all N matches in one pass and writes zero Filter rows", async () => {
+		const store = createMemoryVectorStore();
+		const matching = ["msg-1", "msg-2", "msg-3", "msg-4", "msg-5"];
+		await store.upsert([
+			...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
+			bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
+		]);
+
+		const p = predicate({ actionLabelId: "lbl-1" });
+
+		// The preview and the apply share the same matcher — the previewed set is
+		// exactly what gets applied.
+		const previewed = await matchOrganize(
+			matchDeps(store),
+			ACCOUNT_CONFIG_ID,
+			p,
+		);
+		const applied = await matchOrganize(matchDeps(store), ACCOUNT_CONFIG_ID, p);
+		assert.deepEqual(previewed, applied);
+
+		const tracked = trackingClient();
+		const result = await applyOrganize(
+			{ client: tracked.client },
+			ACCOUNT_CONFIG_ID,
+			applied,
+			p,
+		);
+
+		assert.equal(result.applied, matching.length);
+		assert.equal(result.failed, 0);
+		assert.deepEqual(
+			tracked.labeled.map((row) => row.messageId).sort(),
+			matching,
+		);
+		assert.ok(
+			tracked.labeled.every((row) => row.labelId === "lbl-1"),
+			"every matching message gets the requested label",
+		);
+		assert.equal(tracked.filterWrites(), 0, "no Filter row is ever created");
+		assert.equal(
+			tracked.filterAnchorWrites(),
+			0,
+			"no FilterAnchor row is ever created",
+		);
+	});
+
+	it("counts a requested move as failed when no move service is wired", async () => {
+		const store = createMemoryVectorStore();
+		await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
+		const p = predicate({ actionMailboxId: "mbox-target" });
+
+		const matched = await matchOrganize(matchDeps(store), ACCOUNT_CONFIG_ID, p);
+		const tracked = trackingClient();
+		const result = await applyOrganize(
+			{ client: tracked.client },
+			ACCOUNT_CONFIG_ID,
+			matched,
+			p,
+		);
+
+		assert.equal(result.applied, 0);
+		assert.equal(result.failed, 1);
+	});
+
+	it("moves every match through the wired move service and writes zero Filter rows", async () => {
+		const store = createMemoryVectorStore();
+		const matching = ["msg-1", "msg-2", "msg-3"];
+		await store.upsert([
+			...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
+			bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
+		]);
+		const p = predicate({ actionMailboxId: "mbox-target" });
+
+		const matched = await matchOrganize(matchDeps(store), ACCOUNT_CONFIG_ID, p);
+		const tracked = trackingClient();
+		const mover = trackingMoveService();
+		const result = await applyOrganize(
+			{ client: tracked.client, moveService: mover.moveService },
+			ACCOUNT_CONFIG_ID,
+			matched,
+			p,
+		);
+
+		assert.equal(result.applied, matching.length);
+		assert.equal(result.failed, 0);
+		assert.deepEqual(
+			mover.moves.map((m) => m.messageId).sort(),
+			matching,
+			"every matched message is moved once",
+		);
+		assert.ok(
+			mover.moves.every((m) => m.destinationMailboxId === "mbox-target"),
+			"every move targets the requested mailbox",
+		);
+		assert.equal(
+			tracked.labeled.length,
+			0,
+			"a move-only action applies no label",
+		);
+		assert.equal(tracked.filterWrites(), 0, "no Filter row is ever created");
+		assert.equal(
+			tracked.filterAnchorWrites(),
+			0,
+			"no FilterAnchor row is ever created",
+		);
+	});
+
+	it("applies both a label and a move when both actions are requested", async () => {
+		const store = createMemoryVectorStore();
+		await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
+		const p = predicate({
+			actionLabelId: "lbl-1",
+			actionMailboxId: "mbox-target",
+		});
+
+		const matched = await matchOrganize(matchDeps(store), ACCOUNT_CONFIG_ID, p);
+		const tracked = trackingClient();
+		const mover = trackingMoveService();
+		const result = await applyOrganize(
+			{ client: tracked.client, moveService: mover.moveService },
+			ACCOUNT_CONFIG_ID,
+			matched,
+			p,
+		);
+
+		assert.equal(result.applied, 1);
+		assert.equal(result.failed, 0);
+		assert.deepEqual(tracked.labeled, [
+			{ messageId: "msg-1", labelId: "lbl-1" },
+		]);
+		assert.deepEqual(
+			mover.moves.map((m) => m.messageId),
+			["msg-1"],
+		);
+	});
+
+	it("is idempotent on redelivery: a re-run re-issues the same move without double-enqueuing", async () => {
+		const store = createMemoryVectorStore();
+		const matching = ["msg-1", "msg-2"];
+		await store.upsert(matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)));
+		const p = predicate({ actionMailboxId: "mbox-target" });
+
+		const matched = await matchOrganize(matchDeps(store), ACCOUNT_CONFIG_ID, p);
+		const tracked = trackingClient();
+		const mover = trackingMoveService();
+
+		const first = await applyOrganize(
+			{ client: tracked.client, moveService: mover.moveService },
+			ACCOUNT_CONFIG_ID,
+			matched,
+			p,
+		);
+		const second = await applyOrganize(
+			{ client: tracked.client, moveService: mover.moveService },
+			ACCOUNT_CONFIG_ID,
+			matched,
+			p,
+		);
+
+		assert.equal(first.applied, matching.length);
+		assert.equal(second.applied, matching.length);
+		assert.equal(
+			mover.enqueues(),
+			matching.length,
+			"the second pass drives the marker forward without a fresh enqueue",
+		);
+		assert.deepEqual(
+			[...mover.destinations().entries()].sort(),
+			matching.map((id): [string, string] => [id, "mbox-target"]),
+			"each message rests in the requested mailbox exactly once",
+		);
+	});
+});