@remit/backend 0.0.98 → 0.0.100

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.98",
+  "version": "0.0.100",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",

package/src/handlers/unified-threads.test.ts

@@ -1,9 +1,11 @@
 import assert from "node:assert/strict";
 import { describe, test } from "node:test";
+import type { MessageCategory } from "@remit/api-openapi-types";
 import type {
 	AccountItem,
 	AccountSettingItem,
 	MailboxItem,
+	SearchOptions,
 } from "@remit/data-ports";
 import {
 	buildInboxMailboxMap,
@@ -11,7 +13,10 @@ import {
 	buildListStarredThreadsOptions,
 	buildSearchAllThreadsOptions,
 	dedupeByMessageId,
+	executeUnifiedThreadListing,
 	type InboxMapClient,
+	type UnifiedThreadClient,
+	type UnifiedThreadParams,
 } from "./unified-threads.js";
 
 // #44: Flagged filtered client-side over the newest 50 unified-inbox rows, so a
@@ -432,3 +437,162 @@ describe("buildSearchAllThreadsOptions", () => {
 		assert.equal(search.limit, unified.limit);
 	});
 });
+
+// #308: the cross-account collections filtered and counted in the browser, over
+// the pages a client happened to have loaded. Both are the query's job now.
+describe("executeUnifiedThreadListing", () => {
+	interface Call {
+		mode: "listByDate" | "listByStarred" | "searchByDate";
+		search?: SearchOptions;
+		limit?: number;
+		continuationToken?: string;
+	}
+
+	const emptyPage = { items: [], continuationToken: undefined };
+
+	const buildListingClient = (calls: Call[], count = 0) =>
+		({
+			account: { listAllByAccountConfig: async () => [account("a1")] },
+			mailbox: {
+				listAllByAccount: async () => [
+					mailbox("m-inbox", "a1", "INBOX"),
+					mailbox("m-archive", "a1", "Archive"),
+				],
+			},
+			accountSetting: { listByAccountConfig: async () => [] },
+			message: { get: async () => [] },
+			address: { getAddress: async () => [] },
+			messageLabel: { listByMessageIds: async () => [] },
+			label: { listByAccountConfig: async () => [] },
+			threadMessage: {
+				listByDate: async (
+					_config: string,
+					options?: { search?: SearchOptions; limit?: number },
+				) => {
+					calls.push({ mode: "listByDate", ...options });
+					return emptyPage;
+				},
+				listByStarred: async (
+					_config: string,
+					options?: { search?: SearchOptions; limit?: number },
+				) => {
+					calls.push({ mode: "listByStarred", ...options });
+					return emptyPage;
+				},
+				searchByDate: async (
+					_config: string,
+					search: SearchOptions,
+					options?: { limit?: number },
+				) => {
+					calls.push({ mode: "searchByDate", search, ...options });
+					return emptyPage;
+				},
+				countThreadsInScope: async () => count,
+			},
+		}) as unknown as UnifiedThreadClient;
+
+	const params = (
+		overrides: Partial<UnifiedThreadParams> = {},
+	): UnifiedThreadParams => ({
+		starredOnly: false,
+		count: false,
+		results: true,
+		...overrides,
+	});
+
+	test("sends the row filters into the starred query", async () => {
+		const calls: Call[] = [];
+		await executeUnifiedThreadListing(buildListingClient(calls), CONFIG_ID, {
+			...params({ starredOnly: true }),
+			category: ["social", "personal"] as MessageCategory[],
+			unread: true,
+			attachments: true,
+		});
+
+		assert.equal(calls.length, 1);
+		assert.equal(calls[0].mode, "listByStarred");
+		assert.deepEqual(calls[0].search, {
+			starred: true,
+			category: ["social", "personal"],
+			unread: true,
+			attachments: true,
+		});
+	});
+
+	test("sends the row filters into the INBOX query", async () => {
+		const calls: Call[] = [];
+		await executeUnifiedThreadListing(buildListingClient(calls), CONFIG_ID, {
+			...params(),
+			category: ["uncategorized"] as MessageCategory[],
+		});
+
+		assert.equal(calls[0].mode, "listByDate");
+		assert.deepEqual(calls[0].search, { category: ["uncategorized"] });
+	});
+
+	test("sends the row filters into the search query", async () => {
+		const calls: Call[] = [];
+		await executeUnifiedThreadListing(buildListingClient(calls), CONFIG_ID, {
+			...params({ starredOnly: true }),
+			searchText: "invoice",
+			unread: true,
+		});
+
+		assert.equal(calls[0].mode, "searchByDate");
+		assert.deepEqual(calls[0].search, {
+			query: "invoice",
+			starred: true,
+			unread: true,
+		});
+	});
+
+	// The defect the count replaces: a page-length figure grew with every press
+	// of "load more" while the header presented it as a total.
+	test("counts the same however far the caller has paged", async () => {
+		const calls: Call[] = [];
+		const first = await executeUnifiedThreadListing(
+			buildListingClient(calls, 42),
+			CONFIG_ID,
+			params({ starredOnly: true, count: true, unread: true }),
+		);
+		const later = await executeUnifiedThreadListing(
+			buildListingClient(calls, 42),
+			CONFIG_ID,
+			params({
+				starredOnly: true,
+				count: true,
+				unread: true,
+				continuationToken: "page-3",
+				limit: 10,
+			}),
+		);
+
+		assert.equal(first.count, 42);
+		assert.equal(later.count, 42);
+	});
+
+	test("reads the count alone when results are not wanted", async () => {
+		const calls: Call[] = [];
+		const response = await executeUnifiedThreadListing(
+			buildListingClient(calls, 7),
+			CONFIG_ID,
+			params({ starredOnly: true, count: true, results: false }),
+		);
+
+		assert.deepEqual(calls, []);
+		assert.equal(response.items, undefined);
+		assert.equal(response.count, 7);
+	});
+
+	test("omits the count when it was not asked for", async () => {
+		const calls: Call[] = [];
+		const response = await executeUnifiedThreadListing(
+			buildListingClient(calls, 7),
+			CONFIG_ID,
+			params({ starredOnly: true }),
+		);
+
+		assert.equal(response.count, undefined);
+		assert.deepEqual(response.items, []);
+	});
+});

package/src/handlers/unified-threads.ts

@@ -1,7 +1,14 @@
+import type {
+	MessageCategory,
+	ThreadSearchResponse,
+} from "@remit/api-openapi-types";
 import type {
 	AccountItem,
 	IAccountSettingRepository,
 	MailboxItem,
+	ResultList,
+	SearchOptions,
+	ThreadMessageItem,
 } from "@remit/data-ports";
 import { isVirtualCopyMailbox } from "@remit/data-ports/virtual-copy";
 import { MailboxSpecialUse } from "@remit/domain-enums";
@@ -9,7 +16,10 @@ import type { APIGatewayProxyEvent } from "aws-lambda";
 import type { Context } from "openapi-backend";
 import pMap from "p-map";
 import { getAccountConfigIdFromEvent } from "../auth.js";
-import { enrichThreadRows } from "../derive/enrichThreadRows.js";
+import {
+	type EnrichClient,
+	enrichThreadRows,
+} from "../derive/enrichThreadRows.js";
 import { getClient } from "../service/data-client.js";
 import type { OperationHandler, UnifiedThreadOperationIds } from "../types.js";
 import {
@@ -209,12 +219,14 @@ export const buildListAllThreadsOptions = (
 		limit?: number;
 	},
 	inboxMailboxIds: Set<string>,
+	search?: SearchOptions,
 ) => ({
 	order: query.order ?? ("desc" as const),
 	continuationToken: query.continuationToken,
 	limit: query.limit ?? DEFAULT_UNIFIED_THREADS_PAGE_SIZE,
 	inboxMailboxIds,
 	excludeDeleted: true,
+	search,
 });
 
 /**
@@ -231,12 +243,14 @@ export const buildListStarredThreadsOptions = (
 		limit?: number;
 	},
 	starredMailboxIds: Set<string>,
+	search?: SearchOptions,
 ) => ({
 	order: query.order ?? ("desc" as const),
 	continuationToken: query.continuationToken,
 	limit: query.limit ?? DEFAULT_UNIFIED_THREADS_PAGE_SIZE,
 	mailboxIds: starredMailboxIds,
 	excludeDeleted: true,
+	search,
 });
 
 /**
@@ -311,75 +325,140 @@ export const attachAccountIds = (
 		accountId: mailboxIdToAccountId.get(row.mailboxId),
 	}));
 
-export const UnifiedThreadOperations: Record<
-	UnifiedThreadOperationIds,
-	OperationHandler<UnifiedThreadOperationIds>
-> = {
-	UnifiedThreadOperations_listAllThreads: async (
-		context: Context,
-		...args: unknown[]
-	) => {
-		const event = args[0] as APIGatewayProxyEvent;
-		const accountConfigId = getAccountConfigIdFromEvent(event);
-		const { continuationToken, order, limit, starred, query } = context.request
-			.query as {
-			continuationToken?: string;
-			order?: "asc" | "desc";
-			limit?: number;
-			starred?: boolean | string;
-			query?: string;
-		};
-		const starredOnly = starred === true || starred === "true";
-		// Whitespace-only text is not a search: it would widen the scope to every
-		// folder while matching nothing in particular.
-		const searchText = query?.trim();
-		const searching = searchText !== undefined && searchText.length > 0;
+/**
+ * The row criteria the listing carries, whichever mode answers it.
+ *
+ * `category`, `unread` and `attachments` are columns on the ThreadMessage row,
+ * so each is a predicate inside the query. Filtering the rows a page returned
+ * instead is the defect this replaces: a category whose mail sits below the
+ * newest page rendered an empty list however much of it the collection held
+ * (#308). `starred` and `query` are in here too so one `SearchOptions` says
+ * what the whole request narrows by, which is what lets the count run the same
+ * predicate as the listing.
+ */
+export const buildUnifiedThreadSearch = (params: {
+	starredOnly: boolean;
+	searchText?: string;
+	category?: MessageCategory[];
+	unread?: boolean;
+	attachments?: boolean;
+}): SearchOptions => ({
+	...(params.searchText ? { query: params.searchText } : {}),
+	...(params.starredOnly ? { starred: true } : {}),
+	...(params.category?.length ? { category: params.category } : {}),
+	...(params.unread !== undefined ? { unread: params.unread } : {}),
+	...(params.attachments !== undefined
+		? { attachments: params.attachments }
+		: {}),
+});
 
-		const client = await getClient();
+/**
+ * Minimal client surface `executeUnifiedThreadListing` needs, declared
+ * structurally (like `ThreadSearchClient`) so the mode selection, the row
+ * filters and the count are testable with an in-memory fake.
+ */
+export interface UnifiedThreadClient extends EnrichClient, InboxMapClient {
+	threadMessage: {
+		listByDate(
+			accountConfigId: string,
+			options?: ReturnType<typeof buildListAllThreadsOptions>,
+		): Promise<ResultList<ThreadMessageItem>>;
+		listByStarred(
+			accountConfigId: string,
+			options?: ReturnType<typeof buildListStarredThreadsOptions>,
+		): Promise<ResultList<ThreadMessageItem>>;
+		searchByDate(
+			accountConfigId: string,
+			search: SearchOptions,
+			options?: ReturnType<typeof buildSearchAllThreadsOptions>,
+		): Promise<ResultList<ThreadMessageItem>>;
+		countThreadsInScope(
+			accountConfigId: string,
+			search: SearchOptions,
+			options?: { mailboxIds?: Set<string>; excludeDeleted?: boolean },
+		): Promise<number>;
+	};
+}
 
-		const {
-			mailboxIdToAccountId,
-			inboxMailboxIds,
-			starredMailboxIds,
-			searchMailboxIds,
-			virtualCopyMailboxIds,
-		} = await buildInboxMailboxMap(accountConfigId, client);
-
-		// Search widens past INBOX to every folder it may reach; `starred=true`
-		// still narrows it to the starred scope, so the two compose.
-		const searchScope = starredOnly ? starredMailboxIds : searchMailboxIds;
-		const scope = searching
-			? searchScope
-			: starredOnly
-				? starredMailboxIds
-				: inboxMailboxIds;
-		if (scope.size === 0) {
-			return { items: [], continuationToken: undefined };
-		}
+export type UnifiedThreadParams = {
+	continuationToken?: string;
+	order?: "asc" | "desc";
+	limit?: number;
+	starredOnly: boolean;
+	searchText?: string;
+	category?: MessageCategory[];
+	unread?: boolean;
+	attachments?: boolean;
+	count: boolean;
+	results: boolean;
+};
+
+/**
+ * Run the cross-account listing: pick the mode, apply the row criteria inside
+ * the query, and optionally count the matches.
+ *
+ * `count` names the whole match, never the page: a page size bounds the rows a
+ * response carries and has no bearing on how much matches, so pressing "load
+ * more" cannot move it. It counts conversations, the unit the listing renders
+ * once its per-mailbox rows are collapsed by `threadId`. `results: false` reads
+ * the count alone, which is how a header total is fetched without also paying
+ * for a page of mail.
+ */
+export const executeUnifiedThreadListing = async (
+	client: UnifiedThreadClient,
+	accountConfigId: string,
+	params: UnifiedThreadParams,
+): Promise<ThreadSearchResponse> => {
+	const searching =
+		params.searchText !== undefined && params.searchText.length > 0;
+	const {
+		mailboxIdToAccountId,
+		inboxMailboxIds,
+		starredMailboxIds,
+		searchMailboxIds,
+		virtualCopyMailboxIds,
+	} = await buildInboxMailboxMap(accountConfigId, client);
 
+	// Search widens past INBOX to every folder it may reach; `starred=true`
+	// still narrows it to the starred scope, so the two compose.
+	const searchScope = params.starredOnly ? starredMailboxIds : searchMailboxIds;
+	const scope = searching
+		? searchScope
+		: params.starredOnly
+			? starredMailboxIds
+			: inboxMailboxIds;
+
+	const search = buildUnifiedThreadSearch(params);
+	const page = {
+		continuationToken: params.continuationToken,
+		order: params.order,
+		limit: params.limit,
+	};
+
+	if (scope.size === 0) {
+		return {
+			...(params.results ? { items: [] } : {}),
+			...(params.count ? { count: 0 } : {}),
+		};
+	}
+
+	const response: ThreadSearchResponse = {};
+
+	if (params.results) {
 		const result = searching
 			? await client.threadMessage.searchByDate(
 					accountConfigId,
-					{ query: searchText, starred: starredOnly ? true : undefined },
-					buildSearchAllThreadsOptions(
-						{ continuationToken, order, limit },
-						searchScope,
-					),
+					search,
+					buildSearchAllThreadsOptions(page, searchScope),
 				)
-			: starredOnly
+			: params.starredOnly
 				? await client.threadMessage.listByStarred(
 						accountConfigId,
-						buildListStarredThreadsOptions(
-							{ continuationToken, order, limit },
-							starredMailboxIds,
-						),
+						buildListStarredThreadsOptions(page, starredMailboxIds, search),
 					)
 				: await client.threadMessage.listByDate(
 						accountConfigId,
-						buildListAllThreadsOptions(
-							{ continuationToken, order, limit },
-							inboxMailboxIds,
-						),
+						buildListAllThreadsOptions(page, inboxMailboxIds, search),
 					);
 
 		// Search spans folders, so it is the one mode that can see the same mail
@@ -393,11 +472,97 @@ export const UnifiedThreadOperations: Record<
 			: result.items;
 
 		const enriched = await enrichThreadRows(rows, client, accountConfigId);
-		const items = attachAccountIds(enriched, mailboxIdToAccountId);
+		response.items = attachAccountIds(enriched, mailboxIdToAccountId);
+		response.continuationToken = result.continuationToken;
+	}
 
-		return {
-			items,
-			continuationToken: result.continuationToken,
+	if (params.count) {
+		response.count = await client.threadMessage.countThreadsInScope(
+			accountConfigId,
+			search,
+			{ mailboxIds: scope, excludeDeleted: true },
+		);
+	}
+
+	return response;
+};
+
+const toArray = <T>(value: T | T[] | undefined): T[] | undefined => {
+	if (value === undefined) return undefined;
+	return Array.isArray(value) ? value : [value];
+};
+
+/**
+ * A query-string boolean, as three states.
+ *
+ * Query strings carry text; openapi-backend coerces where the schema says
+ * boolean and leaves the raw string where it cannot, so both forms are read.
+ * Anything else — absent, or a value that is neither — is `undefined`, and
+ * every caller below decides what that means for its own parameter. Folding
+ * "not stated" into `false` would turn an absent `unread` into "only read
+ * mail": a filter nobody asked for, and the opposite of the one they might
+ * have meant.
+ */
+const toBoolean = (
+	value: boolean | string | undefined,
+): boolean | undefined => {
+	if (value === true || value === "true") return true;
+	if (value === false || value === "false") return false;
+	return undefined;
+};
+
+export const UnifiedThreadOperations: Record<
+	UnifiedThreadOperationIds,
+	OperationHandler<UnifiedThreadOperationIds>
+> = {
+	UnifiedThreadOperations_listAllThreads: async (
+		context: Context,
+		...args: unknown[]
+	) => {
+		const event = args[0] as APIGatewayProxyEvent;
+		const accountConfigId = getAccountConfigIdFromEvent(event);
+		const {
+			continuationToken,
+			order,
+			limit,
+			starred,
+			query,
+			category,
+			unread,
+			attachments,
+			count,
+			results,
+		} = context.request.query as {
+			continuationToken?: string;
+			order?: "asc" | "desc";
+			limit?: number;
+			starred?: boolean | string;
+			query?: string;
+			category?: MessageCategory | MessageCategory[];
+			unread?: boolean | string;
+			attachments?: boolean | string;
+			count?: boolean | string;
+			results?: boolean | string;
 		};
+
+		// Whitespace-only text is not a search: it would widen the scope to every
+		// folder while matching nothing in particular.
+		const searchText = query?.trim();
+
+		return executeUnifiedThreadListing(await getClient(), accountConfigId, {
+			continuationToken,
+			order,
+			limit,
+			// Absent means unstated for all five, and each says what it does with
+			// that: the three filters drop out of the predicate, `count` is off
+			// unless it is asked for, and `results` is on unless it is refused.
+			starredOnly: toBoolean(starred) === true,
+			searchText: searchText || undefined,
+			category: toArray(category),
+			unread: toBoolean(unread),
+			attachments: toBoolean(attachments),
+			count: toBoolean(count) === true,
+			results: toBoolean(results) !== false,
+		});
 	},
 };

package/src/log-redaction.test.ts

@@ -33,9 +33,7 @@ process.stdout.write = ((
 
 process.env.LOG_LEVEL = "trace";
 
-const { calendarFeedPath, mintCalendarFeedToken } = await import(
-	"@remit/calendar-service"
-);
+const { mintCalendarFeedToken } = await import("@remit/calendar-service");
 const { _resetForTest, setClient } = await import("./service/data-client.js");
 const { createCalendarSqliteClient } = await import(
 	"./handlers/calendar-sqlite-fixture.js"
@@ -96,7 +94,7 @@ describe("what a request leaves in the log", () => {
 		const { token } = mintCalendarFeedToken();
 
 		const { response, logged } = await capture(() =>
-			send({ httpMethod: "GET", path: calendarFeedPath(token) }),
+			send({ httpMethod: "GET", path: `/feeds/calendar/${token}.ics` }),
 		);
 
 		assert.equal(