@remit/drizzle-service 0.0.31 → 0.0.33

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.31",
+  "version": "0.0.33",
   "description": "Drizzle ORM service parameterized by dialect (Postgres / SQLite)",
   "type": "module",
   "main": "src/index.ts",

package/src/repos/category-fixture.ts

@@ -0,0 +1,156 @@
+import type { ThreadMessageItem } from "@remit/data-ports";
+
+/**
+ * The measured category shape of the owner's INBOX, shared by the sqlite and
+ * Postgres category suites (#304).
+ *
+ * It lives in one module because the shape is the load-bearing part of the
+ * regression: a filter resolved over the page the server returned is empty at
+ * every page size precisely because the rare categories sit outside the newest
+ * page. Two copies of those numbers can drift apart, and the one that drifts
+ * stops being the argument.
+ *
+ * Measured on the live instance (docs/architecture/mail-list-boundary.md).
+ */
+
+export type Category =
+	| "personal"
+	| "marketing"
+	| "automated"
+	| "newsletter"
+	| "transactional"
+	| "social";
+
+export type CategoryTotals = Record<Category, number>;
+
+/** INBOX as measured: 14,187 non-deleted rows. */
+export const LIVE_TOTALS: CategoryTotals = {
+	personal: 4753,
+	marketing: 3942,
+	automated: 2680,
+	newsletter: 2295,
+	transactional: 429,
+	social: 88,
+};
+
+/**
+ * The same shape with the four common categories trimmed, for the Postgres twin.
+ *
+ * The rare tail is untouched — `social` stays at its measured 88 — so every
+ * assertion the suites make about a rare category holds identically. What
+ * shrinks is the bulk that only exists to make the common categories common,
+ * which the Postgres suite does not assert on and which costs ~20s of seeding.
+ */
+export const TRIMMED_TOTALS: CategoryTotals = {
+	personal: 700,
+	marketing: 500,
+	automated: 400,
+	newsletter: 200,
+	transactional: 130,
+	social: 88,
+};
+
+/**
+ * The newest 100 rows of that INBOX, as measured. Two of the 4,753 personal
+ * messages and two of the 88 social ones are in it, which is why selecting
+ * either chip over the newest page returns almost nothing.
+ */
+export const NEWEST_100: CategoryTotals = {
+	automated: 77,
+	newsletter: 14,
+	marketing: 3,
+	social: 2,
+	transactional: 2,
+	personal: 2,
+};
+
+export const BASE_DATE = 1_767_225_600_000;
+
+export const totalRows = (totals: CategoryTotals): number =>
+	Object.values(totals).reduce((sum, count) => sum + count, 0);
+
+// mulberry32: a real 32-bit generator, so the stream is the one the shuffle
+// claims. The interleaving is part of the fixture — a failure has to be
+// reproducible — so this is seeded rather than Math.random.
+const mulberry32 = (seed: number): (() => number) => {
+	let state = seed >>> 0;
+	return () => {
+		state = (state + 0x6d2b79f5) >>> 0;
+		let t = state;
+		t = Math.imul(t ^ (t >>> 15), t | 1);
+		t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+		return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+	};
+};
+
+const shuffled = (values: Category[], seed: number): Category[] => {
+	const random = mulberry32(seed);
+	const out = [...values];
+	for (let i = out.length - 1; i > 0; i--) {
+		const j = Math.floor(random() * (i + 1));
+		[out[i], out[j]] = [out[j], out[i]];
+	}
+	return out;
+};
+
+const expand = (counts: Partial<CategoryTotals>): Category[] => {
+	const out: Category[] = [];
+	for (const [category, count] of Object.entries(counts) as Array<
+		[Category, number]
+	>) {
+		for (let i = 0; i < count; i++) out.push(category);
+	}
+	return out;
+};
+
+type FixtureRow = Omit<ThreadMessageItem, "star"> & { star: "none" };
+
+/**
+ * One mailbox of thread-message rows in the given category distribution,
+ * newest first, with the measured mix in the newest 100.
+ *
+ * Every seventh row repeats its predecessor's `sentDate`, so a keyset walk over
+ * the fixture crosses tie groups instead of a strictly distinct sequence.
+ */
+export const categoryFixtureRows = (options: {
+	totals: CategoryTotals;
+	accountConfigId: string;
+	mailboxId: string;
+}): FixtureRow[] => {
+	const { totals, accountConfigId, mailboxId } = options;
+	const tail = Object.fromEntries(
+		(Object.keys(totals) as Category[]).map((category) => [
+			category,
+			totals[category] - NEWEST_100[category],
+		]),
+	) as CategoryTotals;
+
+	const byRecency = [
+		...shuffled(expand(NEWEST_100), 11),
+		...shuffled(expand(tail), 29),
+	];
+
+	return byRecency.map((category, index) => {
+		const step = index - (index % 7 === 0 && index > 0 ? 1 : 0);
+		const sentDate = BASE_DATE - step * 1000;
+		return {
+			threadMessageId: `tm-category-${index}`,
+			accountConfigId,
+			threadId: `t-category-${index}`,
+			messageId: `m-category-${index}`,
+			mailboxId,
+			uid: index + 1,
+			referenceOrder: 0,
+			internalDate: sentDate,
+			sentDate,
+			isRead: false,
+			hasAttachment: false,
+			star: "none",
+			hasStars: false,
+			isDeleted: false,
+			category,
+			createdAt: 0,
+			updatedAt: 0,
+		};
+	});
+};

package/src/repos/filter.test.ts

@@ -36,6 +36,10 @@ describe("FilterRepo", () => {
 			filter.ruleChangedAt > 0,
 			"ruleChangedAt should be set on create",
 		);
+		assert.ok(
+			filter.actionChangedAt > 0,
+			"actionChangedAt should be set on create",
+		);
 		assert.equal(filter.expiresAt, undefined);
 		assert.equal(filter.ttl, undefined);
 	});
@@ -97,9 +101,14 @@ describe("FilterRepo", () => {
 			filter.ruleChangedAt,
 			"a name-only edit must not move ruleChangedAt (RFC 034 Decision 3.2)",
 		);
+		assert.equal(
+			renamed.actionChangedAt,
+			filter.actionChangedAt,
+			"a name-only edit must not move actionChangedAt either",
+		);
 	});
 
-	test("update bumps ruleChangedAt when the action changes", async () => {
+	test("update bumps both ruleChangedAt and actionChangedAt when the action changes", async () => {
 		const accountConfigId = randomId();
 		const filter = await repo.create({
 			accountConfigId,
@@ -117,9 +126,13 @@ describe("FilterRepo", () => {
 			updated.ruleChangedAt > filter.ruleChangedAt,
 			"an action edit must move ruleChangedAt forward",
 		);
+		assert.ok(
+			updated.actionChangedAt > filter.actionChangedAt,
+			"an action edit must move actionChangedAt forward too — it is what a matching message's exclusive move goes by",
+		);
 	});
 
-	test("update bumps ruleChangedAt when scope changes (reader #266)", async () => {
+	test("update bumps ruleChangedAt but NOT actionChangedAt when scope changes (reader #266, #384)", async () => {
 		const accountConfigId = randomId();
 		const filter = await repo.create({
 			accountConfigId,
@@ -141,11 +154,16 @@ describe("FilterRepo", () => {
 		assert.equal(updated.expiresAt, expiresAt);
 		assert.ok(
 			updated.ruleChangedAt > filter.ruleChangedAt,
-			"a scope edit must move ruleChangedAt forward",
+			"a scope edit must move ruleChangedAt forward, so a lapsed filter's back-application can be offered again",
+		);
+		assert.equal(
+			updated.actionChangedAt,
+			filter.actionChangedAt,
+			"a scope edit changes the filter's lifecycle, not what it matches or does — it must not move actionChangedAt, or it would wrongly promote the filter to move-winner",
 		);
 	});
 
-	test("update bumps ruleChangedAt when only expiresAt changes (reader #266)", async () => {
+	test("update bumps ruleChangedAt but NOT actionChangedAt when only expiresAt changes (reader #266, #384)", async () => {
 		const accountConfigId = randomId();
 		const initialExpiresAt = "2026-08-01T00:00:00+00:00";
 		const filter = await repo.create({
@@ -169,6 +187,11 @@ describe("FilterRepo", () => {
 			updated.ruleChangedAt > filter.ruleChangedAt,
 			"an expiresAt edit must move ruleChangedAt forward",
 		);
+		assert.equal(
+			updated.actionChangedAt,
+			filter.actionChangedAt,
+			"extending a filter's expiry alone must not promote it to move-winner (reader #384) — actionChangedAt must not move",
+		);
 	});
 
 	test("update to Standing clears expiresAt and ttl at the storage layer, even if the caller didn't null them (reader #266)", async () => {

package/src/repos/filter.ts

@@ -29,6 +29,20 @@ const RULE_ASSERTION_FIELDS = [
 	"expiresAt",
 ] as const satisfies readonly (keyof UpdateFilterInput)[];
 
+/**
+ * The subset of {@link RULE_ASSERTION_FIELDS} that changes what a filter
+ * matches or what it does — excluding `scope`/`expiresAt`, which only change
+ * how long the filter runs. Backs `actionChangedAt` (reader #384), the
+ * narrower signal `selectMoveWinner` reads for exclusive-move precedence.
+ */
+const ACTION_ASSERTION_FIELDS = [
+	"hasAnchor",
+	"matchOperator",
+	"literalClauses",
+	"actionLabelId",
+	"actionMailboxId",
+] as const satisfies readonly (keyof UpdateFilterInput)[];
+
 const nowSeconds = (): number => Math.floor(Date.now() / 1000);
 
 /**
@@ -38,10 +52,24 @@ const nowSeconds = (): number => Math.floor(Date.now() / 1000);
  * alongside the predicate/action: re-asserting how long a filter runs is the
  * same kind of "the user just told Remit something new" moment, and it is
  * what lets a lapsed filter's back-application be offered again.
+ *
+ * This is deliberately broader than {@link changesActionAssertion} — it backs
+ * `ruleChangedAt`, the settings UI's back-apply-offer signal, not exclusive-
+ * move precedence (reader #384).
  */
 const changesRuleAssertion = (input: UpdateFilterInput): boolean =>
 	RULE_ASSERTION_FIELDS.some((field) => field in input);
 
+/**
+ * Whether `input` touches the predicate or the action — never `scope` or
+ * `expiresAt` (reader #384). Backs `actionChangedAt`, which `selectMoveWinner`
+ * reads to decide an exclusive-move conflict: extending a filter's expiry or
+ * flipping Standing/Temporary changes its lifecycle, not where a matching
+ * message goes, so it must not promote the filter to move-winner.
+ */
+const changesActionAssertion = (input: UpdateFilterInput): boolean =>
+	ACTION_ASSERTION_FIELDS.some((field) => field in input);
+
 function rowToFilter(row: typeof filterTable.$inferSelect): FilterItem {
 	return {
 		filterId: row.filterId,
@@ -53,6 +81,7 @@ function rowToFilter(row: typeof filterTable.$inferSelect): FilterItem {
 		state: row.state,
 		hasAnchor: row.hasAnchor,
 		ruleChangedAt: row.ruleChangedAt,
+		actionChangedAt: row.actionChangedAt,
 		matchOperator: row.matchOperator,
 		literalClauses: row.literalClauses as FilterItem["literalClauses"],
 		actionLabelId: row.actionLabelId,
@@ -66,12 +95,13 @@ export class FilterRepo implements IFilterRepository {
 	constructor(private db: DB) {}
 
 	/**
-	 * `ruleChangedAt` is set to the same instant as creation: a new filter's
-	 * predicate/action is, by definition, freshly asserted (RFC 034 Decision
-	 * 3.2).
+	 * `ruleChangedAt` and `actionChangedAt` are both set to the same instant as
+	 * creation: a new filter's predicate/action (and its scope/expiry) are, by
+	 * definition, freshly asserted (RFC 034 Decision 3.2).
 	 */
 	async create(input: CreateFilterInput): Promise<FilterItem> {
 		const now = Date.now();
+		const createdRuleChangedAt = nowSeconds();
 		const [row] = await this.db
 			.insert(filterTable)
 			.values({
@@ -83,7 +113,8 @@ export class FilterRepo implements IFilterRepository {
 				ttl: input.ttl ?? null,
 				state: input.state ?? FilterState.Active,
 				hasAnchor: input.hasAnchor ?? false,
-				ruleChangedAt: nowSeconds(),
+				ruleChangedAt: createdRuleChangedAt,
+				actionChangedAt: createdRuleChangedAt,
 				matchOperator: input.matchOperator ?? FilterMatchOperator.And,
 				literalClauses: input.literalClauses ?? [],
 				actionLabelId: input.actionLabelId ?? "None",
@@ -112,9 +143,13 @@ export class FilterRepo implements IFilterRepository {
 	}
 
 	/**
-	 * `ruleChangedAt` only advances when `input` touches the predicate, the
-	 * action, the scope, or the expiry — never on a cosmetic `name` edit (RFC
-	 * 034 Decision 3.2, reader #266).
+	 * `ruleChangedAt` advances when `input` touches the predicate, the action,
+	 * the scope, or the expiry — never on a cosmetic `name` edit (RFC 034
+	 * Decision 3.2, reader #266). `actionChangedAt` advances only for the
+	 * narrower predicate/action subset (reader #384): a scope/expiresAt-only
+	 * edit bumps `ruleChangedAt` (so a lapsed filter's back-application can be
+	 * offered again) without touching `actionChangedAt`, so it cannot flip
+	 * `selectMoveWinner`'s exclusive-move precedence.
 	 *
 	 * A patch moving `scope` to `Standing` clears `expiresAt`/`ttl` at the SQL
 	 * layer regardless of what `input` carries for them: the caller (the
@@ -130,9 +165,12 @@ export class FilterRepo implements IFilterRepository {
 		filterId: string,
 		input: UpdateFilterInput,
 	): Promise<FilterItem> {
-		const patch = changesRuleAssertion(input)
-			? { ...input, ruleChangedAt: nowSeconds() }
-			: input;
+		const changedAt = nowSeconds();
+		const patch = {
+			...input,
+			...(changesRuleAssertion(input) ? { ruleChangedAt: changedAt } : {}),
+			...(changesActionAssertion(input) ? { actionChangedAt: changedAt } : {}),
+		};
 		const updates: Partial<typeof filterTable.$inferInsert> = {
 			...patch,
 			updatedAt: Date.now(),

package/src/repos/thread-message-category.test.ts

@@ -0,0 +1,156 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import { drizzle } from "drizzle-orm/node-postgres";
+import type pg from "pg";
+import type { Db } from "../db.js";
+import { threadMessageTable } from "../schema/thread-message.js";
+import * as schema from "../schema.js";
+import { createTestDb } from "../test-db.js";
+import { categoryFixtureRows, TRIMMED_TOTALS } from "./category-fixture.js";
+import { DrizzleThreadMessageRepository } from "./thread-message.js";
+
+/**
+ * The Postgres half of the category-predicate guard (#304). The behavioural
+ * coverage is in thread-message.sqlite.test.ts against the full measured shape;
+ * what this file exists for is the query plan, which is dialect-specific:
+ * `EXPLAIN QUERY PLAN` is SQLite syntax and Postgres needs its own `EXPLAIN`.
+ *
+ * It runs against the embedded Postgres harness, which pushes the generated
+ * drizzle schema, so `tm_by_mailbox_category_date` is present here for the same
+ * reason it is present in the sqlite harness: the index is declared in TypeSpec
+ * and emitted into the schema both dialects derive from. That is what makes the
+ * assertion a real guard rather than a restatement of the predicate — remove the
+ * index from the schema and it fails.
+ *
+ * The fixture is the measured shape with its common categories trimmed
+ * (TRIMMED_TOTALS): the rare tail the assertions use is untouched, and the
+ * planner needs only enough rows to prefer an index over a sequential scan. It
+ * also needs statistics before it will, so the fixture is ANALYZEd; a production
+ * instance gets that from autovacuum.
+ */
+
+const ACCOUNT = "acct-category-pg";
+const MAILBOX = "mbx-category-pg";
+
+describe("thread-message category predicate (postgres)", () => {
+	let pool: pg.Pool;
+	let close: () => Promise<void>;
+	let repo: DrizzleThreadMessageRepository;
+	const logged: Array<{ text: string; params: unknown[] }> = [];
+
+	before(async () => {
+		({ pool, close } = await createTestDb());
+
+		const db = drizzle(pool, {
+			schema,
+			logger: {
+				logQuery(text: string, params: unknown[]) {
+					logged.push({ text, params });
+				},
+			},
+		}) as unknown as Db<Record<string, unknown>>;
+		repo = new DrizzleThreadMessageRepository(db);
+
+		const rows = categoryFixtureRows({
+			totals: TRIMMED_TOTALS,
+			accountConfigId: ACCOUNT,
+			mailboxId: MAILBOX,
+		});
+		const insertDb = drizzle(pool, { schema });
+		for (let i = 0; i < rows.length; i += 500) {
+			await insertDb.insert(threadMessageTable).values(rows.slice(i, i + 500));
+		}
+		await pool.query("ANALYZE thread_message");
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	test("a filtered page is a full page of matches, however far back they sit", async () => {
+		const social = await repo.searchByMailboxWindow(
+			ACCOUNT,
+			MAILBOX,
+			{ category: ["social"] },
+			{ limit: 50, order: "desc", excludeDeleted: true },
+		);
+		assert.equal(social.items.length, 50);
+		assert.ok(social.items.every((item) => item.category === "social"));
+	});
+
+	test("a continuation token walks the whole match set without repeats or gaps", async () => {
+		const seen: string[] = [];
+		let continuationToken: string | undefined;
+		do {
+			const page = await repo.searchByMailboxWindow(
+				ACCOUNT,
+				MAILBOX,
+				{ category: ["social"] },
+				{
+					limit: 25,
+					order: "desc",
+					excludeDeleted: true,
+					continuationToken,
+				},
+			);
+			seen.push(...page.items.map((item) => item.threadMessageId));
+			continuationToken = page.continuationToken;
+		} while (continuationToken);
+		assert.equal(seen.length, TRIMMED_TOTALS.social);
+		assert.equal(new Set(seen).size, TRIMMED_TOTALS.social);
+	});
+
+	describe("query plan", () => {
+		// Each assertion names the category column as well as the index: both
+		// engines can reach this index on its (account_config_id, mailbox_id)
+		// prefix alone, so an index-only assertion would still pass with the
+		// predicate removed.
+		const assertServedByIndex = async (
+			run: () => Promise<unknown>,
+		): Promise<void> => {
+			logged.length = 0;
+			await run();
+			const selects = logged.filter((entry) => /^\s*select/i.test(entry.text));
+			assert.ok(selects.length > 0, "the repo issued a select");
+			const plan: string[] = [];
+			for (const entry of selects) {
+				const explained = await pool.query<{ "QUERY PLAN": string }>({
+					text: `EXPLAIN ${entry.text}`,
+					values: entry.params,
+				});
+				plan.push(...explained.rows.map((row) => row["QUERY PLAN"]));
+			}
+			const text = plan.join("\n");
+			assert.ok(
+				text.includes("tm_by_mailbox_category_date"),
+				`no plan node used the index:\n${text}`,
+			);
+			assert.ok(
+				/Index Cond:[^\n]*category/.test(text),
+				`the index was used but not on category:\n${text}`,
+			);
+		};
+
+		test("the filtered window is served by tm_by_mailbox_category_date", async () => {
+			await assertServedByIndex(() =>
+				repo.searchByMailboxWindow(
+					ACCOUNT,
+					MAILBOX,
+					{ category: ["social"] },
+					{ limit: 50, order: "desc", excludeDeleted: true },
+				),
+			);
+		});
+
+		test("the filtered count is served by tm_by_mailbox_category_date", async () => {
+			await assertServedByIndex(() =>
+				repo.countByMailbox(
+					ACCOUNT,
+					MAILBOX,
+					{ category: ["social"] },
+					{ excludeDeleted: true },
+				),
+			);
+		});
+	});
+});

package/src/repos/thread-message.sqlite.test.ts

@@ -3,6 +3,13 @@ import { after, before, describe, test } from "node:test";
 import type { CreateThreadMessageInput } from "@remit/data-ports";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
+import {
+	BASE_DATE,
+	type Category,
+	categoryFixtureRows,
+	LIVE_TOTALS,
+	totalRows,
+} from "./category-fixture.js";
 import { DrizzleThreadMessageRepository } from "./thread-message.js";
 
 // The thread-message repo on sqlite (RFC 036 D1): CRUD, keyset pagination, and
@@ -36,11 +43,12 @@ function makeInput(
 
 describe("DrizzleThreadMessageRepository (sqlite)", () => {
 	let db: Awaited<ReturnType<typeof createSqliteTestDb>>["db"];
+	let sqlite: Awaited<ReturnType<typeof createSqliteTestDb>>["sqlite"];
 	let close: () => Promise<void>;
 	let repo: DrizzleThreadMessageRepository;
 
 	before(async () => {
-		({ db, close } = await createSqliteTestDb(
+		({ db, sqlite, close } = await createSqliteTestDb(
 			{
 				threadMessage: threadMessageTable,
 			},
@@ -608,4 +616,360 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
 			);
 		});
 	});
+
+	// ─── category as a SQL predicate (#304) ───────────────────────────────────
+	//
+	// The shape is the owner's instance, not a convenience fixture — see
+	// ./category-fixture.ts. A filter applied to the page the server happens to
+	// return is empty or near-empty on that shape whatever the page size, which
+	// is the reported bug.
+	describe("category filter over the whole mailbox", () => {
+		const CAT_ACCOUNT = "acct-category";
+		const CAT_MAILBOX = "mbx-category";
+
+		before(async () => {
+			const rows = categoryFixtureRows({
+				totals: LIVE_TOTALS,
+				accountConfigId: CAT_ACCOUNT,
+				mailboxId: CAT_MAILBOX,
+			});
+			assert.equal(rows.length, totalRows(LIVE_TOTALS));
+			for (let i = 0; i < rows.length; i += 500) {
+				await db.insert(threadMessageTable).values(rows.slice(i, i + 500));
+			}
+		});
+
+		const walk = async (
+			category: Category[],
+			pageSize: number,
+		): Promise<string[]> => {
+			const seen: string[] = [];
+			let continuationToken: string | undefined;
+			do {
+				const page = await repo.searchByMailboxWindow(
+					CAT_ACCOUNT,
+					CAT_MAILBOX,
+					{ category },
+					{
+						limit: pageSize,
+						order: "desc",
+						excludeDeleted: true,
+						continuationToken,
+					},
+				);
+				seen.push(...page.items.map((item) => item.threadMessageId));
+				continuationToken = page.continuationToken;
+			} while (continuationToken);
+			return seen;
+		};
+
+		test("the newest page holds almost none of the rare categories", async () => {
+			const newest = await repo.searchByMailboxWindow(
+				CAT_ACCOUNT,
+				CAT_MAILBOX,
+				{},
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			assert.equal(newest.items.length, 50);
+			const count = (category: Category) =>
+				newest.items.filter((item) => item.category === category).length;
+			assert.ok(
+				count("social") <= 2,
+				"at most 2 social rows in the newest page",
+			);
+			assert.ok(
+				count("personal") <= 2,
+				"at most 2 personal rows in the newest page",
+			);
+		});
+
+		// The regression the epic turns on. A filter resolved over the returned
+		// page cannot produce this answer: on this mailbox the newest 50 rows hold
+		// two personal messages, so the old path returned two rows (or none, for
+		// social) with a continuation token attached.
+		test("a filtered page is a full page of matches, however far back they sit", async () => {
+			const personal = await repo.searchByMailboxWindow(
+				CAT_ACCOUNT,
+				CAT_MAILBOX,
+				{ category: ["personal"] },
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			assert.equal(personal.items.length, 50);
+			assert.ok(
+				personal.items.every((item) => item.category === "personal"),
+				"every row on the page matches the filter",
+			);
+
+			const social = await repo.searchByMailboxWindow(
+				CAT_ACCOUNT,
+				CAT_MAILBOX,
+				{ category: ["social"] },
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			assert.equal(social.items.length, 50);
+			assert.ok(social.items.every((item) => item.category === "social"));
+		});
+
+		test("the filtered page keeps newest-first order and the id tiebreak", async () => {
+			const page = await repo.searchByMailboxWindow(
+				CAT_ACCOUNT,
+				CAT_MAILBOX,
+				{ category: ["social"] },
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			for (let i = 1; i < page.items.length; i++) {
+				const previous = page.items[i - 1];
+				const current = page.items[i];
+				assert.ok(
+					previous.sentDate > current.sentDate ||
+						(previous.sentDate === current.sentDate &&
+							previous.threadMessageId < current.threadMessageId),
+					"rows descend by sentDate, ascending by id inside a tie group",
+				);
+			}
+		});
+
+		test("a continuation token walks the whole match set without repeats or gaps", async () => {
+			const seen = await walk(["social"], 25);
+			assert.equal(seen.length, LIVE_TOTALS.social);
+			assert.equal(new Set(seen).size, LIVE_TOTALS.social);
+		});
+
+		test("multiple categories behave as a union", async () => {
+			const page = await repo.searchByMailboxWindow(
+				CAT_ACCOUNT,
+				CAT_MAILBOX,
+				{ category: ["social", "transactional"] },
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			assert.equal(page.items.length, 50);
+			assert.ok(
+				page.items.every(
+					(item) =>
+						item.category === "social" || item.category === "transactional",
+				),
+			);
+
+			const seen = await walk(["social", "transactional"], 200);
+			assert.equal(seen.length, LIVE_TOTALS.social + LIVE_TOTALS.transactional);
+		});
+
+		// No `limit`: countByMailbox still clamps its answer to the clamped limit,
+		// which is what #305 changes. What this asserts is the predicate — the
+		// count is over the mailbox rather than over a page of it.
+		test("countByMailbox counts the matches in the mailbox, not the page", async () => {
+			assert.equal(
+				await repo.countByMailbox(
+					CAT_ACCOUNT,
+					CAT_MAILBOX,
+					{ category: ["social"] },
+					{ excludeDeleted: true },
+				),
+				LIVE_TOTALS.social,
+			);
+		});
+
+		test("the category predicate composes with the other filters", async () => {
+			const page = await repo.searchByMailboxWindow(
+				CAT_ACCOUNT,
+				CAT_MAILBOX,
+				{ category: ["social"], unread: true },
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			assert.equal(page.items.length, 50);
+			assert.ok(
+				page.items.every(
+					(item) => item.category === "social" && item.isRead === false,
+				),
+			);
+
+			const none = await repo.searchByMailboxWindow(
+				CAT_ACCOUNT,
+				CAT_MAILBOX,
+				{ category: ["social"], starred: true },
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			assert.equal(none.items.length, 0);
+		});
+
+		test("an empty category set is not a filter", async () => {
+			const page = await repo.searchByMailboxWindow(
+				CAT_ACCOUNT,
+				CAT_MAILBOX,
+				{ category: [] },
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			assert.equal(page.items.length, 50);
+			assert.ok(
+				page.items.some((item) => item.category !== page.items[0]?.category),
+				"the page is unfiltered, so it holds more than one category",
+			);
+		});
+
+		// #45: `uncategorized` is the not-yet-classified state as a named value.
+		// Folding it into `personal` made a classification gap read as a large
+		// personal inbox, so the filter has to be able to ask for it and must
+		// never answer with the other.
+		test("uncategorized is its own filterable value", async () => {
+			const account = "acct-category-default";
+			const mailbox = "mbx-category-default";
+			const base = BASE_DATE;
+			await repo.create(
+				makeInput({
+					accountConfigId: account,
+					mailboxId: mailbox,
+					messageId: "m-default",
+					subject: "not yet classified",
+					sentDate: base,
+					internalDate: base,
+				}),
+			);
+			await repo.create(
+				makeInput({
+					accountConfigId: account,
+					mailboxId: mailbox,
+					messageId: "m-personal",
+					subject: "classified personal",
+					category: "personal",
+					sentDate: base - 1000,
+					internalDate: base - 1000,
+				}),
+			);
+
+			const uncategorized = await repo.searchByMailboxWindow(
+				account,
+				mailbox,
+				{ category: ["uncategorized"] },
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			assert.deepEqual(
+				uncategorized.items.map((item) => item.subject),
+				["not yet classified"],
+			);
+
+			const personal = await repo.searchByMailboxWindow(
+				account,
+				mailbox,
+				{ category: ["personal"] },
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			assert.deepEqual(
+				personal.items.map((item) => item.subject),
+				["classified personal"],
+			);
+		});
+
+		test("soft-deleted rows stay out of a filtered page", async () => {
+			const account = "acct-category-deleted";
+			const mailbox = "mbx-category-deleted";
+			await repo.create(
+				makeInput({
+					accountConfigId: account,
+					mailboxId: mailbox,
+					messageId: "m-live-social",
+					subject: "live",
+					category: "social",
+				}),
+			);
+			await repo.create(
+				makeInput({
+					accountConfigId: account,
+					mailboxId: mailbox,
+					messageId: "m-trashed-social",
+					subject: "trashed",
+					category: "social",
+					isDeleted: true,
+				}),
+			);
+
+			const page = await repo.searchByMailboxWindow(
+				account,
+				mailbox,
+				{ category: ["social"] },
+				{ limit: 50, order: "desc", excludeDeleted: true },
+			);
+			assert.deepEqual(
+				page.items.map((item) => item.subject),
+				["live"],
+			);
+		});
+
+		// The guard on I1. Without it a rare-category page silently costs a
+		// mailbox scan and no test notices: the index is invisible to
+		// vps-migrations-drift.sqlite.test.ts, which compares the committed
+		// migration set against the same generated schema the index comes from.
+		//
+		// The statements are the ones the repo itself issued, captured from the
+		// driver, so this cannot pass against a query the repo does not run. Each
+		// assertion names the category column as well as the index: SQLite picks
+		// this index on its (account_config_id, mailbox_id) prefix alone, so an
+		// index-only assertion would still pass with the predicate removed.
+		describe("query plan", () => {
+			const selectsDuring = async (
+				run: () => Promise<unknown>,
+			): Promise<string[]> => {
+				const captured: string[] = [];
+				const original = sqlite.prepare.bind(sqlite);
+				const record = (source: string) => {
+					captured.push(source);
+					return original(source);
+				};
+				sqlite.prepare = record as typeof sqlite.prepare;
+				try {
+					await run();
+				} finally {
+					sqlite.prepare = original as typeof sqlite.prepare;
+				}
+				return captured.filter((source) => /^\s*select/i.test(source));
+			};
+
+			const plansFor = (statements: string[]): string[] => {
+				assert.ok(statements.length > 0, "the repo issued a select");
+				return statements.flatMap((source) => {
+					const parameters = new Array((source.match(/\?/g) ?? []).length).fill(
+						0,
+					);
+					const rows = sqlite
+						.prepare(`EXPLAIN QUERY PLAN ${source}`)
+						.all(...parameters) as Array<{ detail: string }>;
+					return rows.map((row) => row.detail);
+				});
+			};
+
+			const assertServedByIndex = (plan: string[]): void => {
+				assert.ok(
+					plan.some(
+						(detail) =>
+							detail.includes("tm_by_mailbox_category_date") &&
+							detail.includes("category=?"),
+					),
+					`no plan step matched on category through the index: ${plan.join(" | ")}`,
+				);
+			};
+
+			test("the filtered window is served by tm_by_mailbox_category_date", async () => {
+				const statements = await selectsDuring(() =>
+					repo.searchByMailboxWindow(
+						CAT_ACCOUNT,
+						CAT_MAILBOX,
+						{ category: ["social"] },
+						{ limit: 50, order: "desc", excludeDeleted: true },
+					),
+				);
+				assertServedByIndex(plansFor(statements));
+			});
+
+			test("the filtered count is served by tm_by_mailbox_category_date", async () => {
+				const statements = await selectsDuring(() =>
+					repo.countByMailbox(
+						CAT_ACCOUNT,
+						CAT_MAILBOX,
+						{ category: ["social"] },
+						{ excludeDeleted: true },
+					),
+				);
+				assertServedByIndex(plansFor(statements));
+			});
+		});
+	});
 });

package/src/repos/thread-message.ts

@@ -158,6 +158,13 @@ function buildSearchConditions(search: SearchOptions): SQL[] {
 	if (search.attachments !== undefined) {
 		conditions.push(eq(threadMessageTable.hasAttachment, search.attachments));
 	}
+	// An equality over a column on the row, so it sits inside the keyset window
+	// rather than filtering what the window returned: a page is a full page of
+	// matches however rare the category is. Served by
+	// tm_by_mailbox_category_date.
+	if (search.category?.length) {
+		conditions.push(inArray(threadMessageTable.category, search.category));
+	}
 
 	return conditions;
 }