@remit/drizzle-service 0.0.82 → 0.0.84

package/src/repos/thread-message-body-muted.sqlite.test.ts

@@ -0,0 +1,353 @@
+/**
+ * The two criteria the unified listing could not previously ask about: the body
+ * text a message carries, and whether its sender is muted.
+ *
+ * Both were client passes over the rows a page had already fetched, so both
+ * answered "among the mail loaded so far" while being presented as answers about
+ * the collection. The fixture puts the row that matters below the newest page in
+ * each case, which is exactly what such a pass cannot see (#1135, #1137).
+ */
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { after, before, describe, test } from "node:test";
+import type { CreateThreadMessageInput } from "@remit/data-ports";
+import { addressTable } from "../schema/i4-address.js";
+import { threadMessageTable } from "../schema/thread-message.js";
+import { createSqliteTestDb } from "../test-db-sqlite.js";
+import { DrizzleThreadMessageRepository } from "./thread-message.js";
+
+// The index the migrator installs for this predicate, read from the one
+// committed source so the plan assertion below is about the real object.
+const mutedIndexDdl = readFileSync(
+	new URL(
+		"../../../../npm-scripts/sqlite-address-muted-index.sql",
+		import.meta.url,
+	),
+	"utf8",
+);
+
+const ACCOUNT = "acct-seam";
+const MAILBOX = "mbx-seam";
+const SCOPE = new Set([MAILBOX]);
+
+const BASE_DATE = 1_700_000_000_000;
+
+let sequence = 0;
+const makeInput = (
+	overrides: Partial<CreateThreadMessageInput> = {},
+): CreateThreadMessageInput => {
+	sequence += 1;
+	return {
+		accountConfigId: ACCOUNT,
+		threadId: `t-${sequence}`,
+		messageId: `m-${sequence}`,
+		mailboxId: MAILBOX,
+		uid: sequence,
+		referenceOrder: 0,
+		internalDate: BASE_DATE,
+		sentDate: BASE_DATE,
+		isRead: false,
+		isDeleted: false,
+		hasAttachment: false,
+		hasStars: false,
+		...overrides,
+	};
+};
+
+describe("thread-message body and muted-sender predicates (sqlite)", () => {
+	let db: Awaited<ReturnType<typeof createSqliteTestDb>>["db"];
+	let sqlite: Awaited<ReturnType<typeof createSqliteTestDb>>["sqlite"];
+	let close: () => Promise<void>;
+	let repo: DrizzleThreadMessageRepository;
+
+	before(async () => {
+		({ db, sqlite, close } = await createSqliteTestDb(
+			{ threadMessage: threadMessageTable, address: addressTable },
+			{ searchIndex: true },
+		));
+		sqlite.exec(mutedIndexDdl);
+		repo = new DrizzleThreadMessageRepository(db);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	describe("a body-only match", () => {
+		before(async () => {
+			// Newer noise, so the matching row is not on the newest page.
+			for (let index = 0; index < 20; index += 1) {
+				await repo.create(
+					makeInput({
+						subject: "unrelated note",
+						fromEmail: "noise@example.com",
+						fromName: "Noise",
+						snippet: "nothing to see",
+						sentDate: BASE_DATE + 1000 + index,
+					}),
+				);
+			}
+			await repo.create(
+				makeInput({
+					subject: "unrelated note",
+					fromEmail: "noise@example.com",
+					fromName: "Noise",
+					snippet: "Your parcel was left with the concierge",
+					sentDate: BASE_DATE,
+				}),
+			);
+		});
+
+		test("the term reaches the body text, not just subject and From", async () => {
+			const result = await repo.searchByDate(
+				ACCOUNT,
+				{ query: "concierge" },
+				{ mailboxIds: SCOPE, limit: 50 },
+			);
+
+			assert.deepEqual(
+				result.items.map((item) => item.snippet),
+				["Your parcel was left with the concierge"],
+			);
+		});
+
+		// The point of moving it into the query: a page smaller than the noise
+		// still returns the match, where a pass over the loaded rows returns
+		// nothing until the reader has scrolled past it.
+		test("a match below the newest page is still returned", async () => {
+			const result = await repo.searchByDate(
+				ACCOUNT,
+				{ query: "concierge" },
+				{ mailboxIds: SCOPE, limit: 5 },
+			);
+
+			assert.equal(result.items.length, 1);
+		});
+
+		test("a term under the trigram floor reads the body too", async () => {
+			const result = await repo.searchByDate(
+				ACCOUNT,
+				{ query: "ge" },
+				{ mailboxIds: SCOPE, limit: 50 },
+			);
+
+			assert.ok(
+				result.items.some((item) => item.snippet?.includes("concierge")),
+				"the folded scan matched the body preview",
+			);
+		});
+
+		test("every term must still match somewhere", async () => {
+			const result = await repo.searchByDate(
+				ACCOUNT,
+				{ query: "concierge unrelated" },
+				{ mailboxIds: SCOPE, limit: 50 },
+			);
+			assert.equal(result.items.length, 1);
+
+			const none = await repo.searchByDate(
+				ACCOUNT,
+				{ query: "concierge zyxwvut" },
+				{ mailboxIds: SCOPE, limit: 50 },
+			);
+			assert.equal(none.items.length, 0);
+		});
+	});
+
+	describe("the muted-sender term", () => {
+		const MUTED_ACCOUNT = "acct-muted";
+		const MUTED_MAILBOX = "mbx-muted";
+		const MUTED_SCOPE = new Set([MUTED_MAILBOX]);
+
+		before(async () => {
+			await db.insert(addressTable).values([
+				{
+					addressId: "addr-muted",
+					accountConfigId: MUTED_ACCOUNT,
+					displayName: "Loud Marketer",
+					localPart: "loud",
+					domain: "example.com",
+					normalizedEmail: "loud@example.com",
+					normalizedCompound: "loud marketer loud@example.com",
+					flags: { muted: { value: true, setAt: 0 } } as never,
+					inboundCount: 0,
+					outboundCount: 0,
+					replyCount: 0,
+					lastInboundAt: 0,
+					lastReplyAt: 0,
+					createdAt: BASE_DATE,
+					updatedAt: BASE_DATE,
+				},
+				{
+					addressId: "addr-kept",
+					accountConfigId: MUTED_ACCOUNT,
+					displayName: "Colleague",
+					localPart: "kept",
+					domain: "example.com",
+					normalizedEmail: "kept@example.com",
+					normalizedCompound: "colleague kept@example.com",
+					flags: { muted: { value: false, setAt: 0 } } as never,
+					inboundCount: 0,
+					outboundCount: 0,
+					replyCount: 0,
+					lastInboundAt: 0,
+					lastReplyAt: 0,
+					createdAt: BASE_DATE,
+					updatedAt: BASE_DATE,
+				},
+			]);
+
+			// Three from the kept sender on top, the muted sender's mail below
+			// them: the arrangement the old client pass could not see.
+			for (let index = 0; index < 3; index += 1) {
+				await repo.create(
+					makeInput({
+						accountConfigId: MUTED_ACCOUNT,
+						mailboxId: MUTED_MAILBOX,
+						category: "marketing",
+						fromEmail: "kept@example.com",
+						fromName: "Colleague",
+						subject: `kept ${index}`,
+						sentDate: BASE_DATE + 1000 + index,
+					}),
+				);
+			}
+			for (let index = 0; index < 4; index += 1) {
+				await repo.create(
+					makeInput({
+						accountConfigId: MUTED_ACCOUNT,
+						mailboxId: MUTED_MAILBOX,
+						category: "marketing",
+						fromEmail: "loud@example.com",
+						fromName: "Loud Marketer",
+						subject: `muted ${index}`,
+						sentDate: BASE_DATE + index,
+					}),
+				);
+			}
+			// No Address row at all: unknown is not muted.
+			await repo.create(
+				makeInput({
+					accountConfigId: MUTED_ACCOUNT,
+					mailboxId: MUTED_MAILBOX,
+					category: "marketing",
+					fromEmail: "stranger@example.com",
+					fromName: "Stranger",
+					subject: "stranger",
+					sentDate: BASE_DATE - 1,
+				}),
+			);
+		});
+
+		test("muted=false drops the muted sender's mail from the listing", async () => {
+			const page = await repo.listByDate(MUTED_ACCOUNT, {
+				inboxMailboxIds: MUTED_SCOPE,
+				search: { muted: false },
+				limit: 50,
+			});
+
+			assert.deepEqual(page.items.map((item) => item.subject).sort(), [
+				"kept 0",
+				"kept 1",
+				"kept 2",
+				"stranger",
+			]);
+		});
+
+		// The defect: the header counted the muted sender's mail while the list
+		// dropped it, so "Show all" opened rows the brief would not render.
+		test("the count answers the same predicate as the listing", async () => {
+			const counted = await repo.countThreadsInScope(
+				MUTED_ACCOUNT,
+				{ muted: false },
+				{ mailboxIds: MUTED_SCOPE },
+			);
+			const wider = await repo.countThreadsInScope(
+				MUTED_ACCOUNT,
+				{},
+				{ mailboxIds: MUTED_SCOPE },
+			);
+
+			assert.equal(counted, 4);
+			assert.equal(wider, 8);
+		});
+
+		test("muted=true asks for the muted sender's mail alone", async () => {
+			const page = await repo.listByDate(MUTED_ACCOUNT, {
+				inboxMailboxIds: MUTED_SCOPE,
+				search: { muted: true },
+				limit: 50,
+			});
+
+			assert.equal(page.items.length, 4);
+			assert.ok(
+				page.items.every((item) => item.fromEmail === "loud@example.com"),
+				"only the muted sender",
+			);
+		});
+
+		test("an unstated muted term filters nothing", async () => {
+			const page = await repo.listByDate(MUTED_ACCOUNT, {
+				inboxMailboxIds: MUTED_SCOPE,
+				search: {},
+				limit: 50,
+			});
+
+			assert.equal(page.items.length, 8);
+		});
+
+		test("mute composes with the other criteria", async () => {
+			const counted = await repo.countThreadsInScope(
+				MUTED_ACCOUNT,
+				{ muted: false, category: ["marketing"] },
+				{ mailboxIds: MUTED_SCOPE },
+			);
+
+			assert.equal(counted, 4);
+		});
+
+		// The subquery runs once per candidate row, and the generated schema's
+		// only address index is on `normalized_compound`, which it cannot use. A
+		// brief counts seven sections, so an unindexed lookup here is seven scans
+		// of every address the config has ever seen.
+		test("the muted lookup is served by an index, never a scan", async () => {
+			const captured: string[] = [];
+			const original = sqlite.prepare.bind(sqlite);
+			sqlite.prepare = ((source: string) => {
+				captured.push(source);
+				return original(source);
+			}) as typeof sqlite.prepare;
+			try {
+				await repo.countThreadsInScope(
+					MUTED_ACCOUNT,
+					{ muted: false },
+					{ mailboxIds: MUTED_SCOPE },
+				);
+			} finally {
+				sqlite.prepare = original as typeof sqlite.prepare;
+			}
+
+			const selects = captured.filter((source) => /^\s*select/i.test(source));
+			assert.ok(selects.length > 0, "the repo issued a select");
+			const plan = selects.flatMap((source) => {
+				const parameters = new Array((source.match(/\?/g) ?? []).length).fill(
+					"",
+				);
+				return (
+					sqlite
+						.prepare(`EXPLAIN QUERY PLAN ${source}`)
+						.all(...parameters) as Array<{ detail: string }>
+				).map((row) => row.detail);
+			});
+
+			assert.ok(
+				plan.some((detail) => detail.includes("address_by_normalized_email")),
+				`the address lookup was not served by its index: ${plan.join(" | ")}`,
+			);
+			assert.ok(
+				!plan.some((detail) => /scan address/i.test(detail)),
+				`the address lookup fell back to a scan: ${plan.join(" | ")}`,
+			);
+		});
+	});
+});

package/src/repos/thread-message.ts

@@ -24,8 +24,10 @@ import type { Db } from "../db.js";
 import { NotFoundError } from "../error.js";
 import { deterministicBase36Id } from "../id.js";
 import { decodeToken } from "../pagination.js";
+import { addressTable } from "../schema/i4-address.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import {
+	bodyMatch,
 	fromMatch,
 	isNarrowableTerm,
 	listIdMatch,
@@ -127,10 +129,27 @@ function toItem(row: Row): ThreadMessageItem {
 // text-search seam; they live in ./thread-search-predicates.ts (the FTS5 trigram
 // index, with a folded LIKE fallback below three characters, RFC 036 D4).
 
+/**
+ * Whether the row's From address is muted, as a correlated subquery over the
+ * Address table.
+ *
+ * Muting is a flag on the address rather than a column on the row, so this is
+ * the one criterion that reaches outside `thread_message` — the read path
+ * denormalizes it onto the response afterwards, which is too late to count by.
+ * `normalized_email` is written folded, on the same rule the fold here applies,
+ * so the two meet. Both sides are already scoped to one account config, and
+ * correlating on that rather than binding it keeps the predicate usable from
+ * every caller without threading the id through.
+ */
+const mutedSender = (): SQL =>
+	sql`exists (select 1 from ${addressTable} where ${addressTable.accountConfigId} = ${threadMessageTable.accountConfigId} and ${addressTable.normalizedEmail} = lower(coalesce(${threadMessageTable.fromEmail}, '')) and json_extract(coalesce(nullif(${addressTable.flags}, ''), '{}'), '$.muted.value') = 1)`;
+
 // Translate SearchOptions into SQL conditions: subject/from/query as indexed
-// text predicates, the rest as plain column equalities. A multi-word `query`
-// matches rows where every token appears in the subject or the from fields
-// (AND across tokens, OR across fields) — the same shape as the DynamoDB model.
+// text predicates, muted as a subquery over the sender's address, the rest as
+// plain column equalities. A multi-word `query`
+// matches rows where every token appears in the subject, the from fields or the
+// body preview (AND across tokens, OR across fields) — the same shape as the
+// DynamoDB model.
 function buildSearchConditions(search: SearchOptions): SQL[] {
 	const conditions: SQL[] = [];
 
@@ -140,10 +159,16 @@ function buildSearchConditions(search: SearchOptions): SQL[] {
 	if (search.query) {
 		const tokens = search.query.split(/\s+/).filter(Boolean);
 		for (const token of tokens) {
-			conditions.push(sql`(${subjectMatch(token)} or ${fromMatch(token)})`);
+			conditions.push(
+				sql`(${subjectMatch(token)} or ${fromMatch(token)} or ${bodyMatch(token)})`,
+			);
 		}
 	}
 
+	if (search.muted !== undefined) {
+		conditions.push(search.muted ? mutedSender() : sql`not ${mutedSender()}`);
+	}
+
 	if (search.unread !== undefined) {
 		conditions.push(eq(threadMessageTable.isRead, !search.unread));
 	}

package/src/repos/thread-search-predicates.ts

@@ -10,7 +10,8 @@ const escapeLike = (term: string): string => term.replace(/[\\%_]/g, "\\$&");
 
 // Text search is the external-content FTS5 trigram index that
 // npm-scripts/sqlite-search-index.sql installs (RFC 036 D4): `thread_message_fts`
-// indexes the folded subject and sender, and MATCH is an accent- and
+// indexes the folded subject, the sender, and the body preview the row carries,
+// and MATCH is an accent- and
 // case-insensitive substring search (the tokenizer folds both sides, so the
 // needle is passed through untransformed). The predicate is a `rowid IN
 // (subquery)` over that index — the outer WHERE still narrows by mailbox.
@@ -53,6 +54,7 @@ const ftsRowidMatch = (matchExpr: string): SQL =>
 
 const SUBJECT_FOLDED = sql`lower(coalesce(subject, ''))`;
 const FROM_FOLDED = sql`lower(coalesce(from_name, '') || ' ' || coalesce(from_email, ''))`;
+const BODY_FOLDED = sql`lower(coalesce(snippet, ''))`;
 const LIST_ID_FOLDED = sql`lower(coalesce(list_id, ''))`;
 
 const likePattern = (term: string): SQL =>
@@ -68,6 +70,22 @@ export const fromMatch = (term: string): SQL =>
 		? ftsRowidMatch(`sender : ${ftsPhrase(term)}`)
 		: sql`${FROM_FOLDED} like ${likePattern(term)} escape '\\'`;
 
+/**
+ * Match the body text the row carries: the stored preview, quoted replies
+ * already removed, which is the same text the list renders under the subject.
+ *
+ * The brief used to reach this text with a pass over the rows a page had
+ * loaded, so what a search found depended on how far the reader had scrolled
+ * (#1135). It is a column like the other two, so it belongs in the index and in
+ * the predicate. A term further into a long message is still out of reach —
+ * the preview is what is stored — but a term the reader can see on the row is
+ * now found wherever that row sits in the collection.
+ */
+export const bodyMatch = (term: string): SQL =>
+	isTrigramIndexable(term)
+		? ftsRowidMatch(`body : ${ftsPhrase(term)}`)
+		: sql`${BODY_FOLDED} like ${likePattern(term)} escape '\\'`;
+
 // The FTS index carries subject and sender only, so a List-Id term is always
 // the folded LIKE scan. It is the narrowing half of a rule back-apply, where a
 // scan of one config's rows beats reading them all into the service (#459).

package/src/schema/i4-mailbox.ts

@@ -2,3 +2,5 @@ import * as entities from "@remit/drizzle-sqlite-schema";
 
 export const mailboxTable = entities.mailboxes;
 export const mailboxSpecialUseTable = entities.mailboxSpecialUseEntries;
+export const mailboxAttributeTable = entities.mailboxAttributeEntries;
+export const mailboxFlagTable = entities.mailboxFlags;