@remit/drizzle-service 0.0.99 → 0.0.101

package/src/repos/thread-search-predicates.ts

@@ -1,4 +1,5 @@
-import { type SQL, sql } from "drizzle-orm";
+import { type SQL, type SQLWrapper, sql } from "drizzle-orm";
+import { threadMessageSearchTable as fts } from "../search/thread-message-search.js";
 
 // Accent- and case-insensitive substring match over the whole mailbox, isolated
 // here as the one text-search seam whose behaviour is engine-specific. The
@@ -8,19 +9,20 @@ import { type SQL, sql } from "drizzle-orm";
 
 const escapeLike = (term: string): string => term.replace(/[\\%_]/g, "\\$&");
 
-// Text search is the external-content FTS5 trigram index that
-// npm-scripts/sqlite-search-index.sql installs (RFC 036 D4): `thread_message_fts`
-// indexes the folded subject, the sender, and the body preview the row carries,
-// and MATCH is an accent- and
-// case-insensitive substring search (the tokenizer folds both sides, so the
-// needle is passed through untransformed). The predicate is a `rowid IN
-// (subquery)` over that index — the outer WHERE still narrows by mailbox.
+// Every predicate here is a condition over `thread_message_fts`, the FTS5
+// trigram index in the tenant search database (#1420). It indexes the folded
+// subject, the sender and the body preview the row carries, and MATCH is an
+// accent- and case-insensitive substring search (the tokenizer folds both
+// sides, so the needle is passed through untransformed). The predicate is a
+// `rowid IN (subquery)` over the same table, so terms combine with AND and OR
+// like any other condition.
 //
 // Trigram indexing needs three characters, so a one- or two-character term
-// falls back to the unindexed folded LIKE scan D4 names — lower() both sides,
-// substring-match, `escape '\'` making the JS-escaped metacharacters literal.
-// It is case-insensitive for ASCII and does not fold diacritics; the accepted
-// difference between a short term and an indexed one (contract C10).
+// falls back to a folded LIKE scan of the index's own copy of the text —
+// lower() both sides, substring-match, `escape '\'` making the JS-escaped
+// metacharacters literal. It is case-insensitive for ASCII and does not fold
+// diacritics; the accepted difference between a short term and an indexed one
+// (contract C10).
 
 // FTS5 treats bare query text as its match grammar (AND/OR/NEAR/`*`/`-`/`:`), so
 // wrap the term as a double-quoted string literal — doubling embedded quotes —
@@ -50,25 +52,23 @@ export const isNarrowableTerm = (term: string): boolean =>
 	isTrigramIndexable(term) || isAscii(term);
 
 const ftsRowidMatch = (matchExpr: string): SQL =>
-	sql`"thread_message"."rowid" in (select "rowid" from "thread_message_fts" where "thread_message_fts" match ${matchExpr})`;
-
-const SUBJECT_FOLDED = sql`lower(coalesce(subject, ''))`;
-const FROM_FOLDED = sql`lower(coalesce(from_name, '') || ' ' || coalesce(from_email, ''))`;
-const BODY_FOLDED = sql`lower(coalesce(snippet, ''))`;
-const LIST_ID_FOLDED = sql`lower(coalesce(list_id, ''))`;
+	sql`${fts}.rowid in (select rowid from ${fts} where ${fts} match ${matchExpr})`;
 
 const likePattern = (term: string): SQL =>
 	sql`'%' || lower(${escapeLike(term)}) || '%'`;
 
+const folded = (column: SQLWrapper, term: string): SQL =>
+	sql`lower(${column}) like ${likePattern(term)} escape '\\'`;
+
 export const subjectMatch = (term: string): SQL =>
 	isTrigramIndexable(term)
 		? ftsRowidMatch(`subject : ${ftsPhrase(term)}`)
-		: sql`${SUBJECT_FOLDED} like ${likePattern(term)} escape '\\'`;
+		: folded(fts.subject, term);
 
 export const fromMatch = (term: string): SQL =>
 	isTrigramIndexable(term)
 		? ftsRowidMatch(`sender : ${ftsPhrase(term)}`)
-		: sql`${FROM_FOLDED} like ${likePattern(term)} escape '\\'`;
+		: folded(fts.sender, term);
 
 /**
  * Match the body text the row carries: the stored preview, quoted replies
@@ -84,13 +84,11 @@ export const fromMatch = (term: string): SQL =>
 export const bodyMatch = (term: string): SQL =>
 	isTrigramIndexable(term)
 		? ftsRowidMatch(`body : ${ftsPhrase(term)}`)
-		: sql`${BODY_FOLDED} like ${likePattern(term)} escape '\\'`;
+		: folded(fts.body, term);
 
-// The FTS index carries subject and sender only, so a List-Id term is always
-// the folded LIKE scan. It is the narrowing half of a rule back-apply, where a
-// scan of one config's rows beats reading them all into the service (#459).
-export const listIdMatch = (term: string): SQL =>
-	sql`${LIST_ID_FOLDED} like ${likePattern(term)} escape '\\'`;
+// The List-Id is carried unindexed, so a List-Id term is always the folded LIKE
+// scan. It is the narrowing half of a rule back-apply (#459).
+export const listIdMatch = (term: string): SQL => folded(fts.listId, term);
 
 export const senderMatchRank = (terms: readonly string[]): SQL<number> =>
 	terms.length === 0

package/src/schema/message-data.ts

@@ -1,24 +1,21 @@
 import * as entities from "@remit/drizzle-sqlite-schema";
-import { OUTBOX_EVENTS, type OutboxEvent, outboxTable } from "./outbox.js";
 
-export { OUTBOX_EVENTS, type OutboxEvent, outboxTable };
-
-const bodyPartContentTable = entities.bodyPartContents;
 const bodyPartParameterTable = entities.bodyPartParameters;
 const bodyPartStorageTable = entities.bodyPartStorages;
 const bodyPartTable = entities.bodyParts;
+const bodyStructureRefetchTable = entities.bodyStructureRefetches;
 const envelopeAddressTable = entities.envelopeAddresses;
 const envelopeTable = entities.envelopes;
 const messageFlagTable = entities.messageFlags;
 const messageReferenceTable = entities.messageReferences;
-const messageTable = entities.messages;
+const messageTable = entities.messageRecords;
 const rawMessageStorageTable = entities.rawMessageStorages;
 
 export {
-	bodyPartContentTable,
 	bodyPartParameterTable,
 	bodyPartStorageTable,
 	bodyPartTable,
+	bodyStructureRefetchTable,
 	envelopeAddressTable,
 	envelopeTable,
 	messageFlagTable,
@@ -35,10 +32,9 @@ export const messageDataSchema = {
 	bodyPartParameter: bodyPartParameterTable,
 	rawMessageStorage: rawMessageStorageTable,
 	bodyPartStorage: bodyPartStorageTable,
-	bodyPartContent: bodyPartContentTable,
+	bodyStructureRefetch: bodyStructureRefetchTable,
 	message: messageTable,
 	messageFlag: messageFlagTable,
-	outbox: outboxTable,
 };
 
 export type MessageDataSchema = typeof messageDataSchema;

package/src/schema/sync-warning.ts

@@ -0,0 +1,3 @@
+import * as entities from "@remit/drizzle-sqlite-schema";
+
+export const syncWarningTable = entities.syncWarnings;

package/src/schema-sets/app.ts

@@ -0,0 +1,17 @@
+// The app database's entity tables (#1420): what lives in the client —
+// accounts, config, rules, settings and calendars. The better-auth identity
+// tables join them from auth-service; the app migration set's drizzle-kit
+// config lists both sources.
+export {
+	accountConfigs,
+	accountExportRequests,
+	accountSettings,
+	accounts,
+	calendarCollections,
+	calendarEventIndexes,
+	calendarFeedTokens,
+	calendarObjects,
+	configImports,
+	filters,
+	labels,
+} from "@remit/drizzle-sqlite-schema";

package/src/schema-sets/mail.ts

@@ -0,0 +1,32 @@
+// The tenant mail database's tables (#1420): what lives on the IMAP server,
+// and the local rows that refer to it — the outbox, quarantine, labels on
+// messages, calendar suggestions, organize jobs and sender standings.
+export {
+	addresses,
+	bodyPartParameters,
+	bodyPartStorages,
+	bodyParts,
+	bodyStructureRefetches,
+	calendarSuggestions,
+	envelopeAddresses,
+	envelopes,
+	mailboxAttributeEntries,
+	mailboxes,
+	mailboxFlags,
+	mailboxLocks,
+	mailboxSpecialUseEntries,
+	messageFlagPushes,
+	messageFlags,
+	messageLabels,
+	messagePlacementMoves,
+	messageRecords,
+	messageReferences,
+	organizeJobRequests,
+	outboxAttachments,
+	outboxMessages,
+	quarantines,
+	rawMessageStorages,
+	senderSignerStandings,
+	syncWarnings,
+	threadMessages,
+} from "@remit/drizzle-sqlite-schema";

package/src/schema-sets/search.ts

@@ -0,0 +1,5 @@
+// The tenant search database (#1420) holds one FTS5 virtual table, which
+// drizzle-kit cannot express: its DDL is the custom migration committed in
+// deploy/vps/migrations-sqlite/search. This set therefore declares no table,
+// and the drift guard holds it to that.
+export {};

package/src/schema-version-accounting.sqlite.test.ts

@@ -21,9 +21,9 @@ import { migrate } from "drizzle-orm/better-sqlite3/migrator";
  * reported ten against a manifest's nine inverted the derivation with no test
  * to catch it.
  *
- * This applies the shipped migration sets with the shipped migrator and asserts
- * the two quantities agree: one applied row per journal entry, in exactly the
- * tables the wrapper counts.
+ * This applies the shipped migration sets with the shipped migrator, each to
+ * its own database file, and asserts the two quantities agree: one applied row
+ * per journal entry, in exactly the tables and files the wrapper counts.
  */
 
 const REPO_ROOT = new URL("../../../", import.meta.url);
@@ -34,24 +34,25 @@ const read = (path: string): string =>
 	readFileSync(new URL(path, REPO_ROOT), "utf8");
 
 interface MigrationSet {
+	database: string;
 	set: string;
 	table: string;
 }
 
 /**
- * The sets the migrate one-shot applies, taken from the entrypoint itself
- * rather than restated here: a set added, renamed or dropped there is one this
- * test then migrates and counts, instead of one it silently stops covering.
- * The folder is the path inside the image, where the migrations are staged at
- * the working directory; in this tree they are under deploy/vps.
+ * The sets the migrate one-shot applies, taken from its own list rather than
+ * restated here: a set added, renamed or dropped there is one this test then
+ * migrates and counts, instead of one it silently stops covering. The folder is
+ * the path inside the image, where the migrations are staged at the working
+ * directory; in this tree they are under deploy/vps.
  */
 const migrationSets = (): MigrationSet[] => {
-	const source = read("packages/migrate/src/run-migrate.ts");
+	const source = read("packages/migrate/src/migration-sets.ts");
 	const sets = [
 		...source.matchAll(
-			/migrationsFolder:\s*"migrations-sqlite\/(\w+)",\s*migrationsTable:\s*"(\w+)",/g,
+			/database:\s*"(\w+)",\s*migrationsFolder:\s*"migrations-sqlite\/(\w+)",\s*migrationsTable:\s*"(\w+)",/g,
 		),
-	].map(([, set, table]) => ({ set, table }));
+	].map(([, database, set, table]) => ({ database, set, table }));
 	assert.ok(
 		sets.length > 0,
 		"no sqlite migration sets found in the migrate entrypoint",
@@ -66,10 +67,13 @@ const journalEntries = (set: string): unknown[] => {
 	return journal.entries;
 };
 
-/** The tables `read_schema_version` in the wrapper sums on the live database. */
+/**
+ * The `<file>:<table>` pairs `read_schema_version` in the wrapper sums on the
+ * live databases.
+ */
 const wrapperCountedTables = (): string[] => {
 	const match = read("deploy/vps/remit").match(
-		/for t in ((?:__drizzle_migrations_\w+ ?)+); do/,
+		/for p in ((?:\w+\.db:__drizzle_migrations_\w+ ?)+); do/,
 	);
 	assert.ok(match, "the wrapper no longer sums any __drizzle_migrations table");
 	return match[1].trim().split(/\s+/);
@@ -78,35 +82,49 @@ const wrapperCountedTables = (): string[] => {
 describe("schema version accounting", () => {
 	const dir = mkdtempSync(join(tmpdir(), "remit-schema-version-"));
 	const sets = migrationSets();
-	const sqlite = new Database(join(dir, "remit.db"));
-	sqlite.pragma("journal_mode = WAL");
-	sqlite.pragma("foreign_keys = ON");
-	const db = drizzle(sqlite);
-	for (const { set, table } of sets) {
-		migrate(db, {
-			migrationsFolder: fileURLToPath(new URL(set, MIGRATIONS_ROOT)),
-			migrationsTable: table,
-		});
-	}
+	const databases = new Map(
+		[...new Set(sets.map(({ database }) => database))].map((database) => {
+			const sqlite = new Database(join(dir, `${database}.db`));
+			sqlite.pragma("journal_mode = WAL");
+			sqlite.pragma("foreign_keys = ON");
+			return [database, sqlite] as const;
+		}),
+	);
+	const sqliteOf = (database: string): Database.Database => {
+		const sqlite = databases.get(database);
+		assert.ok(sqlite, `no database file for ${database}`);
+		return sqlite;
+	};
+	const migrateAll = (): void => {
+		for (const { database, set, table } of sets) {
+			migrate(drizzle(sqliteOf(database)), {
+				migrationsFolder: fileURLToPath(new URL(set, MIGRATIONS_ROOT)),
+				migrationsTable: table,
+			});
+		}
+	};
+	migrateAll();
 
 	after(() => {
-		sqlite.close();
+		for (const sqlite of databases.values()) sqlite.close();
 		rmSync(dir, { recursive: true, force: true });
 	});
 
-	const rows = (table: string): number =>
+	const rows = ({ database, table }: MigrationSet): number =>
 		(
-			sqlite.prepare(`SELECT count(*) AS n FROM ${table}`).get() as {
+			sqliteOf(database)
+				.prepare(`SELECT count(*) AS n FROM ${table}`)
+				.get() as {
 				n: number;
 			}
 		).n;
 
 	test("a fully migrated database holds one row per journal entry", () => {
-		for (const { set, table } of sets) {
+		for (const set of sets) {
 			assert.equal(
-				rows(table),
-				journalEntries(set).length,
-				`${table} does not hold one row per entry in the ${set} journal`,
+				rows(set),
+				journalEntries(set.set).length,
+				`${set.table} does not hold one row per entry in the ${set.set} journal`,
 			);
 		}
 	});
@@ -115,7 +133,7 @@ describe("schema version accounting", () => {
 	// from these same journals — so this is `deriveSchemaVersion` against the
 	// count the wrapper reads back off a migrated instance.
 	test("the total equals the schema version the manifest derives", () => {
-		const applied = sets.reduce((total, { table }) => total + rows(table), 0);
+		const applied = sets.reduce((total, set) => total + rows(set), 0);
 		const derived = sets.reduce(
 			(total, { set }) => total + journalEntries(set).length,
 			0,
@@ -123,13 +141,13 @@ describe("schema version accounting", () => {
 		assert.equal(applied, derived);
 	});
 
-	// Same number, same tables. A set the migrator writes and the wrapper does
-	// not count is a version that reads low forever, which inverts the consent
-	// screen's comparison rather than failing it.
+	// Same number, same tables, same files. A set the migrator writes and the
+	// wrapper does not count is a version that reads low forever, which inverts
+	// the consent screen's comparison rather than failing it.
 	test("the wrapper counts exactly the tables the migrator writes", () => {
 		assert.deepEqual(
 			wrapperCountedTables().sort(),
-			sets.map(({ table }) => table).sort(),
+			sets.map(({ database, table }) => `${database}.db:${table}`).sort(),
 		);
 	});
 
@@ -137,15 +155,10 @@ describe("schema version accounting", () => {
 	// re-recorded an applied migration would lift the count above the journal
 	// sum — the shape of the drift #279 reported.
 	test("a second migrate run records nothing further", () => {
-		const before = sets.reduce((total, { table }) => total + rows(table), 0);
-		for (const { set, table } of sets) {
-			migrate(db, {
-				migrationsFolder: fileURLToPath(new URL(set, MIGRATIONS_ROOT)),
-				migrationsTable: table,
-			});
-		}
+		const before = sets.reduce((total, set) => total + rows(set), 0);
+		migrateAll();
 		assert.equal(
-			sets.reduce((total, { table }) => total + rows(table), 0),
+			sets.reduce((total, set) => total + rows(set), 0),
 			before,
 		);
 	});

package/src/schema.ts

@@ -2,9 +2,8 @@
 // the names the repos and `pushSchema` (test-db.ts) consume. Each table appears
 // exactly once here — `pushSchema` registers a table's indexes per exported
 // binding, so exposing one table under two names creates duplicate index names
-// and breaks `apply()`. The committed-migration `generate` reads
-// schema-full-sqlite.ts instead (the entity package wholesale), so the
-// migration is driven by the entities, not by this hand-maintained alias list.
+// and breaks `apply()`. The committed migrations are generated from the
+// per-database lists in schema-sets/, not from this alias list.
 
 import * as entities from "@remit/drizzle-sqlite-schema";
 
@@ -13,7 +12,6 @@ export const calendarObjectTable = entities.calendarObjects;
 export const calendarEventIndexTable = entities.calendarEventIndexes;
 export const calendarFeedTokenTable = entities.calendarFeedTokens;
 export const calendarSuggestionTable = entities.calendarSuggestions;
-export const filterAnchorTable = entities.filterAnchors;
 export const filterTable = entities.filters;
 export const labelTable = entities.labels;
 export const messageLabelTable = entities.messageLabels;
@@ -32,4 +30,5 @@ export * from "./schema/i4-outbox-attachment.js";
 export * from "./schema/i4-outbox-message.js";
 export * from "./schema/message-data.js";
 export * from "./schema/quarantine.js";
+export * from "./schema/sync-warning.js";
 export { threadMessageTable } from "./schema/thread-message.js";

package/src/search/thread-message-search.sqlite.test.ts

@@ -0,0 +1,131 @@
+import assert from "node:assert/strict";
+import { after, before, describe, test } from "node:test";
+import type { CreateThreadMessageInput } from "@remit/data-ports";
+import Database from "better-sqlite3";
+import { drizzle } from "drizzle-orm/better-sqlite3";
+import type { Db } from "../db.js";
+import { DrizzleThreadMessageRepository } from "../repos/thread-message.js";
+import { applyMigrationSet } from "../test-shipped-sqlite-schema.js";
+import {
+	SearchIndex,
+	threadMessageSearchTable,
+} from "./thread-message-search.js";
+
+// The search index lives in its own database (#1420): a thread message written
+// to the mail database is findable once the search write that follows it has
+// run, and a search never reads the mail database's text to find it.
+
+const ACCOUNT = "acct-search";
+const MAILBOX = "mbx-search";
+
+const input = (
+	overrides: Partial<CreateThreadMessageInput>,
+): CreateThreadMessageInput => ({
+	accountConfigId: ACCOUNT,
+	threadId: `t-${Math.random().toString(36).slice(2)}`,
+	messageId: `m-${Math.random().toString(36).slice(2)}`,
+	mailboxId: MAILBOX,
+	uid: 1,
+	referenceOrder: 0,
+	internalDate: 0,
+	sentDate: 0,
+	isRead: false,
+	isDeleted: false,
+	hasAttachment: false,
+	hasStars: false,
+	...overrides,
+});
+
+const open = (set: "mail" | "search") => {
+	const sqlite = new Database(":memory:");
+	applyMigrationSet(sqlite, set);
+	return {
+		sqlite,
+		db: drizzle(sqlite) as unknown as Db<Record<string, unknown>>,
+	};
+};
+
+describe("the thread-message search index in its own database", () => {
+	const mail = open("mail");
+	const search = open("search");
+	const index = new SearchIndex(mail.db, search.db);
+	const repo = new DrizzleThreadMessageRepository(mail.db, index);
+
+	const found = async (query: string): Promise<string[]> =>
+		(await repo.searchByMailboxWindow(ACCOUNT, MAILBOX, { query })).items.map(
+			(item) => item.threadMessageId,
+		);
+
+	const documents = (): number =>
+		(
+			search.sqlite
+				.prepare("SELECT count(*) AS n FROM thread_message_fts")
+				.get() as { n: number }
+		).n;
+
+	before(() => {
+		assert.equal(
+			(
+				mail.sqlite
+					.prepare(
+						"SELECT count(*) AS n FROM sqlite_master WHERE name = 'thread_message_fts'",
+					)
+					.get() as { n: number }
+			).n,
+			0,
+			"the mail database carries no search table",
+		);
+	});
+
+	after(() => {
+		mail.sqlite.close();
+		search.sqlite.close();
+	});
+
+	test("finds a message by subject, sender and body once it is written", async () => {
+		const created = await repo.create(
+			input({
+				subject: "Quarterly invoice",
+				fromName: "Ada Lovelace",
+				fromEmail: "ada@example.test",
+				snippet: "the analytical engine",
+			}),
+		);
+
+		assert.deepEqual(await found("invoice"), [created.threadMessageId]);
+		assert.deepEqual(await found("lovelace"), [created.threadMessageId]);
+		assert.deepEqual(await found("analytical"), [created.threadMessageId]);
+		assert.deepEqual(await found("nowhere"), []);
+	});
+
+	test("follows a rewritten subject and drops the old one", async () => {
+		const created = await repo.create(input({ subject: "Draft agenda" }));
+		await repo.update(ACCOUNT, created.threadMessageId, {
+			subject: "Final minutes",
+		});
+
+		assert.deepEqual(await found("agenda"), []);
+		assert.deepEqual(await found("minutes"), [created.threadMessageId]);
+	});
+
+	test("forgets a deleted message", async () => {
+		const created = await repo.create(input({ subject: "Ephemeral notice" }));
+		const before = documents();
+		await repo.delete(ACCOUNT, created.threadMessageId);
+
+		assert.deepEqual(await found("ephemeral"), []);
+		assert.equal(documents(), before - 1);
+	});
+
+	test("rebuilds every document from the mail database", async () => {
+		const created = await repo.create(input({ subject: "Rebuilt receipt" }));
+		await search.db.delete(threadMessageSearchTable);
+		assert.equal(await index.needsRebuild(), true);
+		assert.deepEqual(await found("receipt"), []);
+
+		await index.rebuild();
+
+		assert.equal(await index.needsRebuild(), false);
+		assert.deepEqual(await found("receipt"), [created.threadMessageId]);
+	});
+});