@remit/drizzle-service 1.0.1 → 2.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "1.0.1",
+  "version": "2.0.0",
   "description": "Drizzle ORM service over SQLite",
   "type": "module",
   "main": "src/index.ts",

package/src/databases.ts

@@ -4,11 +4,13 @@ import type { Db, SqliteStore } from "./db.js";
 import { messageDataSchema } from "./schema/message-data.js";
 import { createSqliteDatabase } from "./sqlite-client.js";
 
-// The three databases a self-host instance runs on (#1420). The app database
-// holds what lives in the client: accounts, config, rules, settings and auth.
-// The tenant mail database holds what lives on the IMAP server, plus the local
-// rows that refer to it. The tenant search database holds the full-text index,
-// rebuilt from the mail database, so it never commits together with it.
+// The three databases an instance runs on (#1420), each on the file its
+// SQLITE_*_DB_PATH names; a self-host points all three at one remit.db. The app
+// database holds what lives in the client: accounts, config, rules, settings
+// and auth. The tenant mail database holds what lives on the IMAP server, plus
+// the local rows that refer to it. The tenant search database holds the
+// full-text index, rebuilt from the mail database, so it never commits together
+// with it.
 export interface SqliteDatabases {
 	app: Db<Record<string, unknown>>;
 	mail: Db<Record<string, unknown>>;

package/src/schema-version-accounting.sqlite.test.ts

@@ -68,15 +68,18 @@ const journalEntries = (set: string): unknown[] => {
 };
 
 /**
- * The `<file>:<table>` pairs `read_schema_version` in the wrapper sums on the
- * live databases.
+ * The `<database>:<table>` pairs `read_schema_version` in the wrapper sums on
+ * the live databases, each database read from the file it is configured on.
  */
 const wrapperCountedTables = (): string[] => {
 	const match = read("deploy/vps/remit").match(
-		/for p in ((?:\w+\.db:__drizzle_migrations_\w+ ?)+); do/,
+		/for p in ((?:\$_sv_\w+:__drizzle_migrations_\w+ ?)+); do/,
 	);
 	assert.ok(match, "the wrapper no longer sums any __drizzle_migrations table");
-	return match[1].trim().split(/\s+/);
+	return match[1]
+		.trim()
+		.split(/\s+/)
+		.map((pair) => pair.replace(/^\$_sv_/, ""));
 };
 
 describe("schema version accounting", () => {
@@ -147,7 +150,7 @@ describe("schema version accounting", () => {
 	test("the wrapper counts exactly the tables the migrator writes", () => {
 		assert.deepEqual(
 			wrapperCountedTables().sort(),
-			sets.map(({ database, table }) => `${database}.db:${table}`).sort(),
+			sets.map(({ database, table }) => `${database}:${table}`).sort(),
 		);
 	});
 

package/src/search/thread-message-search.sqlite.test.ts

@@ -161,6 +161,29 @@ describe("the thread-message search index in its own database", () => {
 		assert.deepEqual(await found("receipt"), [created.threadMessageId]);
 	});
 
+	test("rebuilds while a live writer indexes a message the rebuild has yet to reach", async () => {
+		const created = await repo.create(input({ subject: "Raced reminder" }));
+		const store = syncStore(search.db, BETTER_SQLITE3_LIMITS);
+		let raced = false;
+		const migrator = new SearchIndex(mail.db, {
+			...store,
+			batch: async (items) => {
+				if (!raced) {
+					raced = true;
+					await index.sync([created.threadMessageId]);
+				}
+				return store.batch(items);
+			},
+		});
+
+		await migrator.rebuild();
+		await migrator.rebuild();
+
+		assert.equal(raced, true);
+		assert.equal(await index.needsRebuild(), false);
+		assert.deepEqual(await found("reminder"), [created.threadMessageId]);
+	});
+
 	test("matches the body text past the snippet, through a rewrite and a rebuild", async () => {
 		const created = await repo.create(
 			input({ subject: "Long letter", snippet: "Dear reader, a preview" }),

package/src/search/thread-message-search.ts

@@ -267,13 +267,24 @@ export class SearchIndex {
 				.limit(DOCUMENTS_PER_REBUILD_PAGE);
 			const last = rows.at(-1);
 			if (!last) return indexed;
+			// A live writer may index one of these messages after the delete
+			// above, so each page replaces its documents the way sync does.
 			await this.batch(
-				chunkRows(threadMessageSearchTable, rows, this.maxBoundParameters).map(
-					(slice) =>
-						this.search
-							.insert(threadMessageSearchTable)
-							.values(slice.map(toDocument)),
-				),
+				chunkRows(
+					threadMessageSearchTable,
+					rows,
+					this.maxBoundParameters,
+				).flatMap((slice) => [
+					this.search.delete(threadMessageSearchTable).where(
+						inArray(
+							threadMessageSearchTable.rowid,
+							slice.map((row) => searchRowid(row.threadMessageId)),
+						),
+					),
+					this.search
+						.insert(threadMessageSearchTable)
+						.values(slice.map(toDocument)),
+				]),
 			);
 			indexed += rows.length;
 			after = last.threadMessageId;