@remit/mailbox-service 0.0.112 → 0.0.114

package/src/test-helpers/test-store.ts

@@ -1,299 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { copyFileSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import type {
-	AccountItem,
-	CreateMessageInput,
-	CreateThreadMessageInput,
-	MailboxItem,
-	MailboxSpecialUseValue,
-	MessageItem,
-	ThreadMessageItem,
-} from "@remit/data-ports";
-import { AccountAuthType, ConnectionState } from "@remit/domain-enums";
-import {
-	createSqliteRepositories,
-	openSqliteDatabases,
-} from "@remit/drizzle-service";
-import { applyShippedMigrations } from "@remit/drizzle-service/test-sqlite";
-import type { StorageService } from "@remit/storage-service";
-import { createFilesystemStorageService } from "@remit/storage-service/filesystem";
-import Database from "better-sqlite3";
-import { MessageSyncService } from "../message-sync.js";
-import { SyncWarningService } from "../sync-warning.js";
-import type { FakeImapServer } from "./fake-imap.js";
-
-export {
-	captureQueue,
-	type QueueCapture,
-	type QueuedEvent,
-} from "./test-queue.js";
-
-const UPLOAD_URLS = {
-	origin: "https://reader.test",
-	signingSecret: "test-signing-secret",
-};
-
-/**
- * The store the app opens: the committed migrations applied to a file, then
- * that file opened the way the app opens it — as the app, mail and search
- * database at once — with every repo the app composes.
- * Storage made from it lives in the same directory and goes with it.
- */
-let migrated: string | undefined;
-
-const migratedTemplate = (): string => {
-	if (migrated) return migrated;
-	const dir = mkdtempSync(join(tmpdir(), "mbx-template-"));
-	const filename = join(dir, "template.sqlite");
-	const migrating = new Database(filename);
-	migrating.pragma("synchronous = OFF");
-	applyShippedMigrations(migrating);
-	migrating.close();
-	process.once("exit", () => rmSync(dir, { recursive: true, force: true }));
-	migrated = filename;
-	return filename;
-};
-
-export const openStore = async () => {
-	const dir = mkdtempSync(join(tmpdir(), "mbx-store-"));
-	const filename = join(dir, "store.sqlite");
-	copyFileSync(migratedTemplate(), filename);
-	const { close, ...databases } = await openSqliteDatabases({
-		app: filename,
-		mail: filename,
-		search: filename,
-	});
-	let storages = 0;
-	return {
-		...createSqliteRepositories(databases),
-		storage: (): StorageService =>
-			createFilesystemStorageService(
-				join(dir, `storage-${++storages}`),
-				UPLOAD_URLS,
-			),
-		brokenStorage: (): StorageService => {
-			const root = join(dir, `broken-${++storages}`);
-			writeFileSync(root, "");
-			return createFilesystemStorageService(root);
-		},
-		close: async () => {
-			await close();
-			rmSync(dir, { recursive: true, force: true });
-		},
-	};
-};
-
-export type Store = Awaited<ReturnType<typeof openStore>>;
-
-export const seedAccount = async (
-	store: Store,
-	overrides: Partial<AccountItem> = {},
-): Promise<AccountItem> => {
-	const config = await store.accountConfig.create({
-		accountConfigId: overrides.accountConfigId,
-		userId: "user-1",
-		name: "Me",
-	});
-	return store.account.create({
-		username: "me@example.com",
-		email: "me@example.com",
-		authType: AccountAuthType.Password,
-		passwordHash: '{"ciphertext":"x"}',
-		imapHost: "imap.example.com",
-		imapPort: 993,
-		imapTls: true,
-		imapStartTls: false,
-		isActive: true,
-		connectionState: ConnectionState.Authenticated,
-		...overrides,
-		accountConfigId: config.accountConfigId,
-	});
-};
-
-export const seedMailbox = async (
-	store: Store,
-	account: Pick<AccountItem, "accountId">,
-	fullPath: string,
-	options: {
-		specialUse?: MailboxSpecialUseValue;
-		overrides?: Partial<MailboxItem>;
-	} = {},
-): Promise<MailboxItem> => {
-	const created = await store.mailbox.create({
-		accountId: account.accountId,
-		namespacePrefix: "",
-		hierarchyDelimiter: "/",
-		fullPath,
-		uidValidity: 7,
-		uidNext: 1,
-		highestModseq: "0",
-		messageCount: 0,
-		unseenCount: 0,
-		deletedCount: 0,
-		totalSize: 0,
-		lastSyncUid: 0,
-		highWaterMarkUid: 0,
-		lastMessageSyncAt: 0,
-		...(options.specialUse ? { specialUse: [options.specialUse] } : {}),
-		...options.overrides,
-	});
-	if (created.outcome !== "Created") {
-		throw new Error(`${fullPath} already exists`);
-	}
-	if (options.specialUse) {
-		await store.mailboxSpecialUse.create(
-			created.mailbox.mailboxId,
-			options.specialUse,
-		);
-	}
-	return created.mailbox;
-};
-
-export interface SeededMessage {
-	message: MessageItem;
-	threadMessage: ThreadMessageItem;
-}
-
-export const seedMessage = async (
-	store: Store,
-	account: Pick<AccountItem, "accountConfigId">,
-	mailbox: Pick<MailboxItem, "mailboxId">,
-	uid: number,
-	overrides: {
-		message?: Partial<CreateMessageInput>;
-		threadMessage?: Partial<CreateThreadMessageInput>;
-	} = {},
-): Promise<SeededMessage> => {
-	const messageId = overrides.message?.messageId ?? randomUUID();
-	const message = await store.message.create({
-		messageId,
-		mailboxId: mailbox.mailboxId,
-		uid,
-		sequenceNumber: uid,
-		rfc822Size: 100,
-		internalDate: 1_700_000_000_000 + uid,
-		envelopeId: randomUUID(),
-		rootBodyPartId: randomUUID(),
-		status: "active",
-		syncStatus: "synced",
-		...overrides.message,
-	});
-	const threadMessage = await store.threadMessage.create({
-		accountConfigId: account.accountConfigId,
-		threadId: messageId,
-		messageId,
-		mailboxId: message.mailboxId,
-		uid: message.uid,
-		referenceOrder: 0,
-		internalDate: message.internalDate,
-		sentDate: message.internalDate,
-		isRead: false,
-		isDeleted: false,
-		hasAttachment: false,
-		hasStars: false,
-		...overrides.threadMessage,
-	});
-	return { message, threadMessage };
-};
-
-export const silentLog = {
-	info: () => {},
-	warn: () => {},
-	error: () => {},
-	debug: () => {},
-};
-
-export const syncFolder = async (
-	store: Store,
-	account: Pick<AccountItem, "accountId" | "accountConfigId">,
-	mailbox: Pick<MailboxItem, "mailboxId">,
-	imap: FakeImapServer,
-): Promise<string[]> => {
-	const connection = imap.connection();
-	const service = new MessageSyncService(
-		{ getConnection: () => connection, close: async () => {} },
-		store.mailbox,
-		store.mailboxSpecialUse,
-		store.message,
-		store.envelope,
-		store.address,
-		store.threadMessage,
-		new SyncWarningService(store.syncWarning),
-		silentLog,
-		store.flagPush,
-		store.messageFlag,
-	);
-	const synced: string[] = [];
-	for (;;) {
-		const result = await service.syncMessages(
-			mailbox.mailboxId,
-			account.accountId,
-			account.accountConfigId,
-		);
-		synced.push(...result.syncedMessageIds);
-		if (!result.hasMore) return synced;
-	}
-};
-
-export const messageAtUid = async (
-	store: Store,
-	mailbox: Pick<MailboxItem, "mailboxId">,
-	uid: number,
-): Promise<MessageItem> => {
-	const found = (await store.message.listAllByMailbox(mailbox.mailboxId)).find(
-		(message) => message.uid === uid,
-	);
-	if (!found) throw new Error(`no message at uid ${uid}`);
-	return found;
-};
-
-export const arriveAt = async (
-	store: Store,
-	imap: FakeImapServer,
-	account: Pick<AccountItem, "accountId" | "accountConfigId">,
-	mailbox: Pick<MailboxItem, "mailboxId" | "fullPath">,
-	raw: Buffer,
-): Promise<string> => {
-	const uid = imap.deliver(mailbox.fullPath, raw);
-	await syncFolder(store, account, mailbox, imap);
-	return (await messageAtUid(store, mailbox, uid)).messageId;
-};
-
-export const storeBody = async (
-	store: Store,
-	storage: StorageService,
-	account: Pick<AccountItem, "accountId" | "accountConfigId">,
-	messageId: string,
-	raw: Buffer,
-): Promise<void> => {
-	const ref = await storage.storeMessageBody({
-		accountConfigId: account.accountConfigId,
-		accountId: account.accountId,
-		messageId,
-		content: raw,
-	});
-	await store.message.update(messageId, { bodyStorageKey: ref.uri });
-};
-
-/**
- * Another writer landing between a service's read and its write: `action` runs,
- * for real, the next time `method` is called on `target`, and the call then
- * proceeds as normal.
- */
-export const beforeNext = <T extends object, K extends keyof T>(
-	target: T,
-	method: K,
-	action: () => Promise<unknown>,
-): void => {
-	const original = target[method] as unknown as (...args: unknown[]) => unknown;
-	let pending = true;
-	target[method] = (async (...args: unknown[]) => {
-		if (pending) {
-			pending = false;
-			await action();
-		}
-		return original.apply(target, args);
-	}) as T[K];
-};

package/src/text/normalizer.test.ts

@@ -1,223 +0,0 @@
-import assert from "node:assert";
-import { describe, it } from "node:test";
-import { createTextNormalizer } from "./normalizer.js";
-
-describe("TextNormalizer", () => {
-	const normalizer = createTextNormalizer();
-
-	describe("detectLanguage", () => {
-		it("detects English text", () => {
-			const text =
-				"The quick brown fox jumps over the lazy dog. This is a sample English text.";
-			assert.strictEqual(normalizer.detectLanguage(text), "en");
-		});
-
-		it("detects German text", () => {
-			const text =
-				"Der schnelle braune Fuchs springt über den faulen Hund. Dies ist ein deutscher Text.";
-			assert.strictEqual(normalizer.detectLanguage(text), "de");
-		});
-
-		it("detects French text", () => {
-			const text =
-				"Le renard brun rapide saute par-dessus le chien paresseux. Ceci est un texte français.";
-			assert.strictEqual(normalizer.detectLanguage(text), "fr");
-		});
-
-		it("detects Spanish text", () => {
-			const text =
-				"El rápido zorro marrón salta sobre el perro perezoso. Este es un texto en español.";
-			assert.strictEqual(normalizer.detectLanguage(text), "es");
-		});
-
-		it("detects Italian text", () => {
-			const text =
-				"La volpe marrone veloce salta sopra il cane pigro. Questo è un testo italiano.";
-			assert.strictEqual(normalizer.detectLanguage(text), "it");
-		});
-
-		it("detects Portuguese text", () => {
-			const text =
-				"A rápida raposa marrom salta sobre o cão preguiçoso. Este é um texto em português.";
-			assert.strictEqual(normalizer.detectLanguage(text), "pt");
-		});
-
-		it("detects Dutch text", () => {
-			const text =
-				"De snelle bruine vos springt over de luie hond. Dit is een Nederlandse tekst.";
-			assert.strictEqual(normalizer.detectLanguage(text), "nl");
-		});
-
-		it("defaults to English for short text", () => {
-			const text = "Hello";
-			assert.strictEqual(normalizer.detectLanguage(text), "en");
-		});
-
-		it("defaults to English for empty text", () => {
-			assert.strictEqual(normalizer.detectLanguage(""), "en");
-		});
-	});
-
-	describe("tokenize", () => {
-		it("tokenizes text into words", () => {
-			const text = "Hello, World! How are you?";
-			assert.deepStrictEqual(normalizer.tokenize(text), [
-				"hello",
-				"world",
-				"how",
-				"are",
-				"you",
-			]);
-		});
-
-		it("handles Unicode characters", () => {
-			const text = "Réunion demain café";
-			assert.deepStrictEqual(normalizer.tokenize(text), [
-				"réunion",
-				"demain",
-				"café",
-			]);
-		});
-
-		it("handles German umlauts", () => {
-			const text = "Besprechung über München";
-			assert.deepStrictEqual(normalizer.tokenize(text), [
-				"besprechung",
-				"über",
-				"münchen",
-			]);
-		});
-
-		it("normalizes whitespace", () => {
-			const text = "  multiple   spaces\n\ttabs  ";
-			assert.deepStrictEqual(normalizer.tokenize(text), [
-				"multiple",
-				"spaces",
-				"tabs",
-			]);
-		});
-	});
-
-	describe("stem", () => {
-		it("stems English words", () => {
-			assert.strictEqual(normalizer.stem("running", "en"), "run");
-			assert.strictEqual(normalizer.stem("jumps", "en"), "jump");
-			assert.strictEqual(normalizer.stem("connected", "en"), "connect");
-		});
-
-		it("stems German words", () => {
-			// German Porter stemmer has different rules than English
-			// Just verify it returns a string and doesn't throw
-			const result1 = normalizer.stem("verbindung", "de");
-			const result2 = normalizer.stem("laufend", "de");
-			assert.ok(typeof result1 === "string");
-			assert.ok(typeof result2 === "string");
-			// The stemmer should at least return something
-			assert.ok(result1.length > 0);
-			assert.ok(result2.length > 0);
-		});
-
-		it("stems French words", () => {
-			assert.strictEqual(normalizer.stem("réunion", "fr"), "réunion");
-			assert.strictEqual(normalizer.stem("connexion", "fr"), "connexion");
-		});
-
-		it("defaults to English stemmer for unknown language", () => {
-			assert.strictEqual(normalizer.stem("running"), "run");
-		});
-	});
-
-	describe("removeStopwords", () => {
-		it("removes English stopwords", () => {
-			const words = ["the", "quick", "brown", "fox", "is", "running"];
-			const result = normalizer.removeStopwords(words, "en");
-			assert.ok(!result.includes("the"));
-			assert.ok(!result.includes("is"));
-			assert.ok(result.includes("quick"));
-			assert.ok(result.includes("brown"));
-			assert.ok(result.includes("fox"));
-			assert.ok(result.includes("running"));
-		});
-
-		it("removes German stopwords", () => {
-			const words = ["der", "schnelle", "fuchs", "ist", "braun"];
-			const result = normalizer.removeStopwords(words, "de");
-			assert.ok(!result.includes("der"));
-			assert.ok(!result.includes("ist"));
-			assert.ok(result.includes("schnelle"));
-			assert.ok(result.includes("fuchs"));
-		});
-
-		it("removes French stopwords", () => {
-			const words = ["le", "renard", "est", "rapide"];
-			const result = normalizer.removeStopwords(words, "fr");
-			assert.ok(!result.includes("le"));
-			assert.ok(!result.includes("est"));
-			assert.ok(result.includes("renard"));
-			assert.ok(result.includes("rapide"));
-		});
-
-		it("removes Dutch stopwords", () => {
-			const words = ["de", "snelle", "vos", "is", "bruin"];
-			const result = normalizer.removeStopwords(words, "nl");
-			assert.ok(!result.includes("de"));
-			assert.ok(!result.includes("is"));
-			assert.ok(result.includes("snelle"));
-			assert.ok(result.includes("vos"));
-		});
-	});
-
-	describe("normalize", () => {
-		it("normalizes English text with stemming and stopword removal", () => {
-			const text = "The quick brown fox is running quickly";
-			const result = normalizer.normalize(text, { language: "en" });
-			// Should remove "the" and "is", stem "running" and "quickly"
-			assert.ok(!result.includes("the "));
-			assert.ok(!result.includes(" is "));
-			assert.ok(result.includes("quick"));
-			assert.ok(result.includes("run"));
-		});
-
-		it("normalizes German text", () => {
-			const text = "Der schnelle braune Fuchs ist laufend";
-			const result = normalizer.normalize(text, { language: "de" });
-			assert.ok(!result.includes("der "));
-			assert.ok(!result.includes(" ist "));
-		});
-
-		it("normalizes without stemming when disabled", () => {
-			const text = "running quickly";
-			const result = normalizer.normalize(text, {
-				language: "en",
-				stem: false,
-				removeStopwords: false,
-			});
-			assert.strictEqual(result, "running quickly");
-		});
-
-		it("handles Unicode text", () => {
-			const text = "Réunion à Paris demain";
-			const result = normalizer.normalize(text, { language: "fr" });
-			assert.ok(result.includes("réunion"));
-			assert.ok(result.includes("pari"));
-			assert.ok(result.includes("demain"));
-		});
-
-		it("deduplicates consecutive words", () => {
-			const text = "hello hello world world world";
-			const result = normalizer.normalize(text, {
-				language: "en",
-				stem: false,
-				removeStopwords: false,
-			});
-			assert.strictEqual(result, "hello world");
-		});
-
-		it("defaults to English when language not specified", () => {
-			const text = "The quick brown fox";
-			const result = normalizer.normalize(text);
-			// "the" should be removed as English stopword
-			assert.ok(!result.includes("the "));
-		});
-	});
-});

package/src/text/normalizer.ts

@@ -1,140 +0,0 @@
-/**
- * Multilingual text normalizer for email processing.
- *
- * Uses franc for language detection, natural for stemming,
- * and stopword for stopword removal.
- */
-
-/// <reference path="../types/natural-porter-stemmer.d.ts" />
-import { franc } from "franc";
-// Import each Porter stemmer file directly instead of the `natural` package
-// root or its `stemmers` submodule barrel: the root barrel
-// (`natural/lib/natural/index.js`) loads WordNet and the Brill POS tagger,
-// which pull in `mongoose` and — via its Postgres storage plugin —
-// `pg`/`dotenv` (`dotenv.config()` at module scope on every cold start, see
-// #1244/#1247); the `stemmers` barrel is CJS and unconditionally requires
-// every stemmer it lists, including Japanese (which drags in its own
-// tokenizer) and Indonesian, neither used here. This module only needs the
-// seven Porter stemmers below — see
-// ../types/natural-porter-stemmer.d.ts for their (hand-written, since
-// `natural` ships none for individual files) type declarations.
-import PorterStemmer from "natural/lib/natural/stemmers/porter_stemmer.js";
-import PorterStemmerDe from "natural/lib/natural/stemmers/porter_stemmer_de.js";
-import PorterStemmerEs from "natural/lib/natural/stemmers/porter_stemmer_es.js";
-import PorterStemmerFr from "natural/lib/natural/stemmers/porter_stemmer_fr.js";
-import PorterStemmerIt from "natural/lib/natural/stemmers/porter_stemmer_it.js";
-import PorterStemmerNl from "natural/lib/natural/stemmers/porter_stemmer_nl.js";
-import PorterStemmerPt from "natural/lib/natural/stemmers/porter_stemmer_pt.js";
-import { deu, eng, fra, ita, nld, por, removeStopwords, spa } from "stopword";
-
-export type SupportedLanguage = "en" | "de" | "fr" | "es" | "it" | "nl" | "pt";
-
-export interface NormalizeOptions {
-	language?: SupportedLanguage;
-	stem?: boolean;
-	removeStopwords?: boolean;
-}
-
-export interface TextNormalizer {
-	detectLanguage(text: string): SupportedLanguage;
-	normalize(text: string, options?: NormalizeOptions): string;
-	tokenize(text: string): string[];
-	stem(word: string, language?: SupportedLanguage): string;
-	removeStopwords(words: string[], language?: SupportedLanguage): string[];
-}
-
-// ISO 639-3 (franc) → ISO 639-1 (our SupportedLanguage)
-const iso3ToIso1: Record<string, SupportedLanguage> = {
-	eng: "en",
-	deu: "de",
-	fra: "fr",
-	nld: "nl",
-	spa: "es",
-	ita: "it",
-	por: "pt",
-};
-
-// Restrict franc to only detect languages we can stem
-const supportedIso3 = Object.keys(iso3ToIso1);
-
-const stemmers: Record<SupportedLanguage, { stem: (word: string) => string }> =
-	{
-		en: PorterStemmer,
-		de: PorterStemmerDe,
-		fr: PorterStemmerFr,
-		es: PorterStemmerEs,
-		it: PorterStemmerIt,
-		nl: PorterStemmerNl,
-		pt: PorterStemmerPt,
-	};
-
-const stopwordLists: Record<SupportedLanguage, string[]> = {
-	en: eng,
-	de: deu,
-	fr: fra,
-	es: spa,
-	it: ita,
-	nl: nld,
-	pt: por,
-};
-
-export const createTextNormalizer = (): TextNormalizer => ({
-	detectLanguage: (text) => {
-		// Use franc with trigram analysis, restricted to supported languages
-		const detected = franc(text, { only: supportedIso3, minLength: 10 });
-
-		// Map to our language codes, default to English
-		return iso3ToIso1[detected] ?? "en";
-	},
-
-	normalize: (text, options = {}) => {
-		const {
-			language = "en",
-			stem = true,
-			removeStopwords: removeStop = true,
-		} = options;
-
-		// Normalize unicode and lowercase
-		let normalized = text.normalize("NFKC").toLowerCase();
-
-		// Keep Unicode letters and numbers, remove punctuation
-		normalized = normalized.replace(/[^\p{L}\p{N}\s]/gu, " ");
-
-		// Tokenize
-		let words = normalized.split(/\s+/).filter((w) => w.length > 0);
-
-		// Remove stopwords
-		if (removeStop) {
-			const list = stopwordLists[language] ?? eng;
-			words = removeStopwords(words, list);
-		}
-
-		// Stem
-		if (stem) {
-			const stemmer = stemmers[language] ?? stemmers.en;
-			words = words.map((w) => stemmer.stem(w));
-		}
-
-		// Dedupe consecutive words
-		return words.filter((w, i, arr) => i === 0 || w !== arr[i - 1]).join(" ");
-	},
-
-	tokenize: (text) => {
-		return text
-			.normalize("NFKC")
-			.toLowerCase()
-			.replace(/[^\p{L}\p{N}\s]/gu, " ")
-			.split(/\s+/)
-			.filter((w) => w.length > 0);
-	},
-
-	stem: (word, language = "en") => {
-		const stemmer = stemmers[language] ?? stemmers.en;
-		return stemmer.stem(word.toLowerCase());
-	},
-
-	removeStopwords: (words, language = "en") => {
-		const list = stopwordLists[language] ?? eng;
-		return removeStopwords(words, list);
-	},
-});