@remit/mailbox-service 0.0.1

package/src/text/normalizer.test.ts

@@ -0,0 +1,223 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { createTextNormalizer } from "./normalizer.js";
+
+describe("TextNormalizer", () => {
+	const normalizer = createTextNormalizer();
+
+	describe("detectLanguage", () => {
+		it("detects English text", () => {
+			const text =
+				"The quick brown fox jumps over the lazy dog. This is a sample English text.";
+			assert.strictEqual(normalizer.detectLanguage(text), "en");
+		});
+
+		it("detects German text", () => {
+			const text =
+				"Der schnelle braune Fuchs springt über den faulen Hund. Dies ist ein deutscher Text.";
+			assert.strictEqual(normalizer.detectLanguage(text), "de");
+		});
+
+		it("detects French text", () => {
+			const text =
+				"Le renard brun rapide saute par-dessus le chien paresseux. Ceci est un texte français.";
+			assert.strictEqual(normalizer.detectLanguage(text), "fr");
+		});
+
+		it("detects Spanish text", () => {
+			const text =
+				"El rápido zorro marrón salta sobre el perro perezoso. Este es un texto en español.";
+			assert.strictEqual(normalizer.detectLanguage(text), "es");
+		});
+
+		it("detects Italian text", () => {
+			const text =
+				"La volpe marrone veloce salta sopra il cane pigro. Questo è un testo italiano.";
+			assert.strictEqual(normalizer.detectLanguage(text), "it");
+		});
+
+		it("detects Portuguese text", () => {
+			const text =
+				"A rápida raposa marrom salta sobre o cão preguiçoso. Este é um texto em português.";
+			assert.strictEqual(normalizer.detectLanguage(text), "pt");
+		});
+
+		it("detects Dutch text", () => {
+			const text =
+				"De snelle bruine vos springt over de luie hond. Dit is een Nederlandse tekst.";
+			assert.strictEqual(normalizer.detectLanguage(text), "nl");
+		});
+
+		it("defaults to English for short text", () => {
+			const text = "Hello";
+			assert.strictEqual(normalizer.detectLanguage(text), "en");
+		});
+
+		it("defaults to English for empty text", () => {
+			assert.strictEqual(normalizer.detectLanguage(""), "en");
+		});
+	});
+
+	describe("tokenize", () => {
+		it("tokenizes text into words", () => {
+			const text = "Hello, World! How are you?";
+			assert.deepStrictEqual(normalizer.tokenize(text), [
+				"hello",
+				"world",
+				"how",
+				"are",
+				"you",
+			]);
+		});
+
+		it("handles Unicode characters", () => {
+			const text = "Réunion demain café";
+			assert.deepStrictEqual(normalizer.tokenize(text), [
+				"réunion",
+				"demain",
+				"café",
+			]);
+		});
+
+		it("handles German umlauts", () => {
+			const text = "Besprechung über München";
+			assert.deepStrictEqual(normalizer.tokenize(text), [
+				"besprechung",
+				"über",
+				"münchen",
+			]);
+		});
+
+		it("normalizes whitespace", () => {
+			const text = "  multiple   spaces\n\ttabs  ";
+			assert.deepStrictEqual(normalizer.tokenize(text), [
+				"multiple",
+				"spaces",
+				"tabs",
+			]);
+		});
+	});
+
+	describe("stem", () => {
+		it("stems English words", () => {
+			assert.strictEqual(normalizer.stem("running", "en"), "run");
+			assert.strictEqual(normalizer.stem("jumps", "en"), "jump");
+			assert.strictEqual(normalizer.stem("connected", "en"), "connect");
+		});
+
+		it("stems German words", () => {
+			// German Porter stemmer has different rules than English
+			// Just verify it returns a string and doesn't throw
+			const result1 = normalizer.stem("verbindung", "de");
+			const result2 = normalizer.stem("laufend", "de");
+			assert.ok(typeof result1 === "string");
+			assert.ok(typeof result2 === "string");
+			// The stemmer should at least return something
+			assert.ok(result1.length > 0);
+			assert.ok(result2.length > 0);
+		});
+
+		it("stems French words", () => {
+			assert.strictEqual(normalizer.stem("réunion", "fr"), "réunion");
+			assert.strictEqual(normalizer.stem("connexion", "fr"), "connexion");
+		});
+
+		it("defaults to English stemmer for unknown language", () => {
+			assert.strictEqual(normalizer.stem("running"), "run");
+		});
+	});
+
+	describe("removeStopwords", () => {
+		it("removes English stopwords", () => {
+			const words = ["the", "quick", "brown", "fox", "is", "running"];
+			const result = normalizer.removeStopwords(words, "en");
+			assert.ok(!result.includes("the"));
+			assert.ok(!result.includes("is"));
+			assert.ok(result.includes("quick"));
+			assert.ok(result.includes("brown"));
+			assert.ok(result.includes("fox"));
+			assert.ok(result.includes("running"));
+		});
+
+		it("removes German stopwords", () => {
+			const words = ["der", "schnelle", "fuchs", "ist", "braun"];
+			const result = normalizer.removeStopwords(words, "de");
+			assert.ok(!result.includes("der"));
+			assert.ok(!result.includes("ist"));
+			assert.ok(result.includes("schnelle"));
+			assert.ok(result.includes("fuchs"));
+		});
+
+		it("removes French stopwords", () => {
+			const words = ["le", "renard", "est", "rapide"];
+			const result = normalizer.removeStopwords(words, "fr");
+			assert.ok(!result.includes("le"));
+			assert.ok(!result.includes("est"));
+			assert.ok(result.includes("renard"));
+			assert.ok(result.includes("rapide"));
+		});
+
+		it("removes Dutch stopwords", () => {
+			const words = ["de", "snelle", "vos", "is", "bruin"];
+			const result = normalizer.removeStopwords(words, "nl");
+			assert.ok(!result.includes("de"));
+			assert.ok(!result.includes("is"));
+			assert.ok(result.includes("snelle"));
+			assert.ok(result.includes("vos"));
+		});
+	});
+
+	describe("normalize", () => {
+		it("normalizes English text with stemming and stopword removal", () => {
+			const text = "The quick brown fox is running quickly";
+			const result = normalizer.normalize(text, { language: "en" });
+			// Should remove "the" and "is", stem "running" and "quickly"
+			assert.ok(!result.includes("the "));
+			assert.ok(!result.includes(" is "));
+			assert.ok(result.includes("quick"));
+			assert.ok(result.includes("run"));
+		});
+
+		it("normalizes German text", () => {
+			const text = "Der schnelle braune Fuchs ist laufend";
+			const result = normalizer.normalize(text, { language: "de" });
+			assert.ok(!result.includes("der "));
+			assert.ok(!result.includes(" ist "));
+		});
+
+		it("normalizes without stemming when disabled", () => {
+			const text = "running quickly";
+			const result = normalizer.normalize(text, {
+				language: "en",
+				stem: false,
+				removeStopwords: false,
+			});
+			assert.strictEqual(result, "running quickly");
+		});
+
+		it("handles Unicode text", () => {
+			const text = "Réunion à Paris demain";
+			const result = normalizer.normalize(text, { language: "fr" });
+			assert.ok(result.includes("réunion"));
+			assert.ok(result.includes("pari"));
+			assert.ok(result.includes("demain"));
+		});
+
+		it("deduplicates consecutive words", () => {
+			const text = "hello hello world world world";
+			const result = normalizer.normalize(text, {
+				language: "en",
+				stem: false,
+				removeStopwords: false,
+			});
+			assert.strictEqual(result, "hello world");
+		});
+
+		it("defaults to English when language not specified", () => {
+			const text = "The quick brown fox";
+			const result = normalizer.normalize(text);
+			// "the" should be removed as English stopword
+			assert.ok(!result.includes("the "));
+		});
+	});
+});

package/src/text/normalizer.ts

@@ -0,0 +1,140 @@
+/**
+ * Multilingual text normalizer for email processing.
+ *
+ * Uses franc for language detection, natural for stemming,
+ * and stopword for stopword removal.
+ */
+
+/// <reference path="../types/natural-porter-stemmer.d.ts" />
+import { franc } from "franc";
+// Import each Porter stemmer file directly instead of the `natural` package
+// root or its `stemmers` submodule barrel: the root barrel
+// (`natural/lib/natural/index.js`) loads WordNet and the Brill POS tagger,
+// which pull in `mongoose` and — via its Postgres storage plugin —
+// `pg`/`dotenv` (`dotenv.config()` at module scope on every cold start, see
+// #1244/#1247); the `stemmers` barrel is CJS and unconditionally requires
+// every stemmer it lists, including Japanese (which drags in its own
+// tokenizer) and Indonesian, neither used here. This module only needs the
+// seven Porter stemmers below — see
+// ../types/natural-porter-stemmer.d.ts for their (hand-written, since
+// `natural` ships none for individual files) type declarations.
+import PorterStemmer from "natural/lib/natural/stemmers/porter_stemmer.js";
+import PorterStemmerDe from "natural/lib/natural/stemmers/porter_stemmer_de.js";
+import PorterStemmerEs from "natural/lib/natural/stemmers/porter_stemmer_es.js";
+import PorterStemmerFr from "natural/lib/natural/stemmers/porter_stemmer_fr.js";
+import PorterStemmerIt from "natural/lib/natural/stemmers/porter_stemmer_it.js";
+import PorterStemmerNl from "natural/lib/natural/stemmers/porter_stemmer_nl.js";
+import PorterStemmerPt from "natural/lib/natural/stemmers/porter_stemmer_pt.js";
+import { deu, eng, fra, ita, nld, por, removeStopwords, spa } from "stopword";
+
+export type SupportedLanguage = "en" | "de" | "fr" | "es" | "it" | "nl" | "pt";
+
+export interface NormalizeOptions {
+	language?: SupportedLanguage;
+	stem?: boolean;
+	removeStopwords?: boolean;
+}
+
+export interface TextNormalizer {
+	detectLanguage(text: string): SupportedLanguage;
+	normalize(text: string, options?: NormalizeOptions): string;
+	tokenize(text: string): string[];
+	stem(word: string, language?: SupportedLanguage): string;
+	removeStopwords(words: string[], language?: SupportedLanguage): string[];
+}
+
+// ISO 639-3 (franc) → ISO 639-1 (our SupportedLanguage)
+const iso3ToIso1: Record<string, SupportedLanguage> = {
+	eng: "en",
+	deu: "de",
+	fra: "fr",
+	nld: "nl",
+	spa: "es",
+	ita: "it",
+	por: "pt",
+};
+
+// Restrict franc to only detect languages we can stem
+const supportedIso3 = Object.keys(iso3ToIso1);
+
+const stemmers: Record<SupportedLanguage, { stem: (word: string) => string }> =
+	{
+		en: PorterStemmer,
+		de: PorterStemmerDe,
+		fr: PorterStemmerFr,
+		es: PorterStemmerEs,
+		it: PorterStemmerIt,
+		nl: PorterStemmerNl,
+		pt: PorterStemmerPt,
+	};
+
+const stopwordLists: Record<SupportedLanguage, string[]> = {
+	en: eng,
+	de: deu,
+	fr: fra,
+	es: spa,
+	it: ita,
+	nl: nld,
+	pt: por,
+};
+
+export const createTextNormalizer = (): TextNormalizer => ({
+	detectLanguage: (text) => {
+		// Use franc with trigram analysis, restricted to supported languages
+		const detected = franc(text, { only: supportedIso3, minLength: 10 });
+
+		// Map to our language codes, default to English
+		return iso3ToIso1[detected] ?? "en";
+	},
+
+	normalize: (text, options = {}) => {
+		const {
+			language = "en",
+			stem = true,
+			removeStopwords: removeStop = true,
+		} = options;
+
+		// Normalize unicode and lowercase
+		let normalized = text.normalize("NFKC").toLowerCase();
+
+		// Keep Unicode letters and numbers, remove punctuation
+		normalized = normalized.replace(/[^\p{L}\p{N}\s]/gu, " ");
+
+		// Tokenize
+		let words = normalized.split(/\s+/).filter((w) => w.length > 0);
+
+		// Remove stopwords
+		if (removeStop) {
+			const list = stopwordLists[language] ?? eng;
+			words = removeStopwords(words, list);
+		}
+
+		// Stem
+		if (stem) {
+			const stemmer = stemmers[language] ?? stemmers.en;
+			words = words.map((w) => stemmer.stem(w));
+		}
+
+		// Dedupe consecutive words
+		return words.filter((w, i, arr) => i === 0 || w !== arr[i - 1]).join(" ");
+	},
+
+	tokenize: (text) => {
+		return text
+			.normalize("NFKC")
+			.toLowerCase()
+			.replace(/[^\p{L}\p{N}\s]/gu, " ")
+			.split(/\s+/)
+			.filter((w) => w.length > 0);
+	},
+
+	stem: (word, language = "en") => {
+		const stemmer = stemmers[language] ?? stemmers.en;
+		return stemmer.stem(word.toLowerCase());
+	},
+
+	removeStopwords: (words, language = "en") => {
+		const list = stopwordLists[language] ?? eng;
+		return removeStopwords(words, list);
+	},
+});

package/src/types/mailparser-augment.d.ts

@@ -0,0 +1,15 @@
+/**
+ * `mailparser`'s `simpleParser` populates `partId` on every entry in
+ * `parsed.attachments[]` at runtime (set inside `mail-parser.js:860`), but
+ * the field is missing from `@types/mailparser`. Format is the IMAP
+ * dot-numbered path (`"1"`, `"2.2.2"`, `"3"`) — the same shape mime-walker
+ * writes to `BodyPart.partPath`. The body-part mapper uses it as the
+ * primary structural pairing signal.
+ */
+declare module "mailparser" {
+	interface Attachment {
+		partId?: string;
+	}
+}
+
+export {};

package/src/types/natural-porter-stemmer.d.ts

@@ -0,0 +1,50 @@
+// `natural` ships no per-file type declarations for its stemmer submodules —
+// only the package-root barrel (`natural/lib/natural/index.d.ts`) and the
+// `stemmers` submodule barrel (`natural/lib/natural/stemmers/index.d.ts`) are
+// typed. Both barrels are CommonJS, so importing either unconditionally
+// requires every stemmer they list — including Japanese (`stemmer_ja.js`,
+// which pulls in its own `tokenizer_ja.js`) and Indonesian
+// (`indonesian/stemmer_id.js`) — even when unused. Importing each Porter
+// stemmer file this module actually uses, directly, avoids that; these
+// ambient declarations cover the resulting import paths. Referenced via a
+// triple-slash directive from normalizer.ts so tsc includes it in any
+// program that type-checks normalizer.ts, even a consuming package's (whose
+// own tsconfig `include` doesn't reach into this package's src/types).
+interface NaturalStemmer {
+	stem: (token: string) => string;
+}
+
+declare module "natural/lib/natural/stemmers/porter_stemmer.js" {
+	const stemmer: NaturalStemmer;
+	export default stemmer;
+}
+
+declare module "natural/lib/natural/stemmers/porter_stemmer_de.js" {
+	const stemmer: NaturalStemmer;
+	export default stemmer;
+}
+
+declare module "natural/lib/natural/stemmers/porter_stemmer_es.js" {
+	const stemmer: NaturalStemmer;
+	export default stemmer;
+}
+
+declare module "natural/lib/natural/stemmers/porter_stemmer_fr.js" {
+	const stemmer: NaturalStemmer;
+	export default stemmer;
+}
+
+declare module "natural/lib/natural/stemmers/porter_stemmer_it.js" {
+	const stemmer: NaturalStemmer;
+	export default stemmer;
+}
+
+declare module "natural/lib/natural/stemmers/porter_stemmer_nl.js" {
+	const stemmer: NaturalStemmer;
+	export default stemmer;
+}
+
+declare module "natural/lib/natural/stemmers/porter_stemmer_pt.js" {
+	const stemmer: NaturalStemmer;
+	export default stemmer;
+}

package/src/types/stopword.d.ts

@@ -0,0 +1,69 @@
+declare module "stopword" {
+	export function removeStopwords(
+		words: string[],
+		stopwords?: string[],
+	): string[];
+
+	export const afr: string[];
+	export const ara: string[];
+	export const ben: string[];
+	export const bre: string[];
+	export const bul: string[];
+	export const cat: string[];
+	export const ces: string[];
+	export const dan: string[];
+	export const deu: string[];
+	export const ell: string[];
+	export const eng: string[];
+	export const epo: string[];
+	export const est: string[];
+	export const eus: string[];
+	export const fas: string[];
+	export const fin: string[];
+	export const fra: string[];
+	export const glg: string[];
+	export const guj: string[];
+	export const hau: string[];
+	export const heb: string[];
+	export const hin: string[];
+	export const hrv: string[];
+	export const hun: string[];
+	export const hye: string[];
+	export const ind: string[];
+	export const ita: string[];
+	export const jpn: string[];
+	export const kor: string[];
+	export const kur: string[];
+	export const lat: string[];
+	export const lav: string[];
+	export const lit: string[];
+	export const lgg: string[];
+	export const lggNd: string[];
+	export const mar: string[];
+	export const msa: string[];
+	export const mya: string[];
+	export const nld: string[];
+	export const nob: string[];
+	export const pan: string[];
+	export const pol: string[];
+	export const por: string[];
+	export const porBr: string[];
+	export const ron: string[];
+	export const rus: string[];
+	export const slk: string[];
+	export const slv: string[];
+	export const som: string[];
+	export const sot: string[];
+	export const spa: string[];
+	export const swa: string[];
+	export const swe: string[];
+	export const tgl: string[];
+	export const tha: string[];
+	export const tur: string[];
+	export const ukr: string[];
+	export const urd: string[];
+	export const vie: string[];
+	export const yor: string[];
+	export const zho: string[];
+	export const zul: string[];
+}