@remit/mailbox-service 0.0.111 → 0.0.113

package/src/text/normalizer.ts

@@ -1,140 +0,0 @@
-/**
- * Multilingual text normalizer for email processing.
- *
- * Uses franc for language detection, natural for stemming,
- * and stopword for stopword removal.
- */
-
-/// <reference path="../types/natural-porter-stemmer.d.ts" />
-import { franc } from "franc";
-// Import each Porter stemmer file directly instead of the `natural` package
-// root or its `stemmers` submodule barrel: the root barrel
-// (`natural/lib/natural/index.js`) loads WordNet and the Brill POS tagger,
-// which pull in `mongoose` and — via its Postgres storage plugin —
-// `pg`/`dotenv` (`dotenv.config()` at module scope on every cold start, see
-// #1244/#1247); the `stemmers` barrel is CJS and unconditionally requires
-// every stemmer it lists, including Japanese (which drags in its own
-// tokenizer) and Indonesian, neither used here. This module only needs the
-// seven Porter stemmers below — see
-// ../types/natural-porter-stemmer.d.ts for their (hand-written, since
-// `natural` ships none for individual files) type declarations.
-import PorterStemmer from "natural/lib/natural/stemmers/porter_stemmer.js";
-import PorterStemmerDe from "natural/lib/natural/stemmers/porter_stemmer_de.js";
-import PorterStemmerEs from "natural/lib/natural/stemmers/porter_stemmer_es.js";
-import PorterStemmerFr from "natural/lib/natural/stemmers/porter_stemmer_fr.js";
-import PorterStemmerIt from "natural/lib/natural/stemmers/porter_stemmer_it.js";
-import PorterStemmerNl from "natural/lib/natural/stemmers/porter_stemmer_nl.js";
-import PorterStemmerPt from "natural/lib/natural/stemmers/porter_stemmer_pt.js";
-import { deu, eng, fra, ita, nld, por, removeStopwords, spa } from "stopword";
-
-export type SupportedLanguage = "en" | "de" | "fr" | "es" | "it" | "nl" | "pt";
-
-export interface NormalizeOptions {
-	language?: SupportedLanguage;
-	stem?: boolean;
-	removeStopwords?: boolean;
-}
-
-export interface TextNormalizer {
-	detectLanguage(text: string): SupportedLanguage;
-	normalize(text: string, options?: NormalizeOptions): string;
-	tokenize(text: string): string[];
-	stem(word: string, language?: SupportedLanguage): string;
-	removeStopwords(words: string[], language?: SupportedLanguage): string[];
-}
-
-// ISO 639-3 (franc) → ISO 639-1 (our SupportedLanguage)
-const iso3ToIso1: Record<string, SupportedLanguage> = {
-	eng: "en",
-	deu: "de",
-	fra: "fr",
-	nld: "nl",
-	spa: "es",
-	ita: "it",
-	por: "pt",
-};
-
-// Restrict franc to only detect languages we can stem
-const supportedIso3 = Object.keys(iso3ToIso1);
-
-const stemmers: Record<SupportedLanguage, { stem: (word: string) => string }> =
-	{
-		en: PorterStemmer,
-		de: PorterStemmerDe,
-		fr: PorterStemmerFr,
-		es: PorterStemmerEs,
-		it: PorterStemmerIt,
-		nl: PorterStemmerNl,
-		pt: PorterStemmerPt,
-	};
-
-const stopwordLists: Record<SupportedLanguage, string[]> = {
-	en: eng,
-	de: deu,
-	fr: fra,
-	es: spa,
-	it: ita,
-	nl: nld,
-	pt: por,
-};
-
-export const createTextNormalizer = (): TextNormalizer => ({
-	detectLanguage: (text) => {
-		// Use franc with trigram analysis, restricted to supported languages
-		const detected = franc(text, { only: supportedIso3, minLength: 10 });
-
-		// Map to our language codes, default to English
-		return iso3ToIso1[detected] ?? "en";
-	},
-
-	normalize: (text, options = {}) => {
-		const {
-			language = "en",
-			stem = true,
-			removeStopwords: removeStop = true,
-		} = options;
-
-		// Normalize unicode and lowercase
-		let normalized = text.normalize("NFKC").toLowerCase();
-
-		// Keep Unicode letters and numbers, remove punctuation
-		normalized = normalized.replace(/[^\p{L}\p{N}\s]/gu, " ");
-
-		// Tokenize
-		let words = normalized.split(/\s+/).filter((w) => w.length > 0);
-
-		// Remove stopwords
-		if (removeStop) {
-			const list = stopwordLists[language] ?? eng;
-			words = removeStopwords(words, list);
-		}
-
-		// Stem
-		if (stem) {
-			const stemmer = stemmers[language] ?? stemmers.en;
-			words = words.map((w) => stemmer.stem(w));
-		}
-
-		// Dedupe consecutive words
-		return words.filter((w, i, arr) => i === 0 || w !== arr[i - 1]).join(" ");
-	},
-
-	tokenize: (text) => {
-		return text
-			.normalize("NFKC")
-			.toLowerCase()
-			.replace(/[^\p{L}\p{N}\s]/gu, " ")
-			.split(/\s+/)
-			.filter((w) => w.length > 0);
-	},
-
-	stem: (word, language = "en") => {
-		const stemmer = stemmers[language] ?? stemmers.en;
-		return stemmer.stem(word.toLowerCase());
-	},
-
-	removeStopwords: (words, language = "en") => {
-		const list = stopwordLists[language] ?? eng;
-		return removeStopwords(words, list);
-	},
-});

package/src/thread-message-rows.ts

@@ -1,77 +0,0 @@
-import type { ThreadMessageItem } from "@remit/data-ports";
-
-/**
- * Build the `set` and `composites` payload for the ThreadMessage update on a
- * MESSAGE_DELETE move-to-trash.
- *
- * The CURRENT row state goes in `composites`; the NEW values go in `set`.
- * ElectroDB uses `composites` to run the conditional check on the existing row
- * AND to compute the previous sort-key values needed to recompute the new ones.
- * Passing the NEW values in `composites` makes the conditional check fail with
- * ConditionalCheckFailedException, which ElectroDB wraps as NotFoundError, and
- * the caller silently drops the update. Same root cause as PR #186 fixed for
- * `flag-queue.ts`.
- */
-export type ThreadMessageRowState = Pick<
-	ThreadMessageItem,
-	| "sentDate"
-	| "mailboxId"
-	| "isRead"
-	| "isDeleted"
-	| "hasStars"
-	| "hasAttachment"
->;
-
-const currentComposites = (threadMessage: ThreadMessageRowState) => ({
-	sentDate: threadMessage.sentDate,
-	mailboxId: threadMessage.mailboxId,
-	isRead: threadMessage.isRead,
-	isDeleted: threadMessage.isDeleted,
-	hasStars: threadMessage.hasStars,
-	hasAttachment: threadMessage.hasAttachment,
-});
-
-export const buildThreadMessageTrashUpdate = (
-	threadMessage: ThreadMessageRowState,
-	newUid: number,
-	destinationMailboxId: string,
-) => ({
-	set: {
-		uid: newUid,
-		mailboxId: destinationMailboxId,
-		isDeleted: true,
-	},
-	composites: currentComposites(threadMessage),
-});
-
-/**
- * The inverse payload: put the thread row back where the message still is on
- * the server. The delete was recorded optimistically, so a delete that gives up
- * with the message still at its source has to hand the row back rather than
- * leave it claiming Trash — an invisible `failed` on a row the user cannot see
- * is the shape of the incident this whole change is about.
- */
-export const buildThreadMessageMoveRevert = (
-	threadMessage: ThreadMessageRowState,
-	sourceUid: number,
-	sourceMailboxId: string,
-) => ({
-	set: {
-		uid: sourceUid,
-		mailboxId: sourceMailboxId,
-		isDeleted: false,
-	},
-	composites: currentComposites(threadMessage),
-});
-
-/**
- * Hand a row back after an abandoned expunge. The mail never left Trash, so
- * only the deletion mark reverts — the uid and mailbox on the row are still
- * where the server has it.
- */
-export const buildThreadMessageUndelete = (
-	threadMessage: ThreadMessageRowState,
-) => ({
-	set: { isDeleted: false },
-	composites: currentComposites(threadMessage),
-});

package/src/types/mailparser-augment.d.ts

@@ -1,15 +0,0 @@
-/**
- * `mailparser`'s `simpleParser` populates `partId` on every entry in
- * `parsed.attachments[]` at runtime (set inside `mail-parser.js:860`), but
- * the field is missing from `@types/mailparser`. Format is the IMAP
- * dot-numbered path (`"1"`, `"2.2.2"`, `"3"`) — the same shape mime-walker
- * writes to `BodyPart.partPath`. The body-part mapper uses it as the
- * primary structural pairing signal.
- */
-declare module "mailparser" {
-	interface Attachment {
-		partId?: string;
-	}
-}
-
-export {};

package/src/types/natural-porter-stemmer.d.ts

@@ -1,50 +0,0 @@
-// `natural` ships no per-file type declarations for its stemmer submodules —
-// only the package-root barrel (`natural/lib/natural/index.d.ts`) and the
-// `stemmers` submodule barrel (`natural/lib/natural/stemmers/index.d.ts`) are
-// typed. Both barrels are CommonJS, so importing either unconditionally
-// requires every stemmer they list — including Japanese (`stemmer_ja.js`,
-// which pulls in its own `tokenizer_ja.js`) and Indonesian
-// (`indonesian/stemmer_id.js`) — even when unused. Importing each Porter
-// stemmer file this module actually uses, directly, avoids that; these
-// ambient declarations cover the resulting import paths. Referenced via a
-// triple-slash directive from normalizer.ts so tsc includes it in any
-// program that type-checks normalizer.ts, even a consuming package's (whose
-// own tsconfig `include` doesn't reach into this package's src/types).
-interface NaturalStemmer {
-	stem: (token: string) => string;
-}
-
-declare module "natural/lib/natural/stemmers/porter_stemmer.js" {
-	const stemmer: NaturalStemmer;
-	export default stemmer;
-}
-
-declare module "natural/lib/natural/stemmers/porter_stemmer_de.js" {
-	const stemmer: NaturalStemmer;
-	export default stemmer;
-}
-
-declare module "natural/lib/natural/stemmers/porter_stemmer_es.js" {
-	const stemmer: NaturalStemmer;
-	export default stemmer;
-}
-
-declare module "natural/lib/natural/stemmers/porter_stemmer_fr.js" {
-	const stemmer: NaturalStemmer;
-	export default stemmer;
-}
-
-declare module "natural/lib/natural/stemmers/porter_stemmer_it.js" {
-	const stemmer: NaturalStemmer;
-	export default stemmer;
-}
-
-declare module "natural/lib/natural/stemmers/porter_stemmer_nl.js" {
-	const stemmer: NaturalStemmer;
-	export default stemmer;
-}
-
-declare module "natural/lib/natural/stemmers/porter_stemmer_pt.js" {
-	const stemmer: NaturalStemmer;
-	export default stemmer;
-}

package/src/types/stopword.d.ts

@@ -1,69 +0,0 @@
-declare module "stopword" {
-	export function removeStopwords(
-		words: string[],
-		stopwords?: string[],
-	): string[];
-
-	export const afr: string[];
-	export const ara: string[];
-	export const ben: string[];
-	export const bre: string[];
-	export const bul: string[];
-	export const cat: string[];
-	export const ces: string[];
-	export const dan: string[];
-	export const deu: string[];
-	export const ell: string[];
-	export const eng: string[];
-	export const epo: string[];
-	export const est: string[];
-	export const eus: string[];
-	export const fas: string[];
-	export const fin: string[];
-	export const fra: string[];
-	export const glg: string[];
-	export const guj: string[];
-	export const hau: string[];
-	export const heb: string[];
-	export const hin: string[];
-	export const hrv: string[];
-	export const hun: string[];
-	export const hye: string[];
-	export const ind: string[];
-	export const ita: string[];
-	export const jpn: string[];
-	export const kor: string[];
-	export const kur: string[];
-	export const lat: string[];
-	export const lav: string[];
-	export const lit: string[];
-	export const lgg: string[];
-	export const lggNd: string[];
-	export const mar: string[];
-	export const msa: string[];
-	export const mya: string[];
-	export const nld: string[];
-	export const nob: string[];
-	export const pan: string[];
-	export const pol: string[];
-	export const por: string[];
-	export const porBr: string[];
-	export const ron: string[];
-	export const rus: string[];
-	export const slk: string[];
-	export const slv: string[];
-	export const som: string[];
-	export const sot: string[];
-	export const spa: string[];
-	export const swa: string[];
-	export const swe: string[];
-	export const tgl: string[];
-	export const tha: string[];
-	export const tur: string[];
-	export const ukr: string[];
-	export const urd: string[];
-	export const vie: string[];
-	export const yor: string[];
-	export const zho: string[];
-	export const zul: string[];
-}