@stll/anonymize-wasm 1.4.3 → 1.4.4

package/dist/common-words-en.mjs

@@ -412,7 +412,6 @@ var words = [
 	"ant",
 	"antarctica",
 	"antenna",
-	"anthony",
 	"anthropology",
 	"anti",
 	"antibodies",

package/dist/legal-form-leading-clauses.mjs

@@ -6,7 +6,12 @@ var en = {
 		"by and between",
 		"is between"
 	],
-	"directPrefixes": ["by"]
+	"directPrefixes": [
+		"by",
+		"among",
+		"amongst",
+		"between"
+	]
 };
 var legal_form_leading_clauses_default = {
 	_comment,

package/dist/person-stopwords.mjs

@@ -17,6 +17,7 @@ var person_stopwords_default = {
 		"basic",
 		"belgian",
 		"borrower",
+		"branch",
 		"british",
 		"bulgarian",
 		"buyer",
@@ -35,6 +36,8 @@ var person_stopwords_default = {
 		"default",
 		"delaware",
 		"delivery",
+		"department",
+		"division",
 		"dry",
 		"dutch",
 		"english",
@@ -50,6 +53,7 @@ var person_stopwords_default = {
 		"germany",
 		"greek",
 		"guarantor",
+		"headquarters",
 		"hungarian",
 		"indiana",
 		"interest",
@@ -80,6 +84,7 @@ var person_stopwords_default = {
 		"notice",
 		"november",
 		"october",
+		"office",
 		"ohio",
 		"opportunity",
 		"owner",
@@ -102,6 +107,7 @@ var person_stopwords_default = {
 		"section",
 		"seller",
 		"september",
+		"subsidiary",
 		"slovak",
 		"slovenian",
 		"spanish",

package/dist/wasm.mjs

@@ -231,12 +231,22 @@ const LEGAL_SUFFIXES = [...[
 	"OHG",
 	"Ltd.",
 	"Ltd",
+	"LTD.",
+	"LTD",
 	"LLC",
 	"LLP",
 	"Inc.",
+	"INC.",
+	"Inc",
+	"INC",
 	"Corp.",
+	"CORP.",
+	"Corp",
+	"CORP",
 	"Corporation",
+	"CORPORATION",
 	"Co.",
+	"CO.",
 	"LP",
 	"L.P.",
 	"PLC",
@@ -246,6 +256,8 @@ const LEGAL_SUFFIXES = [...[
 	"B.V.",
 	"Pty Ltd.",
 	"Pty Ltd",
+	"PTY LTD.",
+	"PTY LTD",
 	"S.A.",
 	"SA",
 	"SAS",
@@ -1242,8 +1254,6 @@ const splitEmbeddedLegalFormList = (entityStart, entityText) => {
 			const boundary = LEGAL_LIST_BOUNDARY_RE.exec(afterSuffix);
 			if (boundary === null) continue;
 			const nextStart = suffixEnd + boundary[0].length;
-			const remainder = entityText.slice(nextStart);
-			if (!getAllLegalSuffixesSync().some((form) => remainder.endsWith(form))) continue;
 			cuts.push(nextStart);
 		}
 	}
@@ -1259,6 +1269,7 @@ const splitEmbeddedLegalFormList = (entityStart, entityText) => {
 		if (start === void 0) continue;
 		const segmentText = entityText.slice(start, end).replace(/[,\s;]+$/u, "");
 		if (segmentText.length === 0) continue;
+		if (!getAllLegalSuffixesSync().some((form) => segmentText.endsWith(form))) continue;
 		segments.push({
 			entityStart: entityStart + start,
 			entityText: segmentText
@@ -1442,10 +1453,18 @@ const trimLeadingClause = (text) => {
 		for (const match of text.matchAll(phraseRe)) cut = Math.max(cut, match.index + match[0].length);
 	}
 	const directPrefixAlternation = trims.directPrefixes.map(escapeForRegex).join("|");
+	const COMMA_GATED_DIRECT_PREFIXES = new Set([
+		"among",
+		"amongst",
+		"between"
+	]);
 	if (directPrefixAlternation.length > 0) {
 		const directPrefixRe = new RegExp(`\\b(?:${directPrefixAlternation})${HSPACE}+(?=\\p{Lu})`, "giu");
 		for (const match of text.matchAll(directPrefixRe)) {
-			const words = text.slice(0, match.index).match(/\p{L}[\p{L}\p{M}]*/gu) ?? [];
+			const matchedPrefix = match[0].trim().toLowerCase();
+			const before = text.slice(0, match.index);
+			if (COMMA_GATED_DIRECT_PREFIXES.has(matchedPrefix) && !/,\s*$/u.test(before)) continue;
+			const words = before.match(/\p{L}[\p{L}\p{M}]*/gu) ?? [];
 			if (words.length >= 3 && words.some((word) => /\p{Ll}/u.test(word))) cut = Math.max(cut, match.index + match[0].length);
 		}
 	}
@@ -1572,11 +1591,6 @@ const processLegalFormMatches = (allMatches, sliceStart, sliceEnd, fullText) =>
 			let { prefixEnd, prefixPart } = getPrefixInfo(entityText);
 			let isAllCapsMatch = prefixPart.length > 2 && prefixPart === prefixPart.toUpperCase();
 			if (isAllCapsMatch && fullText) {
-				const lineStart = fullText.lastIndexOf("\n", entityStart);
-				const lineEnd = fullText.indexOf("\n", entityStart + entityText.length);
-				const lineLetters = fullText.slice(lineStart + 1, lineEnd === -1 ? fullText.length : lineEnd).replace(/[^a-zA-ZÀ-ž]/g, "");
-				const upperCount = [...lineLetters].filter((c) => c === c.toUpperCase()).length;
-				if (lineLetters.length > 5 && upperCount / lineLetters.length >= .95) continue;
 				if ((prefixPart.length > 0 ? entityText.slice(0, prefixEnd > 0 ? prefixEnd : entityText.length).trim().split(/\s+/).length : 0) > 3) {
 					entityStart = match.start;
 					entityText = text;
@@ -2145,7 +2159,31 @@ const segmentWords = (fullText) => {
 };
 /** NAME or SURNAME — both represent corpus-matched tokens */
 const isCorpusMatch = (type) => type === TOKEN_TYPE.NAME || type === TOKEN_TYPE.SURNAME;
-const classifyToken = (word, corpus) => {
+const ALL_CAPS_NAME_LINE_RATIO = .9;
+const ALL_CAPS_NAME_LINE_MIN_LETTERS = 3;
+const ALL_CAPS_NAME_LINE_MAX_TOKENS = 6;
+const isAllCapsLineNameShaped = (fullText, start) => {
+	const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
+	const lineEndIdx = fullText.indexOf("\n", start);
+	const line = fullText.slice(lineStart, lineEndIdx === -1 ? fullText.length : lineEndIdx);
+	if (/\d/.test(line)) return false;
+	const tokens = line.match(/\p{L}[\p{L}\p{M}'\-]*/gu) ?? [];
+	return tokens.length > 0 && tokens.length <= ALL_CAPS_NAME_LINE_MAX_TOKENS;
+};
+const isAllCapsContextLine = (fullText, start) => {
+	const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
+	const lineEndIdx = fullText.indexOf("\n", start);
+	const line = fullText.slice(lineStart, lineEndIdx === -1 ? fullText.length : lineEndIdx);
+	let letters = 0;
+	let upper = 0;
+	for (const ch of line) if (/\p{L}/u.test(ch)) {
+		letters += 1;
+		if (ch === ch.toUpperCase() && ch !== ch.toLowerCase()) upper += 1;
+	}
+	if (letters < ALL_CAPS_NAME_LINE_MIN_LETTERS) return false;
+	return upper / letters >= ALL_CAPS_NAME_LINE_RATIO;
+};
+const classifyToken = (word, corpus, fullText) => {
 	const { text, start, end } = word;
 	const lower = text.toLowerCase();
 	const stripped = text.endsWith(".") ? text.slice(0, -1).toLowerCase() : lower;
@@ -2161,6 +2199,26 @@ const classifyToken = (word, corpus) => {
 		start,
 		end
 	};
+	if (text.length === 1 && UPPER_START_RE.test(text) && fullText[end] === ".") {
+		const lineStart = fullText.lastIndexOf("\n", start - 1) + 1;
+		const before = fullText.slice(lineStart, start).trimEnd();
+		const lastWord = /\p{L}[\p{L}\p{M}'\-]*$/u.exec(before)?.[0];
+		if (lastWord) {
+			const lookup = (token) => isFirstNameToken(token, corpus) || isFirstNameToken((token[0] ?? "") + token.slice(1).toLowerCase(), corpus);
+			if (lookup(lastWord)) return {
+				text,
+				type: TOKEN_TYPE.ABBREVIATION,
+				start,
+				end
+			};
+		}
+		return {
+			text,
+			type: TOKEN_TYPE.OTHER,
+			start,
+			end
+		};
+	}
 	if (corpus.excludedWords.has(lower)) return {
 		text,
 		type: TOKEN_TYPE.OTHER,
@@ -2173,12 +2231,29 @@ const classifyToken = (word, corpus) => {
 		start,
 		end
 	};
-	if (text.length > 3 && ALL_UPPER_RE.test(text)) return {
-		text,
-		type: TOKEN_TYPE.OTHER,
-		start,
-		end
-	};
+	if (text.length >= 3 && ALL_UPPER_RE.test(text)) {
+		if (isAllCapsContextLine(fullText, start) && isAllCapsLineNameShaped(fullText, start)) {
+			const titleCased = (text[0] ?? "") + text.slice(1).toLowerCase();
+			if (isFirstNameToken(titleCased, corpus)) return {
+				text,
+				type: TOKEN_TYPE.NAME,
+				start,
+				end
+			};
+			if (isSurnameToken(titleCased, corpus)) return {
+				text,
+				type: TOKEN_TYPE.SURNAME,
+				start,
+				end
+			};
+		}
+		return {
+			text,
+			type: TOKEN_TYPE.OTHER,
+			start,
+			end
+		};
+	}
 	if (!UPPER_START_RE.test(text)) return {
 		text,
 		type: TOKEN_TYPE.OTHER,
@@ -2223,7 +2298,7 @@ const classifyToken = (word, corpus) => {
 const detectNameCorpus = (fullText, ctx = defaultContext) => {
 	const corpus = getCorpus(ctx);
 	if (!corpus) return [];
-	const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus));
+	const tokens = segmentWords(fullText).map((w) => classifyToken(w, corpus, fullText));
 	const entities = [];
 	const consumed = /* @__PURE__ */ new Set();
 	for (let i = 0; i < tokens.length; i++) {
@@ -2261,6 +2336,8 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
 		else if (hasAbbreviation && hasCorpusName) score = .7;
 		else if (hasFirstName && chain.length === 1) {
 			if (isSentenceStart(fullText, token.start)) continue;
+			const first = chain[0];
+			if (first && ALL_UPPER_RE.test(first.text) && first.text.length >= 3) continue;
 			score = .5;
 		} else if (!hasFirstName && chain.length === 1 && chain[0]?.type === TOKEN_TYPE.SURNAME) continue;
 		else if (hasTitle && chain.length === 1) continue;
@@ -2287,6 +2364,115 @@ const detectNameCorpus = (fullText, ctx = defaultContext) => {
 	return entities;
 };
 //#endregion
+//#region src/detectors/signatures.ts
+const SLASH_S_RE = /\/s\/[ \t]*/g;
+const LABELLED_NAME_RE = /\b(?:By|Name)[ \t]*:[ \t]*(?:\/s\/[ \t]+)?([^\n]*?)(?=\s{3,}|[\t]|\n|$)/gi;
+const WITNESS_ANCHOR_RE = /\bIN WITNESS WHEREOF\b[^\n]*\n/gi;
+const NAME_PARTICLE = "(?:de|del|della|der|den|di|du|el|la|le|van|von|y|zu|af|ben|bin|al|d'|d’)";
+const CAP_TOKEN = "\\p{Lu}[\\p{L}\\p{M}.'\\-]{0,30}";
+const NAME_SHAPE_RE = new RegExp(`^${CAP_TOKEN}(?:[ \\t]+(?:${NAME_PARTICLE}|${CAP_TOKEN})){1,4}$`, "u");
+const MAX_NAME_LEN = 60;
+const POST_NOMINAL_SUFFIX_RE = /,\s*(?:Jr|Sr|II|III|IV|V|Esq|Esquire|M\.?D|Ph\.?D|J\.?D|LL\.?M|MBA|CPA|PE|RN|DDS|DVM|DO|MD|CFA|CFP)\.?\s*$/i;
+const COLUMN_SEPARATOR_RE = /\s{3,}|\t+/;
+const ORG_SUFFIX_RE = /\b(?:INC\.?|LLC|LLP|LP|CORP\.?|CORPORATION|LTD\.?|GMBH|AG|SE|KG|OHG|SA|SAS|SARL|S\.A\.?|S\.P\.A\.?|PLC|N\.A\.?|N\.V\.?|B\.V\.?|PTY\s+LTD\.?|CO\.|S\.R\.O\.?|A\.S\.?|Z\.S\.?|S\.\s*P\.?|LTDA\.?|EIRELI|EPP|S\/A)\b/i;
+const IMAGE_STUB_RE = /^(?:\[?img|\[image|\[logo|\(logo\))/i;
+const normaliseCandidate = (text) => {
+	let candidate = text.trim();
+	candidate = candidate.replace(POST_NOMINAL_SUFFIX_RE, "").trim();
+	return candidate.split(COLUMN_SEPARATOR_RE)[0]?.trim() ?? candidate;
+};
+const isNameShape = (text) => {
+	if (text.length === 0 || text.length > MAX_NAME_LEN) return false;
+	if (text.length < 3) return false;
+	if (!NAME_SHAPE_RE.test(text)) return false;
+	return true;
+};
+const findLineEnd = (text, pos) => {
+	const idx = text.indexOf("\n", pos);
+	return idx === -1 ? text.length : idx;
+};
+const tryEmit = (results, fullText, start, end, score) => {
+	const raw = fullText.slice(start, end);
+	if (ORG_SUFFIX_RE.test(raw)) return false;
+	const candidate = normaliseCandidate(raw);
+	if (!isNameShape(candidate)) return false;
+	const offset = raw.indexOf(candidate);
+	if (offset < 0) return false;
+	const absStart = start + offset;
+	results.push({
+		start: absStart,
+		end: absStart + candidate.length,
+		label: "person",
+		text: candidate,
+		score,
+		source: DETECTION_SOURCES.TRIGGER
+	});
+	return true;
+};
+const tryEmitForwardLines = (results, fullText, fromPos, maxLines, score) => {
+	let pos = fromPos;
+	for (let i = 0; i < maxLines; i++) {
+		if (pos >= fullText.length) return false;
+		const lineEnd = findLineEnd(fullText, pos);
+		const line = fullText.slice(pos, lineEnd).trim();
+		if (line.length > 0 && !IMAGE_STUB_RE.test(line)) {
+			if (tryEmit(results, fullText, pos, lineEnd, score)) return true;
+		}
+		pos = lineEnd + 1;
+	}
+	return false;
+};
+const findPrevLine = (fullText, pos) => {
+	let cursor = pos - 1;
+	while (cursor >= 0 && fullText.charAt(cursor) !== "\n") cursor--;
+	while (cursor >= 0) {
+		let lineStart = cursor;
+		while (lineStart > 0 && fullText.charAt(lineStart - 1) !== "\n") lineStart -= 1;
+		const lineEnd = cursor;
+		const line = fullText.slice(lineStart, lineEnd).trim();
+		if (line.length > 0 && !IMAGE_STUB_RE.test(line)) return {
+			lineStart,
+			lineEnd
+		};
+		cursor = lineStart - 1;
+	}
+	return null;
+};
+const detectSignatures = (fullText, _ctx = defaultContext) => {
+	const results = [];
+	SLASH_S_RE.lastIndex = 0;
+	for (let m = SLASH_S_RE.exec(fullText); m !== null; m = SLASH_S_RE.exec(fullText)) {
+		const afterMark = m.index + m[0].length;
+		const lineEnd = findLineEnd(fullText, afterMark);
+		if (fullText.slice(afterMark, lineEnd).trim().length > 0) {
+			const firstCell = fullText.slice(afterMark, lineEnd).split(COLUMN_SEPARATOR_RE)[0] ?? "";
+			if (firstCell.trim().length > 0) tryEmit(results, fullText, afterMark, afterMark + firstCell.length, .95);
+		} else tryEmitForwardLines(results, fullText, lineEnd + 1, 4, .9);
+		const prev = findPrevLine(fullText, m.index);
+		if (prev) tryEmit(results, fullText, prev.lineStart, prev.lineEnd, .85);
+	}
+	LABELLED_NAME_RE.lastIndex = 0;
+	for (let m = LABELLED_NAME_RE.exec(fullText); m !== null; m = LABELLED_NAME_RE.exec(fullText)) {
+		const value = m[1];
+		if (value === void 0) continue;
+		const valueStart = m.index + m[0].length - value.length;
+		const valueEnd = valueStart + value.length;
+		if (value.trim().length === 0) {
+			tryEmitForwardLines(results, fullText, valueEnd + 1, 3, .9);
+			continue;
+		}
+		tryEmit(results, fullText, valueStart, valueEnd, .95);
+	}
+	WITNESS_ANCHOR_RE.lastIndex = 0;
+	for (let m = WITNESS_ANCHOR_RE.exec(fullText); m !== null; m = WITNESS_ANCHOR_RE.exec(fullText)) {
+		const search = fullText.slice(m.index, m.index + 600);
+		const sentenceEnd = /[.:;]\s*\n|\n\s*\n/.exec(search);
+		if (!sentenceEnd) continue;
+		tryEmitForwardLines(results, fullText, m.index + sentenceEnd.index + sentenceEnd[0].length, 6, .85);
+	}
+	return results;
+};
+//#endregion
 //#region src/config/titles.ts
 /**
 * Academic and professional title prefixes.
@@ -4106,6 +4292,47 @@ const MAX_ENTITY_LENGTH = {
 	organization: 80,
 	person: 60
 };
+const MAX_ENTITY_WORDS = { organization: 8 };
+const OPEN_ENDED_SOURCES = new Set(["trigger", "coreference"]);
+const WORD_TOKEN_RE = /\p{L}[\p{L}\p{M}\p{N}'’\-./]*/gu;
+const countWordTokens = (text) => {
+	let count = 0;
+	WORD_TOKEN_RE.lastIndex = 0;
+	while (WORD_TOKEN_RE.exec(text) !== null) count += 1;
+	return count;
+};
+const ALL_CAPS_LINE_LETTER_THRESHOLD = 5;
+const ALL_CAPS_LINE_RATIO = .95;
+const ALL_CAPS_LINE_PROSE_EXTRA_LETTERS = 20;
+const ALL_CAPS_LINE_HEADING_WORD_LIMIT = 5;
+const SECTION_HEADING_PREFIX_RE = /^\s*(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?\s*\p{Lu}/u;
+const LINE_LETTER_RE = /\p{L}/gu;
+const ENTITY_WORD_TOKEN_RE = /\p{L}[\p{L}\p{M}\p{N}'’\-]*/gu;
+const isAllCapsBoilerplateLine = (fullText, start, length) => {
+	const lineStart = fullText.lastIndexOf("\n", start) + 1;
+	const lineEndIdx = fullText.indexOf("\n", start + length);
+	const line = fullText.slice(lineStart, lineEndIdx === -1 ? fullText.length : lineEndIdx);
+	let letterCount = 0;
+	let upperCount = 0;
+	let outsideEntityLetters = 0;
+	const entityRelStart = start - lineStart;
+	const entityRelEnd = entityRelStart + length;
+	LINE_LETTER_RE.lastIndex = 0;
+	for (let m = LINE_LETTER_RE.exec(line); m !== null; m = LINE_LETTER_RE.exec(line)) {
+		const ch = m[0];
+		letterCount += 1;
+		if (ch === ch.toUpperCase() && ch !== ch.toLowerCase()) upperCount += 1;
+		if (m.index < entityRelStart || m.index >= entityRelEnd) outsideEntityLetters += 1;
+	}
+	if (letterCount <= ALL_CAPS_LINE_LETTER_THRESHOLD) return false;
+	if (upperCount / letterCount < ALL_CAPS_LINE_RATIO) return false;
+	if (SECTION_HEADING_PREFIX_RE.test(line)) return true;
+	if (outsideEntityLetters >= ALL_CAPS_LINE_PROSE_EXTRA_LETTERS) return true;
+	const entityText = fullText.slice(start, start + length);
+	if ((entityText.match(ENTITY_WORD_TOKEN_RE) ?? []).length > ALL_CAPS_LINE_HEADING_WORD_LIMIT && !entityText.includes(",")) return true;
+	return false;
+};
+const isAllCapsCandidate = (text) => text === text.toUpperCase() && /\p{Lu}/u.test(text);
 const SECTION_NUMBER_RE = /^(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?$/;
 const STANDALONE_YEAR_RE = /^(?:19|20)\d{2}$/;
 const NUMBER_ABBREV_RE = /(?:^|[\s(])(?:č|čís|nr|no|n)\.\s*$/i;
@@ -4245,6 +4472,9 @@ const filterFalsePositives = (entities, ctx = defaultContext, fullText) => {
 		if (TEMPLATE_PLACEHOLDER_RE.test(trimmed)) continue;
 		const maxLen = MAX_ENTITY_LENGTH[normalized.label];
 		if (maxLen && trimmed.length > maxLen && normalized.source !== "legal-form") continue;
+		const maxWords = MAX_ENTITY_WORDS[normalized.label];
+		if (maxWords && OPEN_ENDED_SOURCES.has(normalized.source) && countWordTokens(trimmed) > maxWords) continue;
+		if (fullText && normalized.label === "organization" && isAllCapsCandidate(trimmed) && isAllCapsBoilerplateLine(fullText, normalized.start, trimmed.length)) continue;
 		if (SECTION_NUMBER_RE.test(trimmed) && normalized.source !== "trigger") continue;
 		if (STANDALONE_YEAR_RE.test(trimmed) && normalized.source !== "trigger") continue;
 		if (fullText && normalized.source !== "trigger" && /^\d/.test(trimmed) && NUMBER_ABBREV_RE.test(fullText.slice(Math.max(0, normalized.start - 10), normalized.start))) continue;
@@ -6310,7 +6540,7 @@ const mergeAdjacent = (entities, fullText) => {
 				break;
 			}
 		}
-		if (!hasLockedBoundary$1(prev) && !(isLegalFormOrganization(prev) && isLegalFormOrganization(entity) && gap.includes(",")) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
+		if (!hasLockedBoundary$1(prev) && !((isLegalFormOrganization(prev) || isLegalFormOrganization(entity)) && gap.includes(",")) && !gapOccupied && gap.length <= MAX_GAP && GAP_PATTERN.test(gap)) {
 			prev.end = entity.end;
 			prev.text = fullText.slice(prev.start, prev.end);
 			prev.score = Math.max(prev.score, entity.score);
@@ -7109,6 +7339,8 @@ const runPipeline = async (options) => {
 	const rawTriggerEntities = config.enableTriggerPhrases ? processTriggerMatches(regexMatches, slices.triggers.start, slices.triggers.end, fullText, search.triggerRules) : [];
 	const triggerEntities = filterAllowedLabels(rawTriggerEntities, preHotwordAllowedLabels);
 	if (triggerEntities.length > 0) log("trigger-phrases", `${triggerEntities.length} matches`);
+	const signatureEntities = filterAllowedLabels(detectSignatures(fullText, ctx), preHotwordAllowedLabels);
+	if (signatureEntities.length > 0) log("signatures", `${signatureEntities.length} matches`);
 	checkAbort(signal);
 	let rawNameCorpusEntities = [];
 	let nameCorpusEntities = [];
@@ -7165,6 +7397,7 @@ const runPipeline = async (options) => {
 	checkAbort(signal);
 	const preAddressEntities = [
 		...triggerEntities,
+		...signatureEntities,
 		...regexEntities,
 		...legalFormEntities,
 		...nameCorpusEntities,