@stll/anonymize 0.0.4 → 0.0.5

package/dist/index.d.ts

@@ -204,7 +204,7 @@ type PipelineConfig = {
  * every pipeline run and never persisted to the database.
  * Renaming a label here requires no migration.
  */
-declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone number", "address", "email address", "date", "date of birth", "bank account number", "iban", "tax identification number", "identity card number", "registration number", "credit card number", "passport number", "monetary amount"];
+declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone number", "address", "email address", "date", "date of birth", "bank account number", "iban", "tax identification number", "identity card number", "registration number", "credit card number", "passport number", "monetary amount", "land parcel"];
 //#endregion
 //#region src/detectors/regex.d.ts
 type RegexMeta = {

package/dist/index.js

@@ -1,6 +1,10 @@
+import { createRequire } from "node:module";
 import { at, be, bg, cy, cz, de, dk, ee, es, fi, fr, gb, gr, hr, hu, ie, it, lt, lu, lv, mt, nl, pl, pt, ro, se, si, sk } from "@stll/stdnum";
 import { toRegex } from "@stll/stdnum/patterns";
 import { TextSearch } from "@stll/text-search";
+//#region \0rolldown/runtime.js
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+//#endregion
 //#region src/types.ts
 /**
 * Source of a detected entity span.
@@ -59,7 +63,8 @@ const DEFAULT_ENTITY_LABELS = [
 	"registration number",
 	"credit card number",
 	"passport number",
-	"monetary amount"
+	"monetary amount",
+	"land parcel"
 ];
 //#endregion
 //#region src/context.ts
@@ -140,6 +145,7 @@ const LOADER_REGISTRIES = {
 		it: () => import("@stll/anonymize-data/config/triggers.it.json"),
 		pl: () => import("@stll/anonymize-data/config/triggers.pl.json"),
 		ro: () => import("@stll/anonymize-data/config/triggers.ro.json"),
+		sk: () => import("@stll/anonymize-data/config/triggers.sk.json"),
 		sv: () => import("@stll/anonymize-data/config/triggers.sv.json")
 	},
 	coreference: {
@@ -160,6 +166,7 @@ const FALLBACK_LANGUAGES = {
 		"it",
 		"pl",
 		"ro",
+		"sk",
 		"sv"
 	],
 	coreference: [
@@ -991,6 +998,68 @@ const POST_NOMINALS = [
 	"CIPP"
 ];
 //#endregion
+//#region src/util/char-groups.ts
+/** Chars that need escaping inside a regex char class. */
+const REGEX_CLASS_SPECIAL = /[\\\]^-]/;
+const escapeForCharClass = (ch) => REGEX_CLASS_SPECIAL.test(ch) ? `\\${ch}` : ch;
+let cached;
+const loadConfig = () => {
+	if (cached) return cached;
+	cached = __require("@stll/anonymize-data/config/char-groups.json");
+	return cached;
+};
+/**
+* Get the raw characters for a named group.
+* Throws if the group does not exist.
+*/
+const charSet = (group) => {
+	const g = loadConfig().groups[group];
+	if (!g) throw new Error(`Unknown char group: "${group}"`);
+	return g.chars.map((entry) => entry.char);
+};
+/**
+* Build a regex character class string for a named
+* group. E.g., charClass("dash") returns a string
+* like "[-\u2013\u2014\u2010\u2011\u2212\u2043]".
+*
+* The hyphen-minus is placed first so it is treated
+* as a literal, not a range indicator.
+*/
+const charClass = (group) => {
+	return `[${[...charSet(group)].sort((a, b) => {
+		if (a === "-") return -1;
+		if (b === "-") return 1;
+		return a.localeCompare(b);
+	}).map(escapeForCharClass).join("")}]`;
+};
+/**
+* Return the inner content of a character class (without
+* the surrounding brackets). Useful for embedding a group
+* inside a larger character class, e.g.:
+*   `[\\s&,.${charClassInner("dash")}]`
+*/
+const charClassInner = (group) => {
+	return [...charSet(group)].sort((a, b) => {
+		if (a === "-") return -1;
+		if (b === "-") return 1;
+		return a.localeCompare(b);
+	}).map(escapeForCharClass).join("");
+};
+/** Regex char class matching all dash variants. */
+const DASH = charClass("dash");
+/**
+* Inner content of the dash char class (no brackets).
+* For embedding inside a larger character class:
+*   `[\\s&,.${DASH_INNER}]`
+*/
+const DASH_INNER = charClassInner("dash");
+charClass("space");
+charClass("quote-double");
+charClass("quote-single");
+charClass("slash");
+charClass("dot");
+charClass("colon");
+//#endregion
 //#region src/detectors/regex.ts
 const MIN_PHONE_LENGTH = 7;
 const MIN_MONTH_NAME_LENGTH = 3;
@@ -1328,9 +1397,9 @@ const buildCurrencyPatterns = (data) => {
 	if (!symbols && !trailingAlt) return [];
 	const NUM = "(?:\\d{1,3}(?:[,.'[^\\S\\n\\t]]\\d{3})+|\\d{1,9})";
 	const patterns = [];
-	if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}(?:[.,](?:\\d{1,2}[-–—]?|[-–—]{1,2})?)?\\b`);
-	if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}(?:[.,](?:\\d{1,2}[-–—]?|[-–—]{1,2})?)?\\b`);
-	if (trailingAlt) patterns.push(`\\b${NUM}(?:[.,](?:\\d{1,2}[-–—]?|[-–—]{1,2})?)?[^\\S\\n\\t]{0,4}(?:${trailingAlt})(?:\\b|(?=\\s|[.,;!?)]|$))`);
+	if (symbols) patterns.push(`(?:[${symbols}])[^\\S\\n\\t]?${NUM}(?:[.,][^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2})?)?\\b`);
+	if (trailingAlt) patterns.push(`\\b(?:${trailingAlt})[^\\S\\n\\t]{0,2}${NUM}(?:[.,][^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2})?)?\\b`);
+	if (trailingAlt) patterns.push(`\\b${NUM}(?:[.,][^\\S\\n\\t]?(?:\\d{1,2}${DASH}?|${DASH}{1,2})?)?[^\\S\\n\\t]{0,4}(?:${trailingAlt})(?:\\b|(?=\\s|[.,;!?)]|$))`);
 	return patterns;
 };
 /** Cached promise for currency patterns. Loaded once. */
@@ -1441,7 +1510,7 @@ const isShortForm = (form) => form.replace(/[.\s]/g, "").length <= 3 && !form.in
 const buildPatternString = (forms, requireCapBefore) => {
 	if (forms.length === 0) return null;
 	const alt = forms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
-	return `${`(?:${CAP_WORD})(?:(?:[\\s&,.\\-–—]{1,4}|\\s+(?:a|and|und|et|e|y|i)\\s+)(?:${ANY_WORD})){0,10}`}${requireCapBefore ? `(?:\\s+|,\\s*)` : `\\s+`}(?:${alt})(?![${LOWER}])`;
+	return `${`(?:${CAP_WORD})(?:${`(?:[\\s&,.${DASH_INNER}]{1,4}|\\s+(?:a|and|und|et|e|y|i)\\s+)`}(?:${ANY_WORD})){0,10}`}${requireCapBefore ? `(?:\\s+|,\\s*)` : `\\s+`}(?:${alt})(?![${LOWER}])`;
 };
 /**
 * Build legal form regex pattern strings.
@@ -1470,7 +1539,7 @@ const buildLegalFormPatterns = async () => {
 	if (longPattern) patterns.push(longPattern);
 	const shortPattern = buildPatternString(allForms.filter(isShortForm), true);
 	if (shortPattern) patterns.push(shortPattern);
-	const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.\\-–—]{1,4}(?:${ALLCAP_WORD})){0,2}`;
+	const allcapPrefix = `(?:${ALLCAP_WORD})(?:[\\s&,.${DASH_INNER}]{1,4}(?:${ALLCAP_WORD})){0,2}`;
 	const allcapAlt = allForms.toSorted((a, b) => b.length - a.length).map(escapeForRegex).join("|");
 	patterns.push(`${allcapPrefix}(?:\\s+|,\\s*)(?:${allcapAlt})(?![${LOWER}])`);
 	return patterns;
@@ -1566,6 +1635,11 @@ const TRIGGER_SCORE = .95;
 const WHITESPACE_RE$1 = /\s+/;
 const LETTER_RE = /\p{L}/u;
 /**
+* Decimal-comma pattern: comma followed by digit or
+* dash notation ("0,05%", "1.529,50 Kč", "98.000,- Kč").
+*/
+const DECIMAL_COMMA_RE = new RegExp(`^,(?:\\d|${DASH}{1,2})`);
+/**
 * Post-nominal degree regex. When a comma-stop is
 * followed by a known post-nominal (Ph.D., CSc., MBA
 * etc.), skip the comma and degree, then continue.
@@ -1759,6 +1833,10 @@ const extractValue = (text, triggerEnd, strategy, label) => {
 				}
 				if (ch === ",") {
 					const afterComma = valueText.slice(end);
+					if (DECIMAL_COMMA_RE.test(afterComma)) {
+						end++;
+						continue;
+					}
 					const degreeMatch = label === "person" ? POST_NOMINAL_RE.exec(afterComma) : null;
 					if (degreeMatch) {
 						end += degreeMatch[0].length;
@@ -2243,6 +2321,7 @@ const filterFalsePositives = (entities, ctx = defaultContext) => {
 		if (maxLen && trimmed.length > maxLen && entity.source !== "legal-form") continue;
 		if (SECTION_NUMBER_RE.test(trimmed) && entity.source !== "trigger") continue;
 		if (STANDALONE_YEAR_RE.test(trimmed)) continue;
+		if (entity.label === "person" && HAS_DIGIT_RE.test(trimmed)) continue;
 		if ((entity.label === "person" || entity.label === "organization") && roles.has(trimmed.toLowerCase())) continue;
 		if (entity.label === "address" && trimmed.length > 40 && !POSTAL_CODE_RE.test(trimmed) && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed)) continue;
 		if (entity.label === "address" && entity.source === "trigger" && !HAS_DIGIT_RE.test(trimmed) && !ADDRESS_COMPONENTS_RE.test(trimmed)) continue;
@@ -2647,7 +2726,11 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
 		if (!first || !last) continue;
 		const extended = extendPersonName(fullText, first.start, last.end, ctx);
 		const score = chain.length >= 2 ? .9 : .5;
-		if (chain.length === 1 && isSentenceStart(fullText, first.start)) continue;
+		if (chain.length === 1 && isSentenceStart(fullText, first.start)) {
+			const afterEnd = last.end;
+			const rest = fullText.slice(afterEnd).trimStart();
+			if (!(rest.length > 1 && /^\p{Lu}\p{Ll}/u.test(rest))) continue;
+		}
 		results.push({
 			start: first.start,
 			end: extended.end,
@@ -2661,7 +2744,7 @@ const processDenyListMatches = (allMatches, sliceStart, sliceEnd, fullText, data
 	return results;
 };
 const DISTRICT_SUFFIX_RE = new RegExp(`^ (\\d{1,2}(?!\\d)|(?:XXX|XXIX|XXVIII|XXVII|XXVI|XXV|XXIV|XXIII|XXII|XXI|XX|XIX|XVIII|XVII|XVI|XV|XIV|XIII|XII|XI|X|IX|VIII|VII|VI|IV|III|II))(?=[\\s,;.)"\\n]|$)`);
-const POSTAL_PREFIX_RE = /(?:\d{5}|\d{3}\s\d{2})\s+$/;
+const POSTAL_PREFIX_RE = new RegExp(`(?:\\d{5}|\\d{3}\\s\\d{2})\\s*${DASH}?\\s*$`);
 const TRAILING_WORD_EXCLUSIONS = new Set([
 	"nájemce",
 	"pronajímatel",
@@ -2715,7 +2798,7 @@ const extendCityDistricts = (entities, fullText) => {
 			entity.text = fullText.slice(entity.start, entity.end);
 		}
 		const afterExt = fullText.slice(entity.end);
-		const trailingWordM = /^[\s]+(\p{Lu}\p{Ll}+)/u.exec(afterExt);
+		const trailingWordM = /^[\s]{1,4}(\p{Lu}\p{Ll}+)/u.exec(afterExt);
 		if (trailingWordM && !trailingWordM[0].includes("\n")) {
 			const candidate = (trailingWordM[1] ?? "").toLowerCase();
 			if (!TRAILING_WORD_EXCLUSIONS.has(candidate)) {
@@ -4135,11 +4218,21 @@ const unmaskNerEntities = (nerEntities, maskResult, fullText) => {
 };
 //#endregion
 //#region src/pipeline.ts
+/**
+* Sources backed by curated literal dictionaries.
+* Longer matches from these sources are more specific,
+* so the containment rule trusts their length.
+*/
+const LITERAL_SOURCES = new Set(["deny-list", "gazetteer"]);
 const shouldReplace = (a, b) => {
+	const aLen = a.end - a.start;
+	const bLen = b.end - b.start;
+	if (a.label === b.label && LITERAL_SOURCES.has(a.source) && a.start <= b.start && a.end >= b.end && aLen > bLen) return true;
+	if (a.label === b.label && LITERAL_SOURCES.has(b.source) && b.start <= a.start && b.end >= a.end && bLen > aLen) return false;
 	const aPri = DETECTOR_PRIORITY[a.source] ?? 0;
 	const bPri = DETECTOR_PRIORITY[b.source] ?? 0;
 	if (aPri !== bPri) return aPri > bPri;
-	return a.score > b.score || a.score === b.score && a.end - a.start > b.end - b.start;
+	return a.score > b.score || a.score === b.score && aLen > bLen;
 };
 /** Labels where colons are structurally significant. */
 const COLON_LABELS = new Set(["ip address", "mac address"]);
@@ -4151,12 +4244,13 @@ const sanitizeEntities = (entities) => entities.flatMap((e) => {
 	const cleaned = leadTrimmed.replace(new RegExp(`${strip.source}$`, strip.flags), "");
 	if (cleaned.length === 0) return [];
 	if (!/[\p{L}\p{N}]/u.test(cleaned)) return [];
-	if (cleaned === e.text) return [e];
+	const collapsed = cleaned.replace(/\s*\n\s*/g, " ").replace(/\s{2,}/g, " ");
+	if (collapsed === e.text) return [e];
 	return [{
 		...e,
 		start: e.start + lead,
-		end: e.start + lead + cleaned.length,
-		text: cleaned
+		end: e.start + lead + collapsed.length,
+		text: collapsed
 	}];
 });
 const mergeAndDedup = (...layers) => {
@@ -4423,7 +4517,7 @@ const normalizeEntityText = (label, text) => {
 	if (upper === "EMAIL_ADDRESS" || upper === "EMAIL") return text.toLowerCase().trim();
 	if (upper === "PHONE_NUMBER" || upper === "PHONE") return text.replace(PHONE_NOISE_RE, "");
 	if (upper === "IBAN" || upper === "BANK_ACCOUNT_NUMBER" || upper === "TAX_IDENTIFICATION_NUMBER" || upper === "REGISTRATION_NUMBER") return text.replace(SPACE_DASH_RE, "").toUpperCase();
-	if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
+	if (upper === "PERSON" || upper === "ORGANIZATION" || upper === "ADDRESS" || upper === "LAND_PARCEL") return text.replace(WHITESPACE_RE, " ").toLowerCase().trim();
 	return text.trim();
 };
 /**