mask-privacy 3.3.0 → 3.5.0

package/README.md

@@ -127,27 +127,18 @@ Performance-sensitive deployments utilize the built-in `LocalTransformersScanner
 ### 7. Sub-string Detokenization
 Mask includes the ability to detokenize PII embedded within larger text blocks (like email bodies or chat messages). `detokenizeText()` uses high-performance regex to find and restore all tokens within a paragraph before they hit your tools.
 
-## Multilingual PII Detection (Waterfall Pipeline)
+## Multilingual PII Detection (2-Tier Waterfall)
 
-Mask is built for the global enterprise. The TypeScript SDK implements a **3-Tier Waterfall Detection** strategy for high-precision PII detection in **English and Spanish** using local ONNX models.
-
-### Supported Language Matrix
-
-Mask provides first-class support for the following languages:
-
-| Language | Code | Tier 0 (DLP) | Tier 2 (NLP Engine) |
-| :--- | :--- | :--- | :--- |
-| **English** | `en` | ✅ Full | DistilBERT (Simple) |
-| **Spanish** | `es` | ✅ Full | BERT Multilingual |
+Mask is built for the global enterprise. The TypeScript SDK implements a **2-Tier Model-Augmented Waterfall** strategy for high-precision PII detection in **English and Spanish**.
 
 ### How the Waterfall Works: The Excising Mechanism
 
-To maintain high performance, the TypeScript SDK does not simply run three separate scans. It uses a **Sequential Mutation** strategy:
+To maintain high performance, the TypeScript SDK does not simply run multiple separate scans. It uses a **Sequential Mutation** strategy:
 
-1.  **Tier 0 & 1 (The Scouts):** The SDK first runs the high-speed DLP and Regex engines synchronously in the main thread.
-2.  **Immediate Tokenization:** Any PII found by these tiers is **immediately replaced** by a token in the string buffer.
-3.  **Tier 2 (The Heavy Infantry):** The expensive NLP engine (Transformers.js) only scans the *remaining* text. Because the PII has already been "excised" (cut out and replaced with tokens), the NLP engine doesn't waste compute on data already identified.
-4.  **Bypass Logic:** All tiers are "token-aware." If a scan encounter a string that is already a Mask token, it skips it entirely, preventing redundant processing or "double-tokenization."
+1.  **Tier 0: Deterministic (The Registry):** The SDK first runs the high-speed DLP and Registry engines. These use regex + checksums (Luhn, Mod-97, Mod-11) + Proximity Keywords to identify structured PII (Bank Accounts, SSNs, DNI, NUSS, etc.) with 100% precision.
+2.  **Immediate Tokenization:** Any PII found by Tier 0 is **immediately replaced** by a token in the string buffer.
+3.  **Tier 1: Probabilistic (Neural NER):** The expensive NLP engine (Transformers.js) only scans the *remaining* text for unstructured entities: **PERSON**, **LOCATION**, and **ORGANIZATION**. Because Tier 0 PII has already been "excised", the NLP engine doesn't waste compute on data already identified, and entity collisions are avoided.
+4.  **Bypass Logic:** All tiers are "token-aware." If a scan encounters a string that is already a Mask token, it skips it entirely.
 
 ---
 

package/dist/index.d.mts

@@ -388,7 +388,7 @@ declare class DLPPatternRegistry {
     getCategoryRegexesMap(locale?: string): Map<string, {
         re: RegExp;
         typeOrder: string[];
-    }>;
+    }[]>;
     getCategoryTypeMap(categoryName: string, locale?: string): string[];
     private compileForLocale;
     private buildCatalogue;
@@ -481,7 +481,7 @@ declare class DLPConfidenceScorer {
  * Provides format-preserving encryption, local/distributed vaulting,
  * and framework-agnostic tool interception hooks.
  */
-declare const VERSION = "3.3.0";
+declare const VERSION = "3.5.0";
 
 /**
  * Detect PII entities in text and return a list of objects with metadata.

package/dist/index.d.ts

@@ -388,7 +388,7 @@ declare class DLPPatternRegistry {
     getCategoryRegexesMap(locale?: string): Map<string, {
         re: RegExp;
         typeOrder: string[];
-    }>;
+    }[]>;
     getCategoryTypeMap(categoryName: string, locale?: string): string[];
     private compileForLocale;
     private buildCatalogue;
@@ -481,7 +481,7 @@ declare class DLPConfidenceScorer {
  * Provides format-preserving encryption, local/distributed vaulting,
  * and framework-agnostic tool interception hooks.
  */
-declare const VERSION = "3.3.0";
+declare const VERSION = "3.5.0";
 
 /**
  * Detect PII entities in text and return a list of objects with metadata.

package/dist/index.js

@@ -43855,7 +43855,8 @@ var init_registry = __esm({
         return this.localeCategoryRegexMap.get(locale);
       }
       getCategoryTypeMap(categoryName, locale = "en") {
-        return this.localeCategoryRegexMap.get(locale)?.get(categoryName)?.typeOrder ?? [];
+        const groups = this.localeCategoryRegexMap.get(locale)?.get(categoryName) ?? [];
+        return groups.flatMap((g6) => g6.typeOrder);
       }
       compileForLocale(locale) {
         const localePool = /* @__PURE__ */ new Map();
@@ -43874,20 +43875,38 @@ var init_registry = __esm({
             if (aVal !== bVal) return aVal - bVal;
             return b6.compiledRe.source.length - a6.compiledRe.source.length;
           });
-          const parts = [];
-          const typeOrder = [];
+          const csParts = [];
+          const csOrder = [];
+          const ciParts = [];
+          const ciOrder = [];
           for (const [typeName, desc] of entries) {
-            parts.push(`(?<${typeName}>${desc.compiledRe.source})`);
-            typeOrder.push(typeName);
+            const named = `(?<${typeName}>${desc.compiledRe.source})`;
+            if (desc.compiledRe.flags.includes("i")) {
+              ciParts.push(named);
+              ciOrder.push(typeName);
+            } else {
+              csParts.push(named);
+              csOrder.push(typeName);
+            }
           }
-          const combinedSource = parts.join("|");
-          const needsI = entries.some(([, d6]) => d6.compiledRe.flags.includes("i"));
-          const flags = needsI ? "gi" : "g";
-          try {
-            const re = new RegExp(combinedSource, flags);
-            categoryMap.set(catKey, { re, typeOrder });
-          } catch (err2) {
-            console.error(`[DLPPatternRegistry] Locale [${locale}] category [${catKey}] failed:`, err2);
+          const groups = [];
+          const subGroups = [
+            [csParts, csOrder, "g"],
+            [ciParts, ciOrder, "gi"]
+          ];
+          for (const [parts, order, flags] of subGroups) {
+            if (parts.length === 0) continue;
+            try {
+              groups.push({ re: new RegExp(parts.join("|"), flags), typeOrder: order });
+            } catch (err2) {
+              console.error(
+                `[DLPPatternRegistry] Locale [${locale}] category [${catKey}] (${flags}) failed:`,
+                err2
+              );
+            }
+          }
+          if (groups.length > 0) {
+            categoryMap.set(catKey, groups);
           }
         }
         this.localeCategoryRegexMap.set(locale, categoryMap);
@@ -59071,6 +59090,11 @@ var init_transformers_scanner = __esm({
 });
 
 // src/core/scanner.ts
+async function chunkEncode(items, fn) {
+  for (let i6 = 0; i6 < items.length; i6 += CHUNK_SIZE) {
+    await Promise.all(items.slice(i6, i6 + CHUNK_SIZE).map(fn));
+  }
+}
 function getScanner() {
   if (scannerInstance === null) {
     const scannerType = config.MASK_SCANNER_TYPE;
@@ -59084,7 +59108,7 @@ function getScanner() {
   }
   return scannerInstance;
 }
-var _dlpLanguageResolver, _dlpPatternRegistry, _dlpValidationEngine, _dlpConfidenceScorer; exports.BaseScanner = void 0; exports.PresidioScanner = void 0; var scannerInstance;
+var _dlpLanguageResolver, _dlpPatternRegistry, _dlpValidationEngine, _dlpConfidenceScorer, CHUNK_SIZE; exports.BaseScanner = void 0; exports.PresidioScanner = void 0; var scannerInstance;
 var init_scanner = __esm({
   "src/core/scanner.ts"() {
     init_config();
@@ -59099,6 +59123,7 @@ var init_scanner = __esm({
     _dlpPatternRegistry = new exports.DLPPatternRegistry();
     _dlpValidationEngine = new exports.DLPValidationEngine();
     _dlpConfidenceScorer = new exports.DLPConfidenceScorer();
+    CHUNK_SIZE = 50;
     exports.BaseScanner = class {
       constructor() {
         this._supportedEntities = [
@@ -59137,51 +59162,53 @@ var init_scanner = __esm({
         const detectedLanguage = _dlpLanguageResolver.resolve(text);
         const spans = [];
         const categoryMap = _dlpPatternRegistry.getCategoryRegexesMap();
-        for (const [catKey, { re, typeOrder }] of categoryMap.entries()) {
-          const megaRe = new RegExp(re.source, re.flags);
-          let m6;
-          while ((m6 = megaRe.exec(text)) !== null) {
-            const groups = m6.groups ?? {};
-            let typeTag;
-            for (const name of typeOrder) {
-              if (groups[name] !== void 0) {
-                typeTag = name;
-                break;
+        for (const [catKey, groups] of categoryMap.entries()) {
+          for (const { re, typeOrder } of groups) {
+            const megaRe = new RegExp(re.source, re.flags);
+            let m6;
+            while ((m6 = megaRe.exec(text)) !== null) {
+              const groups2 = m6.groups ?? {};
+              let typeTag;
+              for (const name of typeOrder) {
+                if (groups2[name] !== void 0) {
+                  typeTag = name;
+                  break;
+                }
               }
-            }
-            if (!typeTag) continue;
-            const matchedStr = m6[0];
-            if (looksLikeToken(matchedStr)) continue;
-            const descriptor = _dlpPatternRegistry.descriptorFor(typeTag);
-            if (!descriptor) continue;
-            const validatorResult = _dlpValidationEngine.run(descriptor.validatorTag, matchedStr);
-            let conf;
-            if (validatorResult === false) {
-              if (descriptor.isHighEntropy) {
-                conf = 0.85;
+              if (!typeTag) continue;
+              const matchedStr = m6[0];
+              if (looksLikeToken(matchedStr)) continue;
+              const descriptor = _dlpPatternRegistry.descriptorFor(typeTag);
+              if (!descriptor) continue;
+              const validatorResult = _dlpValidationEngine.run(descriptor.validatorTag, matchedStr);
+              let conf;
+              if (validatorResult === false) {
+                if (descriptor.isHighEntropy) {
+                  conf = 0.85;
+                } else {
+                  continue;
+                }
               } else {
-                continue;
+                conf = _dlpConfidenceScorer.score({
+                  baseRisk: descriptor.baseRisk,
+                  matchStart: m6.index,
+                  matchEnd: m6.index + matchedStr.length,
+                  fullText: text,
+                  proximityTerms: descriptor.proximityTerms,
+                  validatorPassed: validatorResult
+                });
+              }
+              if (conf >= confidenceThreshold) {
+                spans.push({
+                  start: m6.index,
+                  end: m6.index + matchedStr.length,
+                  entityType: typeTag,
+                  originalValue: matchedStr,
+                  confidence: conf,
+                  method: "dlp_heuristic",
+                  language: detectedLanguage
+                });
               }
-            } else {
-              conf = _dlpConfidenceScorer.score({
-                baseRisk: descriptor.baseRisk,
-                matchStart: m6.index,
-                matchEnd: m6.index + matchedStr.length,
-                fullText: text,
-                proximityTerms: descriptor.proximityTerms,
-                validatorPassed: validatorResult
-              });
-            }
-            if (conf >= confidenceThreshold) {
-              spans.push({
-                start: m6.index,
-                end: m6.index + matchedStr.length,
-                entityType: typeTag,
-                originalValue: matchedStr,
-                confidence: conf,
-                method: "dlp_heuristic",
-                language: detectedLanguage
-              });
             }
           }
         }
@@ -59261,11 +59288,11 @@ var init_scanner = __esm({
       }
       _resolveBoost(context) {
         if (!context) return /* @__PURE__ */ new Set();
-        const lowered = context.toLowerCase();
         const boosted = /* @__PURE__ */ new Set();
         for (const [, desc] of _dlpPatternRegistry.iterDescriptors()) {
           for (const term of desc.proximityTerms) {
-            if (lowered.includes(term)) {
+            const pattern = new RegExp("\\b" + term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + "\\b", "i");
+            if (pattern.test(context)) {
               boosted.add(desc.category.toLowerCase());
               break;
             }
@@ -59284,9 +59311,9 @@ var init_scanner = __esm({
           allSpans.push(...await this._tier0CollectSpans(text, confidenceThreshold));
         }
         const resolved = resolveOverlaps(allSpans);
-        await Promise.all(resolved.map(async (span) => {
+        await chunkEncode(resolved, async (span) => {
           span.maskedValue = await _encode(span.originalValue, { entityType: span.entityType });
-        }));
+        });
         let currentText = reconstruct(text, resolved);
         if (pipeline.includes("nlp")) {
           [currentText] = await this._tier2Nlp(currentText, _encode, boost, !!options.aggressive, confidenceThreshold);
@@ -59305,7 +59332,7 @@ var init_scanner = __esm({
           allSpans.push(...await this._tier0CollectSpans(text, confidenceThreshold));
         }
         const resolved = resolveOverlaps(allSpans);
-        await Promise.all(resolved.map(async (span) => {
+        await chunkEncode(resolved, async (span) => {
           span.maskedValue = await _encode(span.originalValue, { entityType: span.entityType });
           allEntities.push({
             type: span.entityType,
@@ -59315,7 +59342,7 @@ var init_scanner = __esm({
             masked_value: span.maskedValue,
             language: span.language
           });
-        }));
+        });
         const remaining = reconstruct(text, resolved);
         if (pipeline.includes("nlp")) {
           const [, tier2] = await this._tier2Nlp(remaining, _encode, boost, !!options.aggressive, confidenceThreshold);
@@ -59619,7 +59646,7 @@ init_handlers();
 init_scorer();
 
 // src/index.ts
-var VERSION = "3.3.0";
+var VERSION = "3.5.0";
 async function detectEntitiesWithConfidence(text, options = {}) {
   const scanner = getScanner();
   return await scanner.scanAndReturnEntities(text, options);