mask-privacy 3.0.0 → 3.2.0

package/src/core/span.ts

@@ -0,0 +1,76 @@
+/**
+ * Span Resolution Engine — Sweep-Line Overlap Resolver (TypeScript).
+ *
+ * All detection tiers now return Span objects instead of mutating the text.
+ * resolveOverlaps() chooses the winning span in every conflicting region,
+ * and reconstruct() rebuilds the string exactly once.
+ */
+
+export interface Span {
+  start: number;
+  end: number;
+  entityType: string;
+  originalValue: string;
+  confidence: number;
+  method: string;      // "dlp_heuristic" | "regex" | "nlp"
+  language?: string;
+  maskedValue?: string;
+}
+
+/**
+ * Return a non-overlapping, right-to-left-ordered subset of spans.
+ *
+ * Algorithm:
+ * 1. Sort by start ASC, length DESC (prefer longer), confidence DESC.
+ * 2. Walk left-to-right tracking occupiedEnd.
+ * 3. Fully-contained spans are discarded.
+ * 4. Partial overlaps resolve by confidence (higher wins).
+ */
+export function resolveOverlaps(spans: Span[]): Span[] {
+  if (spans.length === 0) return [];
+
+  const sorted = [...spans].sort((a, b) => {
+    if (a.start !== b.start) return a.start - b.start;
+    const lenDiff = (b.end - b.start) - (a.end - a.start);
+    if (lenDiff !== 0) return lenDiff;
+    return b.confidence - a.confidence;
+  });
+
+  const resolved: Span[] = [];
+  let occupiedEnd = -1;
+
+  for (const span of sorted) {
+    if (span.start >= occupiedEnd) {
+      resolved.push(span);
+      occupiedEnd = span.end;
+    } else if (span.end <= occupiedEnd) {
+      // Fully inside an already-accepted span — discard.
+      continue;
+    } else {
+      // Partial overlap — keep highest confidence.
+      const last = resolved[resolved.length - 1];
+      if (span.confidence > last.confidence) {
+        resolved.pop();
+        resolved.push(span);
+        occupiedEnd = span.end;
+      }
+    }
+  }
+
+  // Return descending start order for right-to-left reconstruction.
+  return resolved.sort((a, b) => b.start - a.start);
+}
+
+/**
+ * Rebuild text from a right-to-left-ordered list of resolved spans.
+ * Each span must have maskedValue populated.
+ * This is the single string-construction pass that replaces all slice loops.
+ */
+export function reconstruct(text: string, resolvedSpans: Span[]): string {
+  let result = text;
+  for (const span of resolvedSpans) {
+    if (span.maskedValue == null) continue;
+    result = result.slice(0, span.start) + span.maskedValue + result.slice(span.end);
+  }
+  return result;
+}

package/src/core/transformers_scanner.ts

@@ -96,7 +96,7 @@ export class LocalTransformersScanner extends BaseScanner {
 
   protected async _tier2Nlp(
     text: string,
-    encodeFn: (val: string) => Promise<string>,
+    encodeFn: (val: string, options?: any) => Promise<string>,
     boostEntities: Set<string>,
     aggressive: boolean,
     confidenceThreshold: number,
@@ -136,7 +136,7 @@ export class LocalTransformersScanner extends BaseScanner {
         }
         
         if (confidence >= confidenceThreshold && !looksLikeToken(val) && val.length > 1) {
-          const token = await encodeFn(val);
+          const token = await encodeFn(val, { entityType: entityType });
           entities.push({
             type: entityType,
             value: val,

package/src/core/vault.ts

@@ -538,6 +538,7 @@ export type EncodeOptions = {
     ttl?: number;
     searchBuckets?: ('year' | 'month' | 'day' | 'numeric')[];
     searchBucketSize?: number;
+    entityType?: string;
 };
 
 /**
@@ -565,7 +566,7 @@ export async function encode(rawText: string, options: EncodeOptions = {}): Prom
   }
 
   // 2. Generate new token
-  const token = await generateFPEToken(text);
+  const token = await generateFPEToken(text, options.entityType || 'UNKNOWN');
 
   // 3. Encrypt the plaintext before it touches the vault
   const ciphertext = cryptoEngine.encrypt(text);

package/tests/async.test.ts

@@ -21,7 +21,7 @@ describe('TestAsyncWrappers', () => {
     test('test_module_level_async_wrappers', async () => {
         const token = await aencode("test@async.com");
         expect(looksLikeToken(token)).toBe(true);
-        expect(token).toMatch(/@email\.com$/);
+        expect(token).toMatch(/@async\.com$/);
 
         const plaintext = await adecode(token);
         expect(plaintext).toBe("test@async.com");
@@ -43,6 +43,6 @@ describe('TestAsyncWrappers', () => {
         const safeText = await client.ascanAndTokenize(text);
         expect(safeText).not.toContain("bob@example.com");
         expect(safeText).toContain("tkn-");
-        expect(safeText).toMatch(/@email\.com$/);
+        expect(safeText).toMatch(/@example\.com$/);
     });
 });

package/tests/dlp_hardened.test.ts

@@ -0,0 +1,21 @@
+/* global describe, test, expect */
+import { getScanner } from '../src/index';
+
+async function ascanAndTokenize(text: string, options: any = {}) {
+    return getScanner().scanAndTokenize(text, options);
+}
+
+describe('Multilingual ID Hardening (TS)', () => {
+
+
+
+  test('Locale-Aware Precision (ES_DNI)', async () => {
+    // Spanish DNI in English context
+    const raw = "My DNI is 12345678Z";
+    const masked = await ascanAndTokenize(raw, { pipeline: ['dlp'] });
+    
+    // Should mask using the ES_DNI generator (starts with 000)
+    expect(masked).toContain("000");
+  });
+
+});

package/tests/fpe.test.ts

@@ -17,7 +17,7 @@ describe('TestFPETokenGeneration', () => {
 
   test('test_email_format', async () => {
     const token = await generateFPEToken("user@company.io");
-    expect(token.endsWith("@email.com")).toBe(true);
+    expect(token.endsWith("@company.io")).toBe(true);
     expect(token.startsWith("tkn-")).toBe(true);
     expect(token).toMatch(/^[^@]+@[^@]+\.[^@]+$/);
   });
@@ -60,7 +60,7 @@ describe('TestFPETokenGeneration', () => {
     const t1 = await generateFPEToken("a@b.com");
     const t2 = await generateFPEToken("a@b.com");
     expect(t1).toBe(t2);
-    expect(t1.endsWith("@email.com")).toBe(true);
+    expect(t1.endsWith("@b.com")).toBe(true);
   });
 
   test('test_different_inputs_different_tokens', () => {
@@ -85,7 +85,7 @@ describe('TestFPETokenGeneration', () => {
   });
 
   test('test_whitespace_stripped_determinism', async () => {
-    expect(await generateFPEToken(" someone@email.com ")).toBe(await generateFPEToken("someone@email.com"));
+    expect(await generateFPEToken(" someone@example.com ")).toBe(await generateFPEToken("someone@example.com"));
   });
 
   test('test_fail_fast_when_key_missing', async () => {
@@ -100,7 +100,7 @@ describe('TestFPETokenGeneration', () => {
 
 describe('TestLooksLikeToken', () => {
   test('test_email_token', () => {
-    expect(looksLikeToken("tkn-abcd1234@email.com")).toBe(true);
+    expect(looksLikeToken("tkn-abcd1234abcd@example.com")).toBe(true);
   });
 
   test('test_phone_token', () => {

package/tests/hooks.test.ts

@@ -60,7 +60,7 @@ describe('TestHooks', () => {
         test('test_encodes_raw_email', async () => {
             const result = await deepEncodePII({"email": "test@example.com"});
             expect(looksLikeToken(result.email)).toBe(true);
-            expect(result.email).toMatch(/@email\.com$/);
+            expect(result.email).toMatch(/@example\.com$/);
         });
 
         test('test_does_not_double_encode_token', async () => {
@@ -91,7 +91,7 @@ describe('TestHooks', () => {
             const args = "Contact us at support@example.com for help.";
             const result = await deepEncodePII(args);
             expect(typeof result).toBe('string');
-            expect(result).toContain("@email.com");
+            expect(result).toContain("@example.com");
             expect(result).not.toContain("support@example.com");
         });
 

package/tests/langchain.test.ts

@@ -41,7 +41,7 @@ describe('TestLangchainHooks', () => {
             const result = await secure.run(token, "Welcome");
 
             expect(result.target).not.toBe("user@example.com");
-            expect(result.target).toMatch(/@email\.com$/);
+            expect(result.target).toMatch(/@example\.com$/);
         });
     });
 
@@ -89,7 +89,7 @@ describe('TestLangchainHooks', () => {
 
             const result = await sendEmail(token, "Hello");
             expect(result).not.toContain("dev@mask.ai");
-            expect(result).toContain("@email.com");
+            expect(result).toContain("@mask.ai");
         });
 
         test('test_secure_tool_preserves_non_pii', async () => {

package/tests/llamaindex.test.ts

@@ -42,7 +42,7 @@ describe('TestLlamaindexHooks', () => {
             const result = await secureTool.run(token, "Give me the records");
 
             expect(result.target).not.toBe("admin@hospital.com");
-            expect(result.target).toMatch(/@email\.com$/);
+            expect(result.target).toMatch(/@hospital\.com$/);
         });
     });
 

package/tests/scanner.test.ts

@@ -18,7 +18,6 @@ describe('TestInternationalPhonePatterns', () => {
   });
 
   test.each([
-    "+1 555 123 4567",
     "020 7946 0958",
     "just some text",
   ])('test_intl_phone_no_match: %s', (nonMatch) => {

package/tests/substring.test.ts

@@ -50,7 +50,7 @@ describe('TestSubstringDetokenization', () => {
   });
 
   test('test_detokenize_text_lenient', async () => {
-    const bogus = "tkn-12345678@email.com";
+    const bogus = "tkn-12345678@example.com";
     const paragraph = `Hello ${bogus}`;
     
     const restored = await detokenizeText(paragraph);

package/tests/vault.test.ts

@@ -46,7 +46,7 @@ describe('TestEncodeDecodePublicAPI', () => {
 
   test('test_roundtrip_email', async () => {
     const token = await encode("user@example.com");
-    expect(token.endsWith("@email.com")).toBe(true);
+    expect(token.endsWith("@example.com")).toBe(true);
     expect(await decode(token)).toBe("user@example.com");
   });