mask-privacy 4.0.0 → 4.2.0

package/src/core/vault.ts

@@ -12,10 +12,10 @@
 
 import * as crypto from 'crypto';
 import { config } from '../config';
-import { generateFPEToken } from './fpe';
+import { generateDPToken } from './fpe';
 import { looksLikeToken, TOKEN_PATTERN } from './fpe_utils';
 import { getCryptoEngineAsync } from './crypto';
-import { MaskVaultConnectionError } from './exceptions';
+import { MaskVaultConnectionError, TokenCollisionError } from './exceptions';
 import { getAuditLogger } from '../telemetry/audit_logger';
 import { BucketManager } from './search';
 
@@ -50,8 +50,8 @@ export function _getFailStrategy(): string {
 
 /** Interface every vault backend must implement. */
 export abstract class BaseVault {
-  /** Persist a token → plaintext mapping with a TTL. Optionally save a reverse lookup hash. */
-  abstract store(token: string, plaintext: string, ttlSeconds: number, ptHash?: string | null): Promise<void>;
+  /** Persist a token → encrypted plaintext mapping with a TTL and optional compliance metadata. */
+  abstract store(token: string, plaintext: string, ttlSeconds: number, ptHash?: string | null, metadata?: Record<string, string> | null): Promise<void>;
 
   /** Return the existing unexpired token for a given plaintext hash, or null. */
   abstract getTokenByPlaintextHash(ptHash: string): Promise<string | null>;
@@ -59,6 +59,9 @@ export abstract class BaseVault {
   /** Return the plaintext for token, or null if missing/expired. */
   abstract retrieve(token: string): Promise<string | null>;
 
+  /** Return the plaintext hash stored for this token (used for collision detection), or null. */
+  abstract getPtHashForToken(token: string): Promise<string | null>;
+
   /** Delete a token and its reverse mapping. */
   abstract delete(token: string): Promise<void>;
 }
@@ -78,29 +81,57 @@ export function _hashPlaintext(plaintext: string, secret?: Buffer): string {
   return crypto.createHash('sha256').update(trimmed, 'utf-8').digest('hex');
 }
 
+// ---------------------------------------------------------------------------
+// Tenant-namespaced cache key helpers
+// ---------------------------------------------------------------------------
+
+function _vaultKey(token: string): string {
+  return `mask:${config.MASK_TENANT_ID}:${token}`;
+}
+function _vaultRevKey(ptHash: string): string {
+  return `mask-rev:${config.MASK_TENANT_ID}:${ptHash}`;
+}
+function _vaultHashKey(token: string): string {
+  return `mask-hash:${config.MASK_TENANT_ID}:${token}`;
+}
+
+/**
+ * Extract the ciphertext from a JSON compliance envelope {ct, meta}, or return raw.
+ * Fixes the fatal bug where tokens stored with metadata could never be decrypted.
+ */
+function _unwrapPayload(raw: string): string {
+  if (raw && raw.startsWith('{')) {
+    try {
+      const obj = JSON.parse(raw);
+      if (obj.ct) return obj.ct;
+    } catch {}
+  }
+  return raw;
+}
+
 /**
  * In-memory implementation (single-process, dev / testing)
  *
  * Map-backed vault. Fast, but state is lost across processes.
  */
 export class MemoryVault extends BaseVault {
-  private _store: Map<string, { plaintext: string; expiry: number; ptHash: string | null }>;
+  private _store: Map<string, { plaintext: string; expiry: number; ptHash: string | null; metadata: Record<string, string> }>;
   private _reverseStore: Map<string, string>;
+  private _cleanupTimer: NodeJS.Timeout | null = null;
 
   constructor() {
     super();
     this._store = new Map();
     this._reverseStore = new Map();
+    // Replace probabilistic inline cleanup with a background timer (unreffed
+    // so it never prevents process exit) to eliminate request-path latency jitter.
+    this._cleanupTimer = setInterval(() => this._cleanup(), 60_000);
+    if (this._cleanupTimer && typeof this._cleanupTimer.unref === 'function') {
+      this._cleanupTimer.unref();
+    }
   }
 
   private _cleanup(): void {
-    // Probabilistic cleanup to prevent memory bloat
-    // Configurable frequency (default 1%) balances CPU vs Memory usage
-    const cleanupFreq = config.MASK_VAULT_CLEANUP_FREQUENCY;
-    if (Math.random() > cleanupFreq) {
-      return;
-    }
-
     const now = Date.now() / 1000;
     for (const [token, entry] of this._store.entries()) {
       if (now > entry.expiry) {
@@ -112,12 +143,28 @@ export class MemoryVault extends BaseVault {
     }
   }
 
-  async store(token: string, plaintext: string, ttlSeconds: number, ptHash: string | null = null): Promise<void> {
-    this._cleanup();
+  async store(token: string, ciphertext: string, ttlSeconds: number, ptHash: string | null = null, metadata: Record<string, string> | null = null): Promise<void> {
+    // Enforce max memory keys bound (OOM prevention)
+    if (!this._store.has(token) && this._store.size >= config.MASK_VAULT_MAX_MEMORY_KEYS) {
+      const firstKey = this._store.keys().next().value;
+      if (firstKey !== undefined) {
+        const oldEntry = this._store.get(firstKey);
+        this._store.delete(firstKey);
+        if (oldEntry?.ptHash && this._reverseStore.get(oldEntry.ptHash) === firstKey) {
+          this._reverseStore.delete(oldEntry.ptHash);
+        }
+      }
+    }
+
+    const existing = this._store.get(token);
+    // Delete to reset insertion order (implementing true LRU over simple FIFO)
+    if (existing) this._store.delete(token);
+
     this._store.set(token, {
-      plaintext,
+      plaintext: ciphertext,
       expiry: (Date.now() / 1000) + ttlSeconds,
-      ptHash
+      ptHash,
+      metadata: { ...(existing?.metadata || {}), ...(metadata || {}) },
     });
     if (ptHash) {
       this._reverseStore.set(ptHash, token);
@@ -125,7 +172,6 @@ export class MemoryVault extends BaseVault {
   }
 
   async getTokenByPlaintextHash(ptHash: string): Promise<string | null> {
-    this._cleanup();
     const token = this._reverseStore.get(ptHash);
     if (token && this._store.has(token)) {
       return token;
@@ -133,8 +179,12 @@ export class MemoryVault extends BaseVault {
     return null;
   }
 
+  async getPtHashForToken(token: string): Promise<string | null> {
+    const entry = this._store.get(token);
+    return entry?.ptHash ?? null;
+  }
+
   async retrieve(token: string): Promise<string | null> {
-    this._cleanup();
     const entry = this._store.get(token);
     if (!entry) {
       return null;
@@ -204,13 +254,14 @@ export class RedisVault extends BaseVault {
     }
   }
 
-  async store(token: string, ciphertext: string, ttlSeconds: number, ptHash: string | null = null): Promise<void> {
+  async store(token: string, ciphertext: string, ttlSeconds: number, ptHash: string | null = null, metadata: Record<string, string> | null = null): Promise<void> {
     try {
       const pipeline = this._client.pipeline();
-      pipeline.set(`mask:${token}`, ciphertext, 'EX', ttlSeconds);
+      const payload = metadata ? JSON.stringify({ ct: ciphertext, meta: metadata }) : ciphertext;
+      pipeline.set(_vaultKey(token), payload, 'EX', ttlSeconds);
       if (ptHash) {
-        pipeline.set(`mask-rev:${ptHash}`, token, 'EX', ttlSeconds);
-        pipeline.set(`mask-hash:${token}`, ptHash, 'EX', ttlSeconds);
+        pipeline.set(_vaultRevKey(ptHash), token, 'EX', ttlSeconds);
+        pipeline.set(_vaultHashKey(token), ptHash, 'EX', ttlSeconds);
       }
       const results = await pipeline.exec();
       if (results) {
@@ -223,14 +274,22 @@ export class RedisVault extends BaseVault {
     }
   }
 
+  async getPtHashForToken(token: string): Promise<string | null> {
+    try {
+      return await this._client.get(_vaultHashKey(token));
+    } catch {
+      return null;
+    }
+  }
+
   async getTokenByPlaintextHash(ptHash: string): Promise<string | null> {
     try {
-      const token = await this._client.get(`mask-rev:${ptHash}`);
+      const token = await this._client.get(_vaultRevKey(ptHash));
       if (token) {
-        if (await this._client.exists(`mask:${token}`)) {
+        if (await this._client.exists(_vaultKey(token))) {
           return token;
         } else {
-          await this._client.del(`mask-rev:${ptHash}`);
+          await this._client.del(_vaultRevKey(ptHash));
         }
       }
       return null;
@@ -244,7 +303,8 @@ export class RedisVault extends BaseVault {
 
   async retrieve(token: string): Promise<string | null> {
     try {
-      return await this._client.get(`mask:${token}`);
+      const raw = await this._client.get(_vaultKey(token));
+      return raw ? _unwrapPayload(raw) : null;
     } catch (e) {
       if (_getFailStrategy() === 'closed') {
         throw new MaskVaultConnectionError(`Redis read failed: ${e}`);
@@ -255,12 +315,12 @@ export class RedisVault extends BaseVault {
 
   async delete(token: string): Promise<void> {
     try {
-      const ptHash = await this._client.get(`mask-hash:${token}`);
+      const ptHash = await this._client.get(_vaultHashKey(token));
       const pipeline = this._client.pipeline();
-      pipeline.del(`mask:${token}`);
-      pipeline.del(`mask-hash:${token}`);
+      pipeline.del(_vaultKey(token));
+      pipeline.del(_vaultHashKey(token));
       if (ptHash) {
-        pipeline.del(`mask-rev:${ptHash}`);
+        pipeline.del(_vaultRevKey(ptHash));
       }
       await pipeline.exec();
     } catch (e) {
@@ -312,54 +372,38 @@ export class DynamoDBVault extends BaseVault {
     console.info(`DynamoDBVault connected to table ${this._tableName} in ${this._region}`);
   }
 
-  async store(token: string, ciphertext: string, ttlSeconds: number, ptHash: string | null = null): Promise<void> {
+  async store(token: string, ciphertext: string, ttlSeconds: number, ptHash: string | null = null, metadata: Record<string, string> | null = null): Promise<void> {
     const { TransactWriteCommand, PutCommand } = require('@aws-sdk/lib-dynamodb');
     const now = Math.floor(Date.now() / 1000);
     const ttlVal = now + ttlSeconds;
-    const item = {
-      token: `mask:${token}`,
+    const primaryItem: Record<string, any> = {
+      token: _vaultKey(token),
       ciphertext: ciphertext,
       ttl: ttlVal,
-      ptr_hash: ptHash || undefined
     };
+    if (ptHash) primaryItem.ptr_hash = ptHash;
+    if (metadata) primaryItem.meta_json = JSON.stringify(metadata);
 
     if (ptHash) {
       try {
         await this._client.send(new TransactWriteCommand({
           TransactItems: [
+            { Put: { TableName: this._tableName, Item: primaryItem } },
             {
               Put: {
                 TableName: this._tableName,
-                Item: {
-                  token: `mask:${token}`,
-                  ciphertext: ciphertext,
-                  ttl: ttlVal,
-                  ptr_hash: ptHash
-                }
-              }
-            },
-            {
-              Put: {
-                TableName: this._tableName,
-                Item: {
-                  token: `mask-rev:${ptHash}`,
-                  ciphertext: token,
-                  ttl: ttlVal
-                }
+                Item: { token: _vaultRevKey(ptHash), ciphertext: token, ttl: ttlVal }
               }
             }
           ]
         }));
       } catch (e: any) {
         console.error(`DynamoDB transact_write_items failed: ${e}`);
-        // In both strategies, we must raise for DynamoDB failures to prevent data loss
-        // since we didn't perform the atomic write.
         throw new MaskVaultConnectionError(`DynamoDB atomic write failed: ${e}`);
       }
     } else {
-      // Single store (no reverse index)
       try {
-        await this._client.send(new PutCommand({ TableName: this._tableName, Item: item }));
+        await this._client.send(new PutCommand({ TableName: this._tableName, Item: primaryItem }));
       } catch (e: any) {
         throw new MaskVaultConnectionError(`DynamoDB individual write failed: ${e}`);
       }
@@ -372,12 +416,12 @@ export class DynamoDBVault extends BaseVault {
       const now = Math.floor(Date.now() / 1000);
       const resp = await this._client.send(new GetCommand({
         TableName: this._tableName,
-        Key: { token: `mask-rev:${ptHash}` }
+        Key: { token: _vaultRevKey(ptHash) }
       }));
       const item = resp.Item;
       if (!item) return null;
       if (now > (item.ttl || 0)) {
-        await this._client.send(new DeleteCommand({ TableName: this._tableName, Key: { token: `mask-rev:${ptHash}` } }));
+        await this._client.send(new DeleteCommand({ TableName: this._tableName, Key: { token: _vaultRevKey(ptHash) } }));
         return null;
       }
       const token = item.ciphertext;
@@ -394,16 +438,16 @@ export class DynamoDBVault extends BaseVault {
       const now = Math.floor(Date.now() / 1000);
       const resp = await this._client.send(new GetCommand({
         TableName: this._tableName,
-        Key: { token: `mask:${token}` }
+        Key: { token: _vaultKey(token) }
       }));
       const item = resp.Item;
       if (!item) return null;
       if (now > (item.ttl || 0)) {
         const ptHash = item.ptr_hash;
         if (ptHash) {
-          await this._client.send(new DeleteCommand({ TableName: this._tableName, Key: { token: `mask-rev:${ptHash}` } }));
+          await this._client.send(new DeleteCommand({ TableName: this._tableName, Key: { token: _vaultRevKey(ptHash) } }));
         }
-        await this._client.send(new DeleteCommand({ TableName: this._tableName, Key: { token: `mask:${token}` } }));
+        await this._client.send(new DeleteCommand({ TableName: this._tableName, Key: { token: _vaultKey(token) } }));
         return null;
       }
       return item.ciphertext;
@@ -413,18 +457,29 @@ export class DynamoDBVault extends BaseVault {
     }
   }
 
+  async getPtHashForToken(token: string): Promise<string | null> {
+    try {
+      const { GetCommand } = require('@aws-sdk/lib-dynamodb');
+      const resp = await this._client.send(new GetCommand({
+        TableName: this._tableName,
+        Key: { token: _vaultKey(token) }
+      }));
+      return resp.Item?.ptr_hash ?? null;
+    } catch { return null; }
+  }
+
   async delete(token: string): Promise<void> {
     try {
       const { GetCommand, DeleteCommand } = require('@aws-sdk/lib-dynamodb');
       const resp = await this._client.send(new GetCommand({
         TableName: this._tableName,
-        Key: { token: `mask:${token}` }
+        Key: { token: _vaultKey(token) }
       }));
       const item = resp.Item;
       if (item && item.ptr_hash) {
-        await this._client.send(new DeleteCommand({ TableName: this._tableName, Key: { token: `mask-rev:${item.ptr_hash}` } }));
+        await this._client.send(new DeleteCommand({ TableName: this._tableName, Key: { token: _vaultRevKey(item.ptr_hash) } }));
       }
-      await this._client.send(new DeleteCommand({ TableName: this._tableName, Key: { token: `mask:${token}` } }));
+      await this._client.send(new DeleteCommand({ TableName: this._tableName, Key: { token: _vaultKey(token) } }));
     } catch (e: any) {
       if (_getFailStrategy() === 'closed') throw new MaskVaultConnectionError(`DynamoDB delete failed: ${e}`);
     }
@@ -450,21 +505,29 @@ export class MemcachedVault extends BaseVault {
     }
   }
 
-  async store(token: string, ciphertext: string, ttlSeconds: number, ptHash: string | null = null): Promise<void> {
+  async store(token: string, ciphertext: string, ttlSeconds: number, ptHash: string | null = null, metadata: Record<string, string> | null = null): Promise<void> {
     try {
-        await this._client.set(`mask:${token}`, Buffer.from(ciphertext), { expires: ttlSeconds });
+        const payload = metadata ? JSON.stringify({ ct: ciphertext, meta: metadata }) : ciphertext;
+        await this._client.set(_vaultKey(token), Buffer.from(payload), { expires: ttlSeconds });
         if (ptHash) {
-           await this._client.set(`mask-rev:${ptHash}`, Buffer.from(token), { expires: ttlSeconds });
-           await this._client.set(`mask-hash:${token}`, Buffer.from(ptHash), { expires: ttlSeconds });
+           await this._client.set(_vaultRevKey(ptHash), Buffer.from(token), { expires: ttlSeconds });
+           await this._client.set(_vaultHashKey(token), Buffer.from(ptHash), { expires: ttlSeconds });
         }
     } catch (e) {
         throw new MaskVaultConnectionError(`Memcached error: ${e}`);
     }
   }
 
+  async getPtHashForToken(token: string): Promise<string | null> {
+    try {
+      const { value } = await this._client.get(_vaultHashKey(token));
+      return value ? value.toString() : null;
+    } catch { return null; }
+  }
+
   async getTokenByPlaintextHash(ptHash: string): Promise<string | null> {
     try {
-        const { value } = await this._client.get(`mask-rev:${ptHash}`);
+        const { value } = await this._client.get(_vaultRevKey(ptHash));
         if (!value) return null;
         const token = value.toString();
         return (await this.retrieve(token)) !== null ? token : null;
@@ -476,8 +539,9 @@ export class MemcachedVault extends BaseVault {
 
   async retrieve(token: string): Promise<string | null> {
     try {
-        const { value } = await this._client.get(`mask:${token}`);
-        return value ? value.toString() : null;
+        const { value } = await this._client.get(_vaultKey(token));
+        if (!value) return null;
+        return _unwrapPayload(value.toString());
     } catch (e) {
         if (_getFailStrategy() === 'closed') throw new MaskVaultConnectionError(`Memcached read failed: ${e}`);
         return null;
@@ -486,12 +550,12 @@ export class MemcachedVault extends BaseVault {
 
   async delete(token: string): Promise<void> {
     try {
-        const { value } = await this._client.get(`mask-hash:${token}`);
+        const { value } = await this._client.get(_vaultHashKey(token));
         const ptHash = value ? value.toString() : null;
-        await this._client.delete(`mask:${token}`);
-        await this._client.delete(`mask-hash:${token}`);
+        await this._client.delete(_vaultKey(token));
+        await this._client.delete(_vaultHashKey(token));
         if (ptHash) {
-          await this._client.delete(`mask-rev:${ptHash}`);
+          await this._client.delete(_vaultRevKey(ptHash));
         }
     } catch (e) {
         if (_getFailStrategy() === 'closed') throw new MaskVaultConnectionError(`Memcached delete failed: ${e}`);
@@ -539,6 +603,7 @@ export type EncodeOptions = {
     searchBuckets?: ('year' | 'month' | 'day' | 'numeric')[];
     searchBucketSize?: number;
     entityType?: string;
+    metadata?: Record<string, string> | null;
 };
 
 /**
@@ -566,16 +631,26 @@ export async function encode(rawText: string, options: EncodeOptions = {}): Prom
   }
 
   // 2. Generate new token
-  const token = await generateFPEToken(text, options.entityType || 'UNKNOWN');
+  const token = await generateDPToken(text, options.entityType || 'UNKNOWN');
 
   // 3. Encrypt the plaintext before it touches the vault
   const ciphertext = cryptoEngine.encrypt(text);
 
-  // 4. Store with primary reverse lookup hash — always fail-shut to prevent PII leakage
+  // 4. Collision Detection — refuse to overwrite a different plaintext under the same token
+  const existingCiphertext = await vault.retrieve(token);
+  if (existingCiphertext !== null) {
+    const existingHash = await vault.getPtHashForToken(token);
+    if (existingHash && existingHash !== ptHash) {
+      throw new TokenCollisionError(token, existingHash, ptHash);
+    }
+    // Same plaintext re-encoded (hash matches) — safe to proceed/update
+  }
+
+  // 5. Store with primary reverse lookup hash and optional compliance metadata
   const ttl = options.ttl || DEFAULT_TTL;
-  await vault.store(token, ciphertext, ttl, ptHash);
+  await vault.store(token, ciphertext, ttl, ptHash, options.metadata || null);
 
-  // 5. Store additional blind indices if buckets are requested
+  // 6. Store additional blind indices if buckets are requested
   if (options.searchBuckets && options.searchBuckets.length > 0) {
       for (const bType of options.searchBuckets) {
           let bucketVal: string;
@@ -586,7 +661,6 @@ export async function encode(rawText: string, options: EncodeOptions = {}): Prom
           }
           const bHash = await BucketManager.getBucketIndex(bucketVal);
           await vault.store(token, ciphertext, ttl, bHash);
-
       }
   }