@privacyscrubber/sdk 2.3.9 → 2.4.0

package/fastmcp.cjs

@@ -26,6 +26,204 @@ const ZTDS_PATTERNS = {
 
 const TOKEN_PATTERN = /\[([A-Z_]+)_TOKEN_([a-zA-Z0-9_-]+)\]/g;
 
+function escapeRegex(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function buildStreamReplacer(tokenMap = {}, options = {}) {
+  if (!tokenMap || typeof tokenMap !== 'object' || Object.keys(tokenMap).length === 0) {
+    return (text) => text || '';
+  }
+
+  const lookup = new Map();
+  const patterns = [];
+  const sortedKeys = Object.keys(tokenMap).sort((a, b) => b.length - a.length);
+
+  for (const k of sortedKeys) {
+    const val = tokenMap[k];
+    const inner = k.replace(/^\[|<|\{\{|__|\]|>|\}\}|__/g, '');
+    const match = inner.match(/^([A-Za-z_0-9]+?)[-_ ]?(\d+)$/);
+
+    if (match) {
+      const label = escapeRegex(match[1]);
+      const idx = match[2];
+      const patternStr = '(?:\\[\\s*' + label + '[-_\\s]*0*' + idx + '\\s*\\]|<\\s*' + label + '[-_\\s]*0*' + idx + '\\s*>|\\{\\{\\s*' + label + '[-_\\s]*0*' + idx + '\\s*\\}\\}|__\\s*' + label + '[-_\\s]*0*' + idx + '\\s*__|\\b' + label + '[-_\\s]+0*' + idx + '\\b)';
+      patterns.push(patternStr);
+      const normKey = (match[1] + '_' + parseInt(idx, 10)).toUpperCase();
+      lookup.set(normKey, val);
+      lookup.set(k, val);
+      lookup.set(k.toUpperCase(), val);
+    } else {
+      const safe = escapeRegex(k);
+      patterns.push(safe);
+      lookup.set(k, val);
+      lookup.set(k.toUpperCase(), val);
+    }
+  }
+
+  const unifiedRegex = new RegExp('(?:' + patterns.join('|') + ')(\'s|’s|s)?', 'gi');
+
+  return function replaceTokens(text) {
+    if (!text) return '';
+    unifiedRegex.lastIndex = 0;
+    return text.replace(unifiedRegex, (match, suffix) => {
+      const isPossessive = suffix && /('s|’s)$/i.test(suffix);
+      const coreMatch = suffix ? match.slice(0, match.length - suffix.length) : match;
+      let restored = null;
+
+      if (lookup.has(coreMatch)) {
+        restored = lookup.get(coreMatch);
+      } else if (lookup.has(coreMatch.toUpperCase())) {
+        restored = lookup.get(coreMatch.toUpperCase());
+      } else {
+        const inner = coreMatch.replace(/^\[|<|\{\{|__|\]|>|\}\}|__/g, '').trim();
+        const m = inner.match(/^([A-Za-z_0-9]+?)[-_ ]?(\d+)$/);
+        if (m) {
+          const normKey = (m[1] + '_' + parseInt(m[2], 10)).toUpperCase();
+          if (lookup.has(normKey)) {
+            restored = lookup.get(normKey);
+          }
+        }
+      }
+
+      if (restored !== null) {
+        return restored + (isPossessive ? "'s" : '');
+      }
+
+      if (lookup.has(match)) {
+        return lookup.get(match);
+      }
+
+      return match;
+    });
+  };
+}
+
+class StreamingDetokenizer {
+  constructor(tokenMap = {}, options = {}) {
+    this.tokenMap = tokenMap && typeof tokenMap === 'object' ? tokenMap : {};
+    this.options = options || {};
+    this.replacer = buildStreamReplacer(this.tokenMap, this.options);
+    this.pendingBuffer = '';
+    this.hasTokens = Object.keys(this.tokenMap).length > 0;
+    this._lastTokenCount = Object.keys(this.tokenMap).length;
+    this.decoder = new TextDecoder('utf-8', { fatal: false });
+  }
+
+  get buffer() {
+    return this.pendingBuffer;
+  }
+
+  set buffer(val) {
+    this.pendingBuffer = typeof val === 'string' ? val : '';
+  }
+
+  updateTokenMap(newMap) {
+    if (newMap && typeof newMap === 'object') {
+      this.tokenMap = newMap;
+      this.replacer = buildStreamReplacer(this.tokenMap, this.options);
+      this.hasTokens = Object.keys(this.tokenMap).length > 0;
+      this._lastTokenCount = Object.keys(this.tokenMap).length;
+    }
+  }
+
+  reset() {
+    this.pendingBuffer = '';
+  }
+
+  feed(chunk) {
+    if (chunk === undefined || chunk === null) return '';
+    let text = '';
+    if (typeof chunk === 'string') {
+      text = chunk;
+    } else if (chunk instanceof Uint8Array || (typeof Buffer !== 'undefined' && Buffer.isBuffer(chunk))) {
+      text = this.decoder.decode(chunk, { stream: true });
+    } else {
+      text = String(chunk);
+    }
+
+    if (!text) return '';
+
+    const currentTokenCount = Object.keys(this.tokenMap).length;
+    if (currentTokenCount !== this._lastTokenCount) {
+      this.replacer = buildStreamReplacer(this.tokenMap, this.options);
+      this.hasTokens = currentTokenCount > 0;
+      this._lastTokenCount = currentTokenCount;
+    }
+
+    if (!this.hasTokens) return text;
+
+    this.pendingBuffer += text;
+
+    const tailLen = 160;
+    const tail = this.pendingBuffer.length > tailLen ? this.pendingBuffer.slice(-tailLen) : this.pendingBuffer;
+    const offset = this.pendingBuffer.length > tailLen ? this.pendingBuffer.length - tailLen : 0;
+    const m = tail.match(/(?:\[|<|\{\{?|__?)[A-Za-z0-9_\-А-Яа-яЁё ]{0,128}$/);
+
+    if (m) {
+      const candidate = m[0];
+      const isClosedUnderscoreToken = candidate.length >= 4 && candidate.startsWith('__') && candidate.endsWith('__');
+      if (isClosedUnderscoreToken && this.replacer(candidate) !== candidate) {
+        const toEmit = this.pendingBuffer;
+        this.pendingBuffer = '';
+        return this.replacer(toEmit);
+      }
+      const splitIdx = offset + m.index;
+      const safeToEmit = this.pendingBuffer.slice(0, splitIdx);
+      this.pendingBuffer = this.pendingBuffer.slice(splitIdx);
+      return safeToEmit ? this.replacer(safeToEmit) : '';
+    }
+
+    const toEmit = this.pendingBuffer;
+    this.pendingBuffer = '';
+    return this.replacer(toEmit);
+  }
+
+  process(chunk) {
+    return this.feed(chunk);
+  }
+
+  flush() {
+    let text = '';
+    try {
+      text = this.decoder.decode(new Uint8Array(), { stream: false });
+    } catch (_) {}
+
+    if (text) {
+      this.pendingBuffer += text;
+    }
+
+    if (!this.pendingBuffer) return '';
+    const remaining = this.pendingBuffer;
+    this.pendingBuffer = '';
+    return this.replacer(remaining);
+  }
+
+  async* wrapStream(asyncIterable) {
+    for await (const chunk of asyncIterable) {
+      if (chunk === null || chunk === undefined) continue;
+      let text = '';
+      if (typeof chunk === 'string') {
+        text = chunk;
+      } else if (typeof chunk?.text === 'string') {
+        text = chunk.text;
+      } else if (typeof chunk?.content === 'string') {
+        text = chunk.content;
+      } else if (chunk instanceof Uint8Array || (typeof Buffer !== 'undefined' && Buffer.isBuffer(chunk))) {
+        text = chunk;
+      } else {
+        text = String(chunk);
+      }
+
+      const emitted = this.feed(text);
+      if (emitted) yield emitted;
+    }
+
+    const finalChunk = this.flush();
+    if (finalChunk) yield finalChunk;
+  }
+}
+
 class ZTDSFastMCPGuard {
   constructor(options = {}) {
     this.enabledEntities = options.enabledEntities || Object.keys(ZTDS_PATTERNS);
@@ -36,6 +234,7 @@ class ZTDSFastMCPGuard {
     this.maxSessionTokens = options.maxSessionTokens || 5000;
     this._sessionMaps = new Map();
     this._entityMaps = new Map();
+    this._streamDetokenizers = new Map();
   }
 
   _generateToken(entityType, tokenMap) {
@@ -157,6 +356,65 @@ class ZTDSFastMCPGuard {
     return val;
   }
 
+  getStreamDetokenizer(sessionId) {
+    if (!this._streamDetokenizers.has(sessionId)) {
+      const mapProxy = this._sessionMaps.get(sessionId) || new Map();
+      const objMap = {};
+      for (const [k, v] of mapProxy.entries()) {
+        objMap[k] = v;
+      }
+      this._streamDetokenizers.set(sessionId, new StreamingDetokenizer(objMap));
+    } else {
+      const mapProxy = this._sessionMaps.get(sessionId);
+      if (mapProxy) {
+        const objMap = {};
+        for (const [k, v] of mapProxy.entries()) {
+          objMap[k] = v;
+        }
+        this._streamDetokenizers.get(sessionId).updateTokenMap(objMap);
+      }
+    }
+    return this._streamDetokenizers.get(sessionId);
+  }
+
+  feedStream(chunk, sessionId) {
+    return this.getStreamDetokenizer(sessionId).feed(chunk);
+  }
+
+  flushStream(sessionId) {
+    const detok = this._streamDetokenizers.get(sessionId);
+    return detok ? detok.flush() : "";
+  }
+
+  async *restoreStream(asyncIterable, sessionId) {
+    const detok = this.getStreamDetokenizer(sessionId);
+    try {
+      for await (const chunk of asyncIterable) {
+        if (chunk === null || chunk === undefined) continue;
+        let text = "";
+        if (typeof chunk === "string") {
+          text = chunk;
+        } else if (typeof chunk?.text === "string") {
+          text = chunk.text;
+        } else if (typeof chunk?.content === "string") {
+          text = chunk.content;
+        } else if (chunk instanceof Uint8Array || (typeof Buffer !== "undefined" && Buffer.isBuffer(chunk))) {
+          text = chunk;
+        } else {
+          text = String(chunk);
+        }
+        const emitted = detok.feed(text);
+        if (emitted) yield emitted;
+      }
+      const finalChunk = detok.flush();
+      if (finalChunk) yield finalChunk;
+    } finally {
+      if (this.autoZeroize) {
+        this.zeroizeSession(sessionId);
+      }
+    }
+  }
+
   wrapTool(toolName, handler) {
     return async (args, context = {}) => {
       const sessionId =
@@ -169,6 +427,7 @@ class ZTDSFastMCPGuard {
         unmask: (val) => this.restoreObject(val, sessionId),
       };
 
+      let isStreaming = false;
       try {
         let processedArgs = args;
         if (this.sanitizeInputs && args !== undefined && args !== null) {
@@ -177,6 +436,11 @@ class ZTDSFastMCPGuard {
 
         const rawResult = await handler(processedArgs, enrichedContext);
 
+        if (rawResult && typeof rawResult[Symbol.asyncIterator] === "function") {
+          isStreaming = true;
+          return this.restoreStream(rawResult, sessionId);
+        }
+
         let finalResult = rawResult;
         if (this.sanitizeOutputs && rawResult !== undefined && rawResult !== null) {
           if (typeof rawResult === "string") {
@@ -234,7 +498,7 @@ class ZTDSFastMCPGuard {
           _ztds: ztdsMeta,
         };
       } finally {
-        if (this.autoZeroize) {
+        if (!isStreaming && this.autoZeroize) {
           this.zeroizeSession(sessionId);
         }
       }
@@ -250,6 +514,10 @@ class ZTDSFastMCPGuard {
       this._entityMaps.get(sessionId).clear();
       this._entityMaps.delete(sessionId);
     }
+    if (this._streamDetokenizers.has(sessionId)) {
+      this._streamDetokenizers.get(sessionId).reset();
+      this._streamDetokenizers.delete(sessionId);
+    }
   }
 }
 
@@ -285,6 +553,8 @@ module.exports = {
   ZTDSFastMCPGuard,
   ZTDS_PATTERNS,
   TOKEN_PATTERN,
+  StreamingDetokenizer,
+  createStreamTokenRestorer: (tokenMap, options) => new StreamingDetokenizer(tokenMap, options),
   createZTDSGuard,
   ztdsGuardTool,
   wrapExecute,

package/fastmcp.d.ts

@@ -26,6 +26,19 @@ export interface ZTDSContext {
 export declare const ZTDS_PATTERNS: Record<string, RegExp>;
 export declare const TOKEN_PATTERN: RegExp;
 
+export declare class StreamingDetokenizer {
+  constructor(tokenMap?: Record<string, string>, options?: object);
+  buffer: string;
+  feed(chunk: string | Uint8Array | Buffer): string;
+  process(chunk: string | Uint8Array | Buffer): string;
+  flush(): string;
+  reset(): void;
+  updateTokenMap(newMap: Record<string, string>): void;
+  wrapStream<T = any>(asyncIterable: AsyncIterable<T>): AsyncGenerator<T>;
+}
+
+export declare function createStreamTokenRestorer(tokenMap: Record<string, string>, options?: object): StreamingDetokenizer;
+
 export declare class ZTDSFastMCPGuard {
   constructor(options?: ZTDSGuardOptions);
   enabledEntities: string[];
@@ -39,6 +52,11 @@ export declare class ZTDSFastMCPGuard {
   sanitizeObject(val: unknown, sessionId: string): unknown;
   restoreObject(val: unknown, sessionId: string): unknown;
 
+  getStreamDetokenizer(sessionId: string): StreamingDetokenizer;
+  feedStream(chunk: string, sessionId: string): string;
+  flushStream(sessionId: string): string;
+  restoreStream<T = any>(asyncIterable: AsyncIterable<T>, sessionId: string): AsyncGenerator<T>;
+
   wrapTool<TArgs = unknown, TResult = unknown>(
     toolName: string,
     handler: (args: TArgs, context: ZTDSContext) => Promise<TResult>

package/fastmcp.js

@@ -11,6 +11,8 @@ const fastmcpGuard = require('./fastmcp.cjs');
 export const ZTDSFastMCPGuard = fastmcpGuard.ZTDSFastMCPGuard;
 export const ZTDS_PATTERNS = fastmcpGuard.ZTDS_PATTERNS;
 export const TOKEN_PATTERN = fastmcpGuard.TOKEN_PATTERN;
+export const StreamingDetokenizer = fastmcpGuard.StreamingDetokenizer;
+export const createStreamTokenRestorer = fastmcpGuard.createStreamTokenRestorer;
 export const createZTDSGuard = fastmcpGuard.createZTDSGuard;
 export const ztdsGuardTool = fastmcpGuard.ztdsGuardTool;
 export const wrapExecute = fastmcpGuard.wrapExecute;
@@ -20,6 +22,8 @@ export default {
   ZTDSFastMCPGuard,
   ZTDS_PATTERNS,
   TOKEN_PATTERN,
+  StreamingDetokenizer,
+  createStreamTokenRestorer,
   createZTDSGuard,
   ztdsGuardTool,
   wrapExecute,

package/index.cjs

@@ -36,16 +36,33 @@ if (scrubberPkg && typeof scrubberPkg.init === 'function') {
  * Built-in patterns for DevOps & Cloud Secrets detection.
  */
 const DEVOPS_SECRETS_DETECTOR = [
-    { name: 'AWS Credentials', regex: /\b(?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA)[A-Z0-9]{16}\b/g, label: 'AWS_KEY' },
+    { name: 'AWS Credentials', regex: /\b(?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA)[A-Z0-9]{12,28}\b/g, label: 'AWS_KEY' },
+    { name: 'Google Cloud / Gemini API Key', regex: /\bAIza[0-9A-Za-z_-]{35}\b/g, label: 'API_TOKEN' },
+    { name: 'Anthropic API Key', regex: /\bsk-ant-(?:api\d{2}|admin\d{2}|sid\d{2})-[a-zA-Z0-9_-]{16,128}\b/g, label: 'API_TOKEN' },
+    { name: 'HuggingFace / Perplexity API Key', regex: /\b(?:hf_[a-zA-Z0-9]{20,64}|pplx-[a-zA-Z0-9]{24,64})\b/g, label: 'API_TOKEN' },
+    { name: 'SendGrid API Key', regex: /\bSG\.[a-zA-Z0-9_-]{16,32}\.[a-zA-Z0-9_-]{32,64}\b/g, label: 'API_TOKEN' },
+    { name: 'Postman API Key', regex: /\bPMAK-[a-fA-F0-9]{24}-[a-fA-F0-9]{34}\b/g, label: 'API_TOKEN' },
+    { name: 'GitHub Fine-Grained PAT', regex: /\bgithub_pat_[a-zA-Z0-9]{20,30}_[a-zA-Z0-9]{40,80}\b/g, label: 'API_TOKEN' },
+    { name: 'Slack Enterprise / Refresh Token', regex: /\bxoxe\.xox[baprs]-\d-[a-zA-Z0-9-]{10,}\b|\bxox[eapbrs]-[a-zA-Z0-9-]{10,}\b/g, label: 'API_TOKEN' },
+    { name: 'Telegram Bot API Token', regex: /\b\d{8,10}:AA[A-Za-z0-9_-]{33}\b/g, label: 'API_TOKEN' },
+    { name: 'Age Encryption Secret Key', regex: /\bAGE-SECRET-KEY-1[A-Z0-9]{58}\b/g, label: 'SECRET' },
+    { name: 'Modern Cloud & AI Platform Key', regex: /\b(?:sbp_[a-fA-F0-9]{40}|dop_v1_[a-fA-F0-9]{64}|doo_v1_[a-fA-F0-9]{64}|dp\.(?:st|pt|ct|sa)\.[a-zA-Z0-9_-]{20,}|lin_api_[a-zA-Z0-9]{32,64}|ntn_[a-zA-Z0-9]{32,64}|secret_[a-zA-Z0-9]{32,64}|gsk_[a-zA-Z0-9]{32,64}|r8_[a-zA-Z0-9]{24,64}|tvly-[a-zA-Z0-9_-]{20,64}|fc-[a-fA-F0-9]{32}|dapi[a-fA-F0-9]{32}(?:-\d)?|re_[a-zA-Z0-9_]{20,64}|napi_[a-zA-Z0-9]{32,64}|[a-zA-Z0-9]{14}\.atlasv1\.[a-zA-Z0-9]{50,80}|pypi-[A-Za-z0-9_-]{16,128}|whsec_[A-Za-z0-9+/=_-]{24,128}|shpat_[a-fA-F0-9]{32}|shpss_[a-fA-F0-9]{32}|shpca_[a-fA-F0-9]{32}|shppa_[a-fA-F0-9]{32}|hvs\.[A-Za-z0-9_-]{24,128}|hvb\.[A-Za-z0-9_-]{24,128}|glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8}|glc_[A-Za-z0-9+/=_-]{20,128}|pscale_tkn_[A-Za-z0-9_.-]{20,64}|pscale_pw_[A-Za-z0-9_.-]{20,64}|pat[A-Za-z0-9]{14}\.[a-fA-F0-9]{64}|pat-(?:na1|eu1)-[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}|sq0atp-[A-Za-z0-9_-]{22,64}|sq0csp-[A-Za-z0-9_-]{32,64}|NRAK-[A-Z0-9]{27}|v1\.0-[a-fA-F0-9]{24}-[a-fA-F0-9]{16,160}|ATATT3xFfGF0[A-Za-z0-9+/=_-]{24,128}|tskey-(?:auth|api|client)-[A-Za-z0-9_-]{20,96}|sntry[su]_[A-Za-z0-9+/=_-]{32,160}|(?:pk|sk)\.eyJ1[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{16,}|key-[a-f0-9]{32}|[a-f0-9]{32}-us\d{1,2})\b/g, label: 'API_TOKEN' },
+    { name: 'Twilio Account SID / API Key', regex: /\b(?:AC|SK)[a-fA-F0-9]{32}\b/g, label: 'API_TOKEN' },
+    { name: 'Bcrypt / Argon2 Password Hash', regex: /(?<![A-Za-z0-9])(?:\$2[aby]\$\d{2}\$[./A-Za-z0-9=]{20,60}|\$argon2(?:id|i|d)\$[^\s"']{15,120})/g, label: 'SECRET' },
     { name: 'JSON Web Token (JWT)', regex: /\beyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\b/g, label: 'JWT_TOKEN' },
-    { name: 'API Token/Key (GitHub/Slack/NPM)', regex: /\b(?:ghp|gho|ghu|ghs|ghr|glpat|npm|xox[baprs])[-_][A-Za-z0-9_]{10,}\b/g, label: 'API_TOKEN' },
-    { name: 'Stripe API Key', regex: /\b(?:[rs]k)_(?:test|live)_[a-zA-Z0-9]{24,}\b/g, label: 'STRIPE_KEY' },
+    { name: 'API Token/Key (GitHub/Slack/NPM)', regex: /\b(?:ghp|gho|ghu|ghs|ghr|glpat|npm|xox[baprs])[-_][A-Za-z0-9_-]{10,}\b/g, label: 'API_TOKEN' },
+    { name: 'Stripe API Key', regex: /\b(?:[rs]k)_(?:test|live)_[a-zA-Z0-9]{10,}\b/g, label: 'STRIPE_KEY' },
     { name: 'OpenAI Project API Key', regex: /\b(?:sk|pk)-(?:proj-)?[a-zA-Z0-9_-]{16,}\b/gi, label: 'OPENAI_KEY' },
+    { name: 'Slack / Discord Incoming Webhook URL', regex: /\bhttps:\/\/(?:hooks\.slack\.com\/services\/T[A-Z0-9_]+\/B[A-Z0-9_]+\/[A-Za-z0-9_-]+|(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d{17,20}\/[A-Za-z0-9_-]{60,70})\b/gi, label: 'SECRET' },
+    { name: 'Azure Connection String / SAS Token', regex: /(?:DefaultEndpointsProtocol=https?;AccountName=[^;\s"']+;AccountKey=[A-Za-z0-9+/=]{20,}(?:;EndpointSuffix=[^;\s"']+)?|Endpoint=sb:\/\/[^;\s"']+;SharedAccessKeyName=[^;\s"']+;SharedAccessKey=[A-Za-z0-9+/=]{16,}|\bSharedAccessSignature=sr=[^\s"']+&sig=[A-Za-z0-9%/+=]{16,}|\bAccountKey=[A-Za-z0-9+/=]{40,88}==)/gi, label: 'SECRET' },
     { name: 'Database Connection URI', regex: /\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp|mssql):\/\/[^\s"']+/gi, label: 'DB_URI' },
-    { name: 'Embedded Credentials URI', regex: /\b(?:https?|s?ftp):\/\/[a-zA-Z0-9_.%+-]+:[^\s/@:"']+@[a-zA-Z0-9_.-]+(?::\d+)?(?:\/[^\s"']*)?/gi, label: 'SECRET' },
+    { name: 'Embedded Credentials URI', regex: /\b(?:https?|hxxps?|s?ftp|fxp|wss?):\/\/[a-zA-Z0-9_.%+-]+:[^\s/@:"']+@[a-zA-Z0-9_.-]+(?:\[\.\][a-zA-Z0-9_.-]+|\(\.\)[a-zA-Z0-9_.-]+)*(?::\d+)?(?:\/[^\s"']*)?/gi, label: 'SECRET' },
     { name: 'URL Query Parameter Secret', regex: /(?<=[?&](?:api_key|apikey|access_token|client_secret|auth_token|token|secret|password|private_key)=)[^\s"'\x26#)>]{8,}/gi, label: 'SECRET' },
+    { name: 'CLI Credential Flag / Curl User Auth', regex: /(?:\bcurl\b[^\r\n]*?\s(?:-u|--user)\s+["']?[A-Za-z0-9_.+-]+:[^\s"']+|\b(?:--password|--api-key|--apikey|--access-token|--auth-token|--client-secret|--secret-key|--private-key)(?:=|\s+)["']?[^\s"']{6,}["']?)/gi, label: 'SECRET' },
+    { name: 'HTTP API Key Header', regex: /\b(?:X-API-Key|X-Amz-Security-Token|X-Functions-Key|X-Shopify-Access-Token|X-Auth-Token|Api-Key)\s*:\s*["']?[A-Za-z0-9_.\-~+/=]{12,}["']?/gi, label: 'SECRET' },
+    { name: 'Docker Registry Auth / K8s Secret Base64', regex: /(?<=["']?(?:auth|identitytoken|registrytoken|\.dockerconfigjson)["']?\s*:\s*["'])[A-Za-z0-9+/=]{16,}(?=["'])/gi, label: 'SECRET' },
     { name: 'Bearer / Auth Header Token', regex: /(?<=\b(?:Bearer|Token)\s+)[a-zA-Z0-9_\-\.~+/=]{16,}\b/gi, label: 'SECRET' },
-    { name: 'Database/API Secret', regex: /\b(DB|POSTGRES|REDIS|MYSQL|AWS|SECRET|PASSWORD|TOKEN|API|KEY)[A-Z0-9_]*\s*[:=]\s*[^ \t\r\n"']{8,}\b/gi, label: 'SECRET' },
+    { name: 'Database/API Secret', regex: /\b(DB|POSTGRES|REDIS|MYSQL|AWS|SECRET|PASSWORD|TOKEN|API|KEY)[A-Z0-9_]*\s*[:=]\s*[^ \t\r\n"']{8,}/gi, label: 'SECRET' },
     { name: 'Private Cryptographic Key (PEM Block)', regex: /-----BEGIN (?:RSA |EC |PGP |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----[\s\S]{10,8192}?-----END (?:RSA |EC |PGP |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----/g, label: 'SECRET' },
     { name: 'Private Cryptographic Key (Header)', regex: /-----BEGIN (?:RSA |EC |PGP |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----/g, label: 'SECRET' }
 ];
@@ -141,8 +158,8 @@ function buildAuditTelemetry(currentTokenMap = {}) {
         frameworksSet.add('CCPA/CPRA');
     }
     if (types.includes('ID') || types.includes('SSN') || types.includes('PASSPORT') || types.includes('NATIONAL_ID')) {
-        frameworksSet.add('SOC 2 Type II');
-        frameworksSet.add('ISO 27001 (A.8.11)');
+        frameworksSet.add('SOC 2 controls');
+        frameworksSet.add('ISO 27001 A.8.11 controls');
     }
     if (types.some(t => ['CREDIT_CARD', 'BANK', 'IBAN', 'FINANCIAL', 'CARD', 'STRIPE_KEY', 'TAX_ID', 'EIN', 'W2', '1099', 'SALARY', 'WAGE'].includes(t))) {
         frameworksSet.add('PCI DSS v4.0');
@@ -1466,13 +1483,13 @@ function buildStreamReplacer(tokenMap = {}, options = {}) {
 
     for (const k of sortedKeys) {
         const val = tokenMap[k];
-        const inner = k.replace(/^\[|<|\{\{|__|\]|>|\}\}|__/g, '');
+        const inner = k.replace(/^\[|<|\{\{?|__|\]|>|\}\}?|__/g, '');
         const match = inner.match(/^([A-Za-z_0-9]+?)[-_ ]?(\d+)$/);
 
         if (match) {
-            const label = escapeRegex(match[1]);
+            const label = escapeRegex(match[1]).replace(/[_-]/g, '\\\\?[_-]');
             const idx = match[2];
-            const patternStr = '(?:\\[\\s*' + label + '[-_\\s]*0*' + idx + '\\s*\\]|<\\s*' + label + '[-_\\s]*0*' + idx + '\\s*>|\\{\\{\\s*' + label + '[-_\\s]*0*' + idx + '\\s*\\}\\}|__\\s*' + label + '[-_\\s]*0*' + idx + '\\s*__|\\b' + label + '[-_\\s]+0*' + idx + '\\b)';
+            const patternStr = '(?:(?:\\[|\\\\u005[Bb]|%5[Bb]|&(?:amp;)?#0*91;|&(?:amp;)?#[xX]0*5[Bb];|&(?:amp;)?lbrack;|【|［|〔|「|『)\\s*[*`~]*' + label + '[-_\\\\\\s]*0*' + idx + '[*`~]*[.,;:!?]?\\s*(?:\\]|\\\\u005[Dd]|%5[Dd]|&(?:amp;)?#0*93;|&(?:amp;)?#[xX]0*5[Dd];|&(?:amp;)?rbrack;|】|］|〕|」|』)|<\\s*' + label + '[-_\\\\\\s]*0*' + idx + '[.,;:!?]?\\s*>|\\{\\{?\\s*' + label + '[-_\\\\\\s]*0*' + idx + '[.,;:!?]?\\s*\\}\\}?|__\\s*' + label + '[-_\\\\\\s]*0*' + idx + '\\s*__|\\b' + label + '[-_\\s]+0*' + idx + '\\b)';
             patterns.push(patternStr);
             const normKey = (match[1] + '_' + parseInt(idx, 10)).toUpperCase();
             lookup.set(normKey, val);
@@ -1501,10 +1518,14 @@ function buildStreamReplacer(tokenMap = {}, options = {}) {
             } else if (lookup.has(coreMatch.toUpperCase())) {
                 restored = lookup.get(coreMatch.toUpperCase());
             } else {
-                const inner = coreMatch.replace(/^\[|<|\{\{|__|\]|>|\}\}|__/g, '').trim();
-                const m = inner.match(/^([A-Za-z_0-9]+?)[-_ ]?(\d+)$/);
+                const inner = coreMatch
+                    .replace(/^(?:\[|\\u005[Bb]|%5[Bb]|&(?:amp;)?#0*91;|&(?:amp;)?#[xX]0*5[Bb];|&(?:amp;)?lbrack;|【|［|〔|「|『|<|\{\{?|__)|(?:\]|\\u005[Dd]|%5[Dd]|&(?:amp;)?#0*93;|&(?:amp;)?#[xX]0*5[Dd];|&(?:amp;)?rbrack;|】|］|〕|」|』|>|\}\}?|__)$/gi, '')
+                    .replace(/^[*`~\s]+|[.,;:!?*`~\s]+$/g, '')
+                    .trim();
+                const m = inner.match(/^([A-Za-z_0-9]+?)[-_\\\s]*(\d+)$/);
                 if (m) {
-                    const normKey = (m[1] + '_' + parseInt(m[2], 10)).toUpperCase();
+                    const cleanLabel = m[1].replace(/\\([_-])/g, '$1');
+                    const normKey = (cleanLabel + '_' + parseInt(m[2], 10)).toUpperCase();
                     if (lookup.has(normKey)) {
                         restored = lookup.get(normKey);
                     }
@@ -1610,18 +1631,17 @@ class StreamingDetokenizer {
         this.pendingBuffer += text;
 
         // Candidate unclosed token at tail of buffer (inspect tail window up to 160 chars)
-        // Matches incomplete tokens starting with [, <, {{, or __, plus single delimiter prefixes { and _
+        // Matches incomplete tokens starting with [, 【, ［, <, {, __, %5B, \u005b, or HTML entity &#91; / &lbrack;
         const tailLen = 160;
         const tail = this.pendingBuffer.length > tailLen ? this.pendingBuffer.slice(-tailLen) : this.pendingBuffer;
         const offset = this.pendingBuffer.length > tailLen ? this.pendingBuffer.length - tailLen : 0;
-        const m = tail.match(/(?:\[|<|\{\{?|__?)[A-Za-z0-9_\-А-Яа-яЁё ]{0,128}$/);
+        const m = tail.match(/(?:(?:\[|【|［|〔|「|『|<|\{\{?|__?|%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;))[A-Za-z0-9_\-А-Яа-яЁё *`~.\\]{0,128}(?:%5?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?3?|[xX]0*5?[Dd]?)?|r(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)?|%[0-9A-Fa-f]?|\\(?:u(?:0(?:0(?:5)?)?)?)?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?1?|[xX]0*5?)?|l(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)$/);
 
         if (m) {
             const candidate = m[0];
-            // Only closed double-underscore tokens (e.g. __NAME_1__) could match m since '_' is in character class.
-            // Bracketed tokens ([, <, {{) cannot end with their closing delimiter here and must be buffered.
-            const isClosedUnderscoreToken = candidate.length >= 4 && candidate.startsWith('__') && candidate.endsWith('__');
-            if (isClosedUnderscoreToken && this.replacer(candidate) !== candidate) {
+            // Closed double-underscore (__NAME_1__) or JSON Unicode (\u005bNAME_1\u005d) tokens match character class and can be emitted immediately.
+            const isClosedUnderscoreOrUnicode = (candidate.length >= 4 && candidate.startsWith('__') && candidate.endsWith('__')) || /\\u005[Dd]/.test(candidate);
+            if (isClosedUnderscoreOrUnicode && this.replacer(candidate) !== candidate) {
                 const toEmit = this.pendingBuffer;
                 this.pendingBuffer = '';
                 return this.replacer(toEmit);
@@ -1843,13 +1863,8 @@ function withPrivacyScrubber(handler, options = {}) {
             tokenMap = res.tokenMap;
             telemetry = res.telemetry;
         } else if (rawBody && typeof rawBody === 'object') {
-            const serialized = JSON.stringify(rawBody);
-            const res = sanitize(serialized, options);
-            try {
-                scrubbedBody = JSON.parse(res.scrubbedText);
-            } catch (_) {
-                scrubbedBody = rawBody;
-            }
+            const res = sanitizeObject(rawBody, options);
+            scrubbedBody = res.sanitized;
             tokenMap = res.tokenMap;
             telemetry = res.telemetry;
         }
@@ -1860,7 +1875,8 @@ function withPrivacyScrubber(handler, options = {}) {
             scrubbedBody,
             tokenMap,
             telemetry,
-            restore: (text) => restore(text, tokenMap),
+            restore: (text) => (text && typeof text === 'object') ? restoreObject(text, tokenMap) : restore(text, tokenMap),
+            restoreObject: (obj) => restoreObject(obj, tokenMap),
             createStreamRestorer: () => createStreamTokenRestorer(tokenMap)
         };
 
@@ -1878,15 +1894,13 @@ function withPrivacyScrubber(handler, options = {}) {
 function createExpressMiddleware(options = {}) {
     return function privacyScrubberExpress(req, res, next) {
         if (req && req.body && typeof req.body === 'object') {
-            const serialized = JSON.stringify(req.body);
-            const { scrubbedText, tokenMap, telemetry } = sanitize(serialized, options);
+            const { sanitized, tokenMap, telemetry } = sanitizeObject(req.body, options);
             req.rawBody = req.body;
-            try {
-                req.body = JSON.parse(scrubbedText);
-            } catch (_) {}
+            req.body = sanitized;
             req.psTokenMap = tokenMap;
             req.psTelemetry = telemetry;
-            req.restore = (aiResponse) => restore(aiResponse, tokenMap);
+            req.restore = (aiResponse) => (aiResponse && typeof aiResponse === 'object') ? restoreObject(aiResponse, tokenMap) : restore(aiResponse, tokenMap);
+            req.restoreObject = (obj) => restoreObject(obj, tokenMap);
             req.createStreamRestorer = () => createStreamTokenRestorer(tokenMap);
         }
         if (typeof next === 'function') next();
@@ -1903,7 +1917,7 @@ function formatCefEvent(eventData = {}) {
     const {
         vendor = 'PrivacyScrubber',
         product = 'ZTDS',
-        version = '2.2.5',
+        version = '2.4.0',
         eventClassId = 'PII_REDACTION',
         eventName = 'Zero-Trust PII Sanitization Executed',
         riskLevel = 'LOW EXPOSURE',
@@ -1972,7 +1986,7 @@ function formatSyslogEvent(eventData = {}) {
 function formatJsonlEvent(eventData = {}) {
     const {
         timestamp = new Date().toISOString(),
-        version = '2.2.5',
+        version = '2.4.0',
         protocol = 'ZTDS',
         event = 'sanitization',
         riskLevel = 'LOW EXPOSURE',
@@ -2508,7 +2522,46 @@ function createGuardedTools(engineOrOptions = {}, toolConfig = {}) {
                     throw new Error(`[guard_read_file] Path is a directory, not a file: ${resolved}`);
                 }
 
-                const raw = fs.readFileSync(resolved, 'utf8');
+                const rawBuf = fs.readFileSync(resolved);
+                const checkLen = Math.min(rawBuf.length, 8000);
+                for (let i = 0; i < checkLen; i++) {
+                    if (rawBuf[i] === 0) {
+                        throw new Error(`[guard_read_file] Binary file detected (${path.basename(resolved)}). Only text, code, and MIME files are supported.`);
+                    }
+                }
+                let raw = rawBuf.toString('utf8');
+                const ext = path.extname(resolved).toLowerCase();
+                if (ext === '.eml') {
+                    const decodeQpUtf8 = (str) =>
+                        str.replace(/=\r?\n/g, '').replace(/(?:=[0-9A-Fa-f]{2})+/g, (seq) => {
+                            try {
+                                const bytes = Buffer.alloc(seq.length / 3);
+                                for (let i = 0; i < bytes.length; i++) {
+                                    bytes[i] = parseInt(seq.slice(i * 3 + 1, i * 3 + 3), 16);
+                                }
+                                return bytes.toString('utf8');
+                            } catch (_) { return seq; }
+                        });
+                    raw = raw.replace(/=\?([^?]+)\?([BQbq])\?([^?]+)\?=/g, (full, _cs, enc, payload) => {
+                        try {
+                            return enc.toUpperCase() === 'B'
+                                ? Buffer.from(payload.replace(/\s+/g, ''), 'base64').toString('utf8')
+                                : decodeQpUtf8(payload.replace(/_/g, ' '));
+                        } catch (_) { return full; }
+                    });
+                    raw = decodeQpUtf8(raw);
+                } else if (ext === '.rtf') {
+                    raw = raw
+                        .replace(/\\par\b|\\line\b/gi, '\n')
+                        .replace(/\\'([0-9a-fA-F]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
+                        .replace(/\\u(-?\d+)\??/g, (_, code) => {
+                            const n = parseInt(code, 10);
+                            return String.fromCharCode(n < 0 ? n + 65536 : n);
+                        })
+                        .replace(/\\[a-zA-Z]+(?:-?\d+)?[ ]?/g, ' ')
+                        .replace(/[{}]/g, ' ')
+                        .replace(/[ \t]+/g, ' ');
+                }
                 const lines = raw.split(/\r?\n/);
                 const capped = lines.slice(0, maxLines).join('\n');
                 const wasTruncated = lines.length > maxLines;