@privacyscrubber/sdk 2.3.9 → 2.4.1

package/fastmcp.cjs

@@ -1,7 +1,7 @@
 /**
  * @privacyscrubber/fastmcp-guard
  * Zero-Trust Data Sanitization (ZTDS) In-Memory Middleware for FastMCP
- * Conforms to IETF Standards Track: draft-sibiryakov-ztds-protocol-02
+ * Conforms to IETF Standards Track: draft-sibiryakov-ztds-protocol-01
  *
  * Core Protocol Invariants:
  * 1. Zero External Egress Prior to Sanitization
@@ -14,7 +14,7 @@
 const crypto = require("node:crypto");
 
 const ZTDS_PATTERNS = {
-  API_SECRET: /\b(?:sk-(?:proj-)?[A-Za-z0-9_-]{20,}|(?:ghp|gho|ghu|ghs|ghr|github_pat_)[A-Za-z0-9_]{20,}|eyJ[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,})\b/g,
+  API_SECRET: /\b(?:sk-ant-(?:api\d{2}|admin\d{2}|sid\d{2})-[a-zA-Z0-9_-]{16,128}|sk-(?:proj-)?[A-Za-z0-9_-]{20,}|AIza[0-9A-Za-z_-]{35}|(?:ghp|gho|ghu|ghs|ghr|github_pat_|glpat-|xox[baprs]-)[A-Za-z0-9_-]{16,}|(?:[rs]k)_(?:test|live)_[a-zA-Z0-9]{10,}|eyJ[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,})\b/g,
   AWS_KEY: /\b(?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA)[A-Z0-9]{16}\b/g,
   CREDIT_CARD: /\b(?:\d{4}[-\s]?){3}\d{4}\b/g,
   EMAIL: /[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]{1,63}(?:\.[A-Za-z0-9-]{1,63})*\.[A-Za-z]{2,24}/g,
@@ -24,7 +24,213 @@ const ZTDS_PATTERNS = {
   SSN: /\b\d{3}-\d{2}-\d{4}\b/g,
 };
 
-const TOKEN_PATTERN = /\[([A-Z_]+)_TOKEN_([a-zA-Z0-9_-]+)\]/g;
+const TOKEN_PATTERN = /(?:%5[Bb]|\\u005[Bb]|&(?:amp;)?#0*91;|\[|<|\{\{?|__|【|［)[*`_~\s]*([A-Z_]+)_TOKEN_([a-zA-Z0-9_-]+)[*`_~.,;:!?\s]*(?:%5[Dd]|\\u005[Dd]|&(?:amp;)?#0*93;|\]|>|\}\}?|__|】|］)/gi;
+
+function escapeRegex(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function buildStreamReplacer(tokenMap = {}, options = {}) {
+  if (!tokenMap || typeof tokenMap !== 'object' || Object.keys(tokenMap).length === 0) {
+    return (text) => text || '';
+  }
+
+  const lookup = new Map();
+  const patterns = [];
+  const sortedKeys = Object.keys(tokenMap).sort((a, b) => b.length - a.length);
+
+  for (const k of sortedKeys) {
+    const val = tokenMap[k];
+    const inner = k.replace(/^(?:\[|<|\{\{?|__|【|［|〔|「|『)|(?:\]|>|\}\}?|__|】|］|〕|」|』)$/g, '');
+    const match = inner.match(/^([A-Za-z_0-9]+?)[-_ ]([a-zA-Z0-9]+)$/);
+
+    if (match) {
+      const label = escapeRegex(match[1]);
+      const idx = escapeRegex(match[2]);
+      const idxPat = /^\d+$/.test(match[2]) ? '0*' + idx : idx;
+      const corePat = '[*`_~\\s]*' + label + '[*`_~]*[-_\\s]*' + idxPat + '[*`_~.,;:!?\\s]*';
+      const patternStr = '(?:\\[' + corePat + '\\]|<' + corePat + '>|\\{\\{' + corePat + '\\}\\}|\\{' + corePat + '\\}|__' + corePat + '__|[【［〔「『]' + corePat + '[】］〕」』]|%5B' + corePat + '%5D|&#(?:91|091|x5[bB]);' + corePat + '&#(?:93|093|x5[dD]);|&(?:lbrack|lsqb);' + corePat + '&(?:rbrack|rsqb);|\\\\u005[bB]' + corePat + '\\\\u005[dD]|\\b' + label + '[-_\\s]+' + idxPat + '\\b)';
+      patterns.push(patternStr);
+      const normSuffix = /^\d+$/.test(match[2]) ? String(parseInt(match[2], 10)) : match[2].toUpperCase();
+      const normKey = (match[1] + '_' + normSuffix).toUpperCase();
+      lookup.set(normKey, val);
+      lookup.set(k, val);
+      lookup.set(k.toUpperCase(), val);
+    } else {
+      const safe = escapeRegex(k);
+      patterns.push(safe);
+      lookup.set(k, val);
+      lookup.set(k.toUpperCase(), val);
+    }
+  }
+
+  const unifiedRegex = new RegExp('(?:' + patterns.join('|') + ')(\'s|’s|s)?', 'gi');
+
+  return function replaceTokens(text) {
+    if (!text) return '';
+    unifiedRegex.lastIndex = 0;
+    return text.replace(unifiedRegex, (match, suffix) => {
+      const isPossessive = suffix && /('s|’s)$/i.test(suffix);
+      const coreMatch = suffix ? match.slice(0, match.length - suffix.length) : match;
+      let restored = null;
+
+      if (lookup.has(coreMatch)) {
+        restored = lookup.get(coreMatch);
+      } else if (lookup.has(coreMatch.toUpperCase())) {
+        restored = lookup.get(coreMatch.toUpperCase());
+      } else {
+        const inner = coreMatch
+          .replace(/^(?:%5B|&#(?:91|091|x5[bB]);|&(?:lbrack|lsqb);|\\u005[bB]|\[|<|\{\{?|__|【|［|〔|「|『)/i, '')
+          .replace(/(?:%5D|&#(?:93|093|x5[dD]);|&(?:rbrack|rsqb);|\\u005[dD]|\]|>|\}\}?|__|】|］|〕|」|』)$/i, '')
+          .replace(/[*`_~.,;:!?\s]+/g, ' ')
+          .trim();
+        const m = inner.match(/^([A-Za-z_0-9]+?)[-_ ]([a-zA-Z0-9]+)$/);
+        if (m) {
+          const normSuffix = /^\d+$/.test(m[2]) ? String(parseInt(m[2], 10)) : m[2].toUpperCase();
+          const normKey = (m[1] + '_' + normSuffix).toUpperCase();
+          if (lookup.has(normKey)) {
+            restored = lookup.get(normKey);
+          }
+        }
+      }
+
+      if (restored !== null) {
+        return restored + (isPossessive ? "'s" : '');
+      }
+
+      if (lookup.has(match)) {
+        return lookup.get(match);
+      }
+
+      return match;
+    });
+  };
+}
+
+class StreamingDetokenizer {
+  constructor(tokenMap = {}, options = {}) {
+    this.tokenMap = tokenMap && typeof tokenMap === 'object' ? tokenMap : {};
+    this.options = options || {};
+    this.replacer = buildStreamReplacer(this.tokenMap, this.options);
+    this.pendingBuffer = '';
+    this.hasTokens = Object.keys(this.tokenMap).length > 0;
+    this._lastTokenCount = Object.keys(this.tokenMap).length;
+    this.decoder = new TextDecoder('utf-8', { fatal: false });
+  }
+
+  get buffer() {
+    return this.pendingBuffer;
+  }
+
+  set buffer(val) {
+    this.pendingBuffer = typeof val === 'string' ? val : '';
+  }
+
+  updateTokenMap(newMap) {
+    if (newMap && typeof newMap === 'object') {
+      this.tokenMap = newMap;
+      this.replacer = buildStreamReplacer(this.tokenMap, this.options);
+      this.hasTokens = Object.keys(this.tokenMap).length > 0;
+      this._lastTokenCount = Object.keys(this.tokenMap).length;
+    }
+  }
+
+  reset() {
+    this.pendingBuffer = '';
+  }
+
+  feed(chunk) {
+    if (chunk === undefined || chunk === null) return '';
+    let text = '';
+    if (typeof chunk === 'string') {
+      text = chunk;
+    } else if (chunk instanceof Uint8Array || (typeof Buffer !== 'undefined' && Buffer.isBuffer(chunk))) {
+      text = this.decoder.decode(chunk, { stream: true });
+    } else {
+      text = String(chunk);
+    }
+
+    if (!text) return '';
+
+    const currentTokenCount = Object.keys(this.tokenMap).length;
+    if (currentTokenCount !== this._lastTokenCount) {
+      this.replacer = buildStreamReplacer(this.tokenMap, this.options);
+      this.hasTokens = currentTokenCount > 0;
+      this._lastTokenCount = currentTokenCount;
+    }
+
+    if (!this.hasTokens) return text;
+
+    this.pendingBuffer += text;
+
+    const tailLen = 160;
+    const tail = this.pendingBuffer.length > tailLen ? this.pendingBuffer.slice(-tailLen) : this.pendingBuffer;
+    const offset = this.pendingBuffer.length > tailLen ? this.pendingBuffer.length - tailLen : 0;
+    const m = tail.match(/(?:(?:\[|【|［|〔|「|『|<|\{\{?|__?|%5[Bb]|\\u005[Bb]|&(?:amp;)?(?:#0*91;|#[xX]0*5[Bb];|lbrack;))[A-Za-z0-9_\-А-Яа-яЁё *`~.\\]{0,128}(?:%5?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?3?|[xX]0*5?[Dd]?)?|r(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)?|%[0-9A-Fa-f]?|\\(?:u(?:0(?:0(?:5)?)?)?)?|&(?:a(?:m(?:p(?:;)?)?)?)?(?:#(?:0*9?1?|[xX]0*5?)?|l(?:b(?:r(?:a(?:c(?:k)?)?)?)?)?)?)$/);
+
+    if (m) {
+      const candidate = m[0];
+      const isClosedUnderscoreOrUnicode = (candidate.length >= 4 && candidate.startsWith('__') && candidate.endsWith('__')) || /\\u005[Dd]/.test(candidate);
+      if (isClosedUnderscoreOrUnicode && this.replacer(candidate) !== candidate) {
+        const toEmit = this.pendingBuffer;
+        this.pendingBuffer = '';
+        return this.replacer(toEmit);
+      }
+      const splitIdx = offset + m.index;
+      const safeToEmit = this.pendingBuffer.slice(0, splitIdx);
+      this.pendingBuffer = this.pendingBuffer.slice(splitIdx);
+      return safeToEmit ? this.replacer(safeToEmit) : '';
+    }
+
+    const toEmit = this.pendingBuffer;
+    this.pendingBuffer = '';
+    return this.replacer(toEmit);
+  }
+
+  process(chunk) {
+    return this.feed(chunk);
+  }
+
+  flush() {
+    let text = '';
+    try {
+      text = this.decoder.decode(new Uint8Array(), { stream: false });
+    } catch (_) {}
+
+    if (text) {
+      this.pendingBuffer += text;
+    }
+
+    if (!this.pendingBuffer) return '';
+    const remaining = this.pendingBuffer;
+    this.pendingBuffer = '';
+    return this.replacer(remaining);
+  }
+
+  async* wrapStream(asyncIterable) {
+    for await (const chunk of asyncIterable) {
+      if (chunk === null || chunk === undefined) continue;
+      let text = '';
+      if (typeof chunk === 'string') {
+        text = chunk;
+      } else if (typeof chunk?.text === 'string') {
+        text = chunk.text;
+      } else if (typeof chunk?.content === 'string') {
+        text = chunk.content;
+      } else if (chunk instanceof Uint8Array || (typeof Buffer !== 'undefined' && Buffer.isBuffer(chunk))) {
+        text = chunk;
+      } else {
+        text = String(chunk);
+      }
+
+      const emitted = this.feed(text);
+      if (emitted) yield emitted;
+    }
+
+    const finalChunk = this.flush();
+    if (finalChunk) yield finalChunk;
+  }
+}
 
 class ZTDSFastMCPGuard {
   constructor(options = {}) {
@@ -36,6 +242,7 @@ class ZTDSFastMCPGuard {
     this.maxSessionTokens = options.maxSessionTokens || 5000;
     this._sessionMaps = new Map();
     this._entityMaps = new Map();
+    this._streamDetokenizers = new Map();
   }
 
   _generateToken(entityType, tokenMap) {
@@ -69,7 +276,9 @@ class ZTDSFastMCPGuard {
 
     const tokenMap = this._sessionMaps.get(sessionId);
     const entityMap = this._entityMaps.get(sessionId);
-    let sanitized = text;
+    let sanitized = text
+      .replace(/[\u200B-\u200F\u202A-\u202E\u2060-\u206F\uFEFF\u00AD]/g, "")
+      .replace(/[\uFF01-\uFF5E]/g, (ch) => String.fromCharCode(ch.charCodeAt(0) - 0xFEE0));
 
     for (const entityType of this.enabledEntities) {
       const pattern = ZTDS_PATTERNS[entityType];
@@ -118,8 +327,9 @@ class ZTDSFastMCPGuard {
       return text;
     }
 
-    return text.replace(TOKEN_PATTERN, (match) => {
-      return tokenMap.get(match) || match;
+    return text.replace(TOKEN_PATTERN, (match, entityType, tokenId) => {
+      const canonical = `[${entityType.toUpperCase()}_TOKEN_${tokenId}]`;
+      return tokenMap.get(canonical) || tokenMap.get(match) || match;
     });
   }
 
@@ -157,6 +367,65 @@ class ZTDSFastMCPGuard {
     return val;
   }
 
+  getStreamDetokenizer(sessionId) {
+    if (!this._streamDetokenizers.has(sessionId)) {
+      const mapProxy = this._sessionMaps.get(sessionId) || new Map();
+      const objMap = {};
+      for (const [k, v] of mapProxy.entries()) {
+        objMap[k] = v;
+      }
+      this._streamDetokenizers.set(sessionId, new StreamingDetokenizer(objMap));
+    } else {
+      const mapProxy = this._sessionMaps.get(sessionId);
+      if (mapProxy) {
+        const objMap = {};
+        for (const [k, v] of mapProxy.entries()) {
+          objMap[k] = v;
+        }
+        this._streamDetokenizers.get(sessionId).updateTokenMap(objMap);
+      }
+    }
+    return this._streamDetokenizers.get(sessionId);
+  }
+
+  feedStream(chunk, sessionId) {
+    return this.getStreamDetokenizer(sessionId).feed(chunk);
+  }
+
+  flushStream(sessionId) {
+    const detok = this._streamDetokenizers.get(sessionId);
+    return detok ? detok.flush() : "";
+  }
+
+  async *restoreStream(asyncIterable, sessionId) {
+    const detok = this.getStreamDetokenizer(sessionId);
+    try {
+      for await (const chunk of asyncIterable) {
+        if (chunk === null || chunk === undefined) continue;
+        let text = "";
+        if (typeof chunk === "string") {
+          text = chunk;
+        } else if (typeof chunk?.text === "string") {
+          text = chunk.text;
+        } else if (typeof chunk?.content === "string") {
+          text = chunk.content;
+        } else if (chunk instanceof Uint8Array || (typeof Buffer !== "undefined" && Buffer.isBuffer(chunk))) {
+          text = chunk;
+        } else {
+          text = String(chunk);
+        }
+        const emitted = detok.feed(text);
+        if (emitted) yield emitted;
+      }
+      const finalChunk = detok.flush();
+      if (finalChunk) yield finalChunk;
+    } finally {
+      if (this.autoZeroize) {
+        this.zeroizeSession(sessionId);
+      }
+    }
+  }
+
   wrapTool(toolName, handler) {
     return async (args, context = {}) => {
       const sessionId =
@@ -169,6 +438,7 @@ class ZTDSFastMCPGuard {
         unmask: (val) => this.restoreObject(val, sessionId),
       };
 
+      let isStreaming = false;
       try {
         let processedArgs = args;
         if (this.sanitizeInputs && args !== undefined && args !== null) {
@@ -177,6 +447,11 @@ class ZTDSFastMCPGuard {
 
         const rawResult = await handler(processedArgs, enrichedContext);
 
+        if (rawResult && typeof rawResult[Symbol.asyncIterator] === "function") {
+          isStreaming = true;
+          return this.restoreStream(rawResult, sessionId);
+        }
+
         let finalResult = rawResult;
         if (this.sanitizeOutputs && rawResult !== undefined && rawResult !== null) {
           if (typeof rawResult === "string") {
@@ -211,7 +486,7 @@ class ZTDSFastMCPGuard {
 
         const ztdsMeta = {
           sessionId,
-          standard: "IETF draft-sibiryakov-ztds-protocol-02",
+          standard: "IETF draft-sibiryakov-ztds-protocol-01",
           zeroEgress: true,
         };
 
@@ -234,7 +509,7 @@ class ZTDSFastMCPGuard {
           _ztds: ztdsMeta,
         };
       } finally {
-        if (this.autoZeroize) {
+        if (!isStreaming && this.autoZeroize) {
           this.zeroizeSession(sessionId);
         }
       }
@@ -250,6 +525,10 @@ class ZTDSFastMCPGuard {
       this._entityMaps.get(sessionId).clear();
       this._entityMaps.delete(sessionId);
     }
+    if (this._streamDetokenizers.has(sessionId)) {
+      this._streamDetokenizers.get(sessionId).reset();
+      this._streamDetokenizers.delete(sessionId);
+    }
   }
 }
 
@@ -285,6 +564,8 @@ module.exports = {
   ZTDSFastMCPGuard,
   ZTDS_PATTERNS,
   TOKEN_PATTERN,
+  StreamingDetokenizer,
+  createStreamTokenRestorer: (tokenMap, options) => new StreamingDetokenizer(tokenMap, options),
   createZTDSGuard,
   ztdsGuardTool,
   wrapExecute,

package/fastmcp.d.ts

@@ -26,6 +26,19 @@ export interface ZTDSContext {
 export declare const ZTDS_PATTERNS: Record<string, RegExp>;
 export declare const TOKEN_PATTERN: RegExp;
 
+export declare class StreamingDetokenizer {
+  constructor(tokenMap?: Record<string, string>, options?: object);
+  buffer: string;
+  feed(chunk: string | Uint8Array | Buffer): string;
+  process(chunk: string | Uint8Array | Buffer): string;
+  flush(): string;
+  reset(): void;
+  updateTokenMap(newMap: Record<string, string>): void;
+  wrapStream<T = any>(asyncIterable: AsyncIterable<T>): AsyncGenerator<T>;
+}
+
+export declare function createStreamTokenRestorer(tokenMap: Record<string, string>, options?: object): StreamingDetokenizer;
+
 export declare class ZTDSFastMCPGuard {
   constructor(options?: ZTDSGuardOptions);
   enabledEntities: string[];
@@ -39,6 +52,11 @@ export declare class ZTDSFastMCPGuard {
   sanitizeObject(val: unknown, sessionId: string): unknown;
   restoreObject(val: unknown, sessionId: string): unknown;
 
+  getStreamDetokenizer(sessionId: string): StreamingDetokenizer;
+  feedStream(chunk: string, sessionId: string): string;
+  flushStream(sessionId: string): string;
+  restoreStream<T = any>(asyncIterable: AsyncIterable<T>, sessionId: string): AsyncGenerator<T>;
+
   wrapTool<TArgs = unknown, TResult = unknown>(
     toolName: string,
     handler: (args: TArgs, context: ZTDSContext) => Promise<TResult>

package/fastmcp.js

@@ -11,6 +11,8 @@ const fastmcpGuard = require('./fastmcp.cjs');
 export const ZTDSFastMCPGuard = fastmcpGuard.ZTDSFastMCPGuard;
 export const ZTDS_PATTERNS = fastmcpGuard.ZTDS_PATTERNS;
 export const TOKEN_PATTERN = fastmcpGuard.TOKEN_PATTERN;
+export const StreamingDetokenizer = fastmcpGuard.StreamingDetokenizer;
+export const createStreamTokenRestorer = fastmcpGuard.createStreamTokenRestorer;
 export const createZTDSGuard = fastmcpGuard.createZTDSGuard;
 export const ztdsGuardTool = fastmcpGuard.ztdsGuardTool;
 export const wrapExecute = fastmcpGuard.wrapExecute;
@@ -20,6 +22,8 @@ export default {
   ZTDSFastMCPGuard,
   ZTDS_PATTERNS,
   TOKEN_PATTERN,
+  StreamingDetokenizer,
+  createStreamTokenRestorer,
   createZTDSGuard,
   ztdsGuardTool,
   wrapExecute,