@link-foundation/sensitive-data-sanitizer 0.0.0-stage → 0.11.31

package/src/detection.js

@@ -0,0 +1,293 @@
+import { cardNetwork, cardEvidence } from './confidence.js';
+import { isIP } from 'node:net';
+import { serviceRules } from './rules.js';
+import { detectContext, valueRange } from './context.js';
+import { detectPersonal } from './personal.js';
+import { detectGitleaks } from './gitleaks.js';
+import { detectCatalog } from './entities.js';
+import { detectIdentity } from './identity.js';
+import { detectNational } from './national.js';
+import { detectPersonalContext } from './personal-context.js';
+import {
+  detectCredentialFormats,
+  credentialPlaceholder,
+} from './credentials.js';
+import { localAddress } from './public.js';
+
+export const REDACTED = '[REDACTED]';
+export const failure = (code = 'ERR_SANITIZATION', location = {}) =>
+  Object.assign(new Error('Sanitization failed; output was blocked.'), {
+    code,
+    ...Object.fromEntries(
+      ['record', 'offset', 'recordOffset']
+        .filter(
+          (key) => Number.isSafeInteger(location[key]) && location[key] >= 0
+        )
+        .map((key) => [key, location[key]])
+    ),
+  });
+export function internalFailure(error) {
+  return /^ERR_[A-Z_]+$/.test(error?.code) ? error : failure();
+}
+export const maxPropagationLength = 256;
+// Verification redacts values found again after a replacement, up to this many
+// extra rounds, before publication fails closed with ERR_RESIDUAL.
+export const residualRounds = 3;
+export function personalPattern(value) {
+  if (value.length > maxPropagationLength) {
+    throw failure('ERR_LIMIT');
+  }
+  // Split before escaping: replacing hyphens in an escaped pattern can turn
+  // a preceding literal backslash into regexp syntax.
+  return value
+    .split(/[ _-]+/)
+    .map(escapePattern)
+    .join('[ \\t_-]+');
+}
+export function escapePattern(text) {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+export function collectMatches(
+  text,
+  regex,
+  emit,
+  details,
+  group = 0,
+  accept = () => true
+) {
+  // Clone expressions so concurrent callers never share lastIndex.
+  const pattern = new RegExp(
+    regex.source,
+    regex.flags.includes('g') ? regex.flags : `${regex.flags}g`
+  );
+  for (const match of text.matchAll(pattern)) {
+    const raw = match[group];
+    const value = details.category === 'personal' ? raw?.trim() : raw;
+    if (!value || value.trim() === REDACTED || !accept(value, match)) {
+      continue;
+    }
+    const offset = group ? match[0].lastIndexOf(value) : 0;
+    emit({
+      start: match.index + offset,
+      end: match.index + offset + value.length,
+      ...details,
+    });
+  }
+}
+
+export function entropy(value) {
+  const counts = new Map();
+  for (const char of value) {
+    counts.set(char, (counts.get(char) ?? 0) + 1);
+  }
+  let result = 0;
+  for (const count of counts.values()) {
+    const p = count / value.length;
+    result -= p * Math.log2(p);
+  }
+  return result;
+}
+
+export function luhn(value) {
+  const digits = value.replace(/\D/g, '');
+  if (digits.length < 13 || digits.length > 19 || /^(\d)\1+$/.test(digits)) {
+    return false;
+  }
+  let total = 0;
+  for (
+    let i = digits.length - 1, double = false;
+    i >= 0;
+    i--, double = !double
+  ) {
+    let n = Number(digits[i]);
+    if (double) {
+      n *= 2;
+      if (n > 9) {
+        n -= 9;
+      }
+    }
+    total += n;
+  }
+  return total % 10 === 0;
+}
+
+function detectCookies(text, emit) {
+  for (const match of text.matchAll(/(?:Set-)?Cookie[ \t]*:[ \t]*/gi)) {
+    const range = valueRange(text, match.index + match[0].length, 'line');
+    if (
+      range &&
+      range.end > range.start &&
+      text.slice(range.start, range.end) !== REDACTED
+    ) {
+      emit({
+        ...range,
+        type: 'SECRET',
+        category: 'credential',
+        rule: 'cookie',
+      });
+    }
+  }
+}
+
+export function nativeDetect(text, options, emit) {
+  const credential = { type: 'SECRET', category: 'credential' };
+  detectGitleaks(text, emit);
+  for (const [rule, pattern] of serviceRules) {
+    collectMatches(text, pattern, emit, { ...credential, rule });
+  }
+  for (const value of options.knownSecrets ?? []) {
+    collectMatches(text, new RegExp(escapePattern(value), 'g'), emit, {
+      ...credential,
+      rule: 'known-secret',
+    });
+  }
+  for (const entry of options.knownPersonal ?? []) {
+    collectMatches(text, new RegExp(escapePattern(entry.value), 'giu'), emit, {
+      type: entry.type,
+      category: 'personal',
+      rule: 'known-personal',
+    });
+  }
+  for (const entry of options.confirmedPersonal ?? []) {
+    const pattern = new RegExp(
+      `(?<![\\p{L}\\p{N}])${personalPattern(entry.value)}(?![\\p{L}\\p{N}])`,
+      'giu'
+    );
+    collectMatches(text, pattern, emit, {
+      type: entry.type,
+      category: 'personal',
+      rule: 'confirmed-personal',
+      confidence: 0.9,
+    });
+  }
+  detectContext(text, emit, options.credentialsOnly);
+  if (!options.credentialsOnly) {
+    detectPersonal(text, emit);
+    detectCatalog(text, emit);
+    detectIdentity(text, emit);
+    detectNational(text, emit);
+    detectPersonalContext(text, emit);
+  }
+  detectCredentialFormats(text, emit);
+  // detect-secrets-style quoted entropy detection; bare commit hashes and
+  // common tool identifiers remain outside this heuristic's scope.
+  collectMatches(
+    text,
+    /["']([A-Za-z0-9+/=_-]{20,256})["']/g,
+    emit,
+    { ...credential, rule: 'quoted-entropy' },
+    1,
+    (value) => entropy(value) >= (/^[a-f0-9]+$/i.test(value) ? 3 : 4.5)
+  );
+  collectMatches(
+    text,
+    /-----BEGIN ([A-Z0-9 ]*PRIVATE KEY)-----[\s\S]*?(?:-----END \1-----|$)/g,
+    emit,
+    { ...credential, rule: 'private-key' }
+  );
+  collectMatches(
+    text,
+    /(?:Proxy-)?Authorization[ \t]*:[ \t]*(?:Bearer|Basic|Digest)[ \t]+([^\r\n"',]+)/gi,
+    emit,
+    { ...credential, rule: 'authorization' },
+    1,
+    (value) => !credentialPlaceholder(value.trim())
+  );
+  detectCookies(text, emit);
+  collectMatches(
+    text,
+    /(?<![a-z0-9+.-])[a-z][a-z0-9+.-]*:\/\/([^\s/@]+)@/gi,
+    emit,
+    { ...credential, rule: 'url-userinfo' },
+    1
+  );
+  collectMatches(
+    text,
+    /https:\/\/(?:hooks\.slack\.com\/services|(?:canary\.)?discord(?:app)?\.com\/api\/webhooks)\/([^\s"'<>]+)/gi,
+    emit,
+    { ...credential, rule: 'webhook' },
+    1
+  );
+  if (options.credentialsOnly && !options.paranoid) {
+    return;
+  }
+  detectPersonalFormats(text, options, emit);
+  if (options.paranoid) {
+    collectMatches(
+      text,
+      /(?<![A-Za-z0-9_+/-])[A-Za-z0-9_+/-]{20,256}={0,2}(?![A-Za-z0-9_+/-])/g,
+      emit,
+      { ...credential, rule: 'entropy' },
+      0,
+      (v) =>
+        !(
+          /^[a-f0-9]{7,64}$/i.test(v) ||
+          /^mcp__|^browser_|^\[REDACTED\]$/.test(v)
+        ) && entropy(v) >= 4.2
+    );
+  }
+}
+
+function detectPersonalFormats(text, options, emit) {
+  collectMatches(
+    text,
+    /(?<![\p{L}\p{N}_.!#$%&'*+/?^`{|}~-])[\p{L}\p{N}_.!#$%&'*+/?^`{|}~-]+@[\p{L}\p{N}](?:[\p{L}\p{N}.-]{0,252})\.[\p{L}]{2,63}(?![\p{L}\p{N}_-])/gu,
+    emit,
+    { type: 'EMAIL', category: 'personal', rule: 'email' }
+  );
+  collectMatches(
+    text,
+    /(?<!\d)\+\d[\d ()-]{6,24}\d(?!\d)/g,
+    emit,
+    { type: 'PHONE', category: 'personal', rule: 'international-phone' },
+    0,
+    (v) => {
+      const n = v.replace(/\D/g, '').length;
+      return n >= 8 && n <= 15;
+    }
+  );
+  collectMatches(
+    text,
+    /(?<!\d)(?:\d[ -]?){12,18}\d(?!\d)/g,
+    emit,
+    { type: 'CREDIT_CARD', category: 'personal', rule: 'luhn' },
+    0,
+    (value, match) =>
+      luhn(value) &&
+      cardNetwork(value) &&
+      cardEvidence(text, match.index, value)
+  );
+  collectMatches(
+    text,
+    /\b\d{3}-\d{2}-\d{4}\b/g,
+    emit,
+    { type: 'ID', category: 'personal', rule: 'us-ssn' },
+    0,
+    (v) => !/^(?:000|666|9\d\d)-|^\d{3}-00-|0000$/.test(v)
+  );
+  collectMatches(
+    text,
+    /(?<![\w.])(?:\d{1,3}\.){3}\d{1,3}(?![\w.])/g,
+    emit,
+    { type: 'IP_ADDRESS', category: 'personal', rule: 'ipv4' },
+    0,
+    (v) =>
+      isIP(v) === 4 && (options.publicKnowledge === false || !localAddress(v))
+  );
+  collectMatches(
+    text,
+    /(?<![\w:])(?:[a-fA-F0-9]{0,4}:){2,7}[a-fA-F0-9]{0,4}(?![\w:])/g,
+    emit,
+    { type: 'IP_ADDRESS', category: 'personal', rule: 'ipv6' },
+    0,
+    // Rust paths such as a::b and namespace prefixes are valid IPv6 syntax,
+    // but carry no useful address evidence in coding-session text.
+    (v) =>
+      isIP(v) === 6 &&
+      (options.publicKnowledge === false || !localAddress(v)) &&
+      v !== '::' &&
+      !/^[a-fA-F0-9]{0,4}::$/.test(v) &&
+      (v.split(':').filter(Boolean).length >= 3 || /\d/.test(v))
+  );
+}

package/src/encoded.js

@@ -0,0 +1,233 @@
+import { TextDecoder } from 'node:util';
+import { Buffer } from 'node:buffer';
+import { failure } from './detection.js';
+
+export function* decodedRuns(text, depth = 0) {
+  if (depth >= 2) {
+    return;
+  }
+  const patterns = [
+    /(?<![A-Za-z0-9_+/-])[A-Za-z0-9_+/-]{16}[A-Za-z0-9_+/-]*={0,2}(?![A-Za-z0-9_+/-])/g,
+    /(?:[^\s"'<>?=&:/]{0,128}%[\da-fA-F]{2})+(?:[^\s"'<>?=&:/]{0,128})/g,
+    /(?<![A-Za-z0-9_+/-])[A-Za-z0-9_+/-]{16,128}(?:\r?\n[A-Za-z0-9_+/-]{1,128})+={0,2}(?![A-Za-z0-9_+/-])/g,
+    /(?:&#(?:x[\da-fA-F]{1,6}|\d{1,7});|&(?:amp|lt|gt|quot|apos|nbsp);){2,2048}/g,
+    /(?:\\x[\da-fA-F]{2}){2,2048}/g,
+    /"(?:[^"\\\r\n]|\\[^\r\n]){1,8192}"/g,
+  ];
+  const seen = new Set();
+  let count = 0;
+  for (const [index, pattern] of patterns.entries()) {
+    const marker = ['', '%', '\n', '&', '\\x', '"'][index];
+    if (marker && !text.includes(marker)) {
+      continue;
+    }
+    for (const match of text.matchAll(pattern)) {
+      const raw = match[0];
+      if (raw.length > 8192) {
+        if (isLimitedTextRun(raw)) {
+          yield {
+            start: match.index,
+            end: match.index + raw.length,
+            limited: true,
+            depth,
+          };
+        }
+        continue;
+      }
+      const candidates = decodeCandidates(raw);
+      for (const candidate of candidates) {
+        const { text: decoded, encoding } = candidate;
+        const key = `${match.index}:${raw.length}:${decoded}`;
+        if (seen.has(key) || !decodedText(raw, decoded)) {
+          continue;
+        }
+        seen.add(key);
+        if (++count > 4096) {
+          throw failure('ERR_LIMIT');
+        }
+        const inset = encoding === 'json-content' ? 1 : 0;
+        const run = {
+          start: match.index + inset,
+          end: match.index + raw.length - inset,
+          text: decoded,
+          encoding,
+          depth,
+        };
+        yield run;
+        for (const inner of decodedRuns(decoded, depth + 1)) {
+          yield { ...run, text: inner.text, depth: inner.depth };
+        }
+      }
+    }
+  }
+}
+
+function decodedText(raw, decoded) {
+  return (
+    decoded !== raw &&
+    Boolean(decoded) &&
+    // eslint-disable-next-line no-control-regex -- Binary controls cannot form text detection views.
+    !/[\x00-\x08\x0e-\x1f\x7f]/.test(decoded)
+  );
+}
+
+function isLimitedTextRun(raw) {
+  // Bare opaque identifiers may share the base64 alphabet. Require a
+  // textual decode, or an explicit percent/wrapped encoding boundary.
+  const probe = decodeCandidates(raw.slice(0, 4096), true);
+  return (
+    raw.includes('%') ||
+    raw.includes('\n') ||
+    probe.some(
+      // eslint-disable-next-line no-control-regex -- Exclude binary controls from the bounded UTF-8 text probe.
+      (c) => c.text && !/[\x00-\x08\x0e-\x1f\x7f]/.test(c.text)
+    )
+  );
+}
+
+function decodeCandidates(raw, partial) {
+  const candidates = [];
+  if (raw.startsWith('"')) {
+    return decodeJson(raw);
+  }
+  if (raw.startsWith('&#') || /^&(?:amp|lt|gt|quot|apos|nbsp);/.test(raw)) {
+    candidates.push({
+      encoding: raw.startsWith('&#x') ? 'html-hex' : 'html-decimal',
+      text: raw.replace(
+        /&(?:#(x[\da-fA-F]+|\d+)|(amp|lt|gt|quot|apos|nbsp));/g,
+        (whole, code, named) => {
+          if (named) {
+            return {
+              amp: '&',
+              lt: '<',
+              gt: '>',
+              quot: '"',
+              apos: "'",
+              nbsp: ' ',
+            }[named];
+          }
+          const point =
+            code[0] === 'x' ? parseInt(code.slice(1), 16) : Number(code);
+          return point <= 0x10ffff && !(point >= 0xd800 && point <= 0xdfff)
+            ? String.fromCodePoint(point)
+            : whole;
+        }
+      ),
+    });
+  } else if (raw.startsWith('\\x')) {
+    try {
+      candidates.push({
+        encoding: 'byte-escape',
+        text: new TextDecoder('utf-8', { fatal: true }).decode(
+          Buffer.from(raw.replaceAll('\\x', ''), 'hex')
+        ),
+      });
+    } catch {
+      /* Non-text bytes. */
+    }
+  } else if (raw.includes('%')) {
+    try {
+      candidates.push({ encoding: 'percent', text: decodeURIComponent(raw) });
+    } catch {
+      return [];
+    }
+  } else {
+    const compact = raw.replace(/\s/g, '');
+    // Decode canonical base64 text only.
+    const bytes = Buffer.from(compact, 'base64url');
+    if (
+      bytes.toString('base64url') ===
+      compact.replace(/=+$/, '').replaceAll('+', '-').replaceAll('/', '_')
+    ) {
+      try {
+        candidates.push({
+          encoding:
+            compact.includes('-') || compact.includes('_')
+              ? 'base64url'
+              : 'base64',
+          text: new TextDecoder('utf-8', { fatal: true }).decode(bytes, {
+            stream: partial,
+          }),
+          wrapped: raw.includes('\n'),
+          padded: raw.endsWith('='),
+        });
+      } catch {
+        /* Binary content. */
+      }
+    }
+    if (/^(?:[a-fA-F0-9]{2}){16,}$/.test(compact)) {
+      try {
+        candidates.push({
+          encoding: 'hex',
+          text: new TextDecoder('utf-8', { fatal: true }).decode(
+            Buffer.from(compact, 'hex')
+          ),
+        });
+      } catch {
+        /* Binary content. */
+      }
+    }
+  }
+  return candidates;
+}
+
+export function encodeRun(run, text) {
+  const bytes = Buffer.from(text, 'utf8');
+  let encoded;
+  if (run.encoding === 'json-content') {
+    encoded = JSON.stringify(text).slice(1, -1);
+  } else if (run.encoding === 'percent') {
+    encoded = Array.from(
+      bytes,
+      (b) => `%${b.toString(16).padStart(2, '0')}`
+    ).join('');
+  } else if (run.encoding === 'byte-escape') {
+    encoded = Array.from(
+      bytes,
+      (b) => `\\x${b.toString(16).padStart(2, '0')}`
+    ).join('');
+  } else if (run.encoding.startsWith('html-')) {
+    encoded = Array.from(text, (c) =>
+      run.encoding === 'html-hex'
+        ? `&#x${c.codePointAt(0).toString(16)};`
+        : `&#${c.codePointAt(0)};`
+    ).join('');
+  } else {
+    encoded = bytes.toString(run.encoding);
+  }
+  const decoded =
+    run.encoding === 'json-content'
+      ? { text: JSON.parse(`"${encoded}"`) }
+      : decodeCandidates(encoded).find((c) => c.encoding === run.encoding);
+  if (!decoded || decoded.text !== text) {
+    throw failure('ERR_ENCODING');
+  }
+  return encoded;
+}
+
+export function encodedValue(raw) {
+  for (const run of decodedRuns(raw)) {
+    if (run.start === 0 && run.end === raw.length && run.depth === 0) {
+      return run;
+    }
+  }
+  if (raw.includes('\\')) {
+    try {
+      return { encoding: 'json-content', text: JSON.parse(`"${raw}"`) };
+    } catch {
+      return undefined;
+    }
+  }
+  return undefined;
+}
+
+function decodeJson(raw) {
+  if (!raw.includes('\\')) {
+    return [];
+  }
+  try {
+    return [{ encoding: 'json-content', text: JSON.parse(raw) }];
+  } catch {
+    return [];
+  }
+}