@link-foundation/sensitive-data-sanitizer 0.0.0-stage → 0.12.0

package/src/engines.js

@@ -0,0 +1,348 @@
+import { actionable } from './confidence.js';
+import {
+  auditFaked,
+  redactAgain,
+  inspect,
+  redactResultAsync,
+  validateOptions,
+  validateInput,
+  sanitizeFinding,
+  validatePublic,
+  inspectFindingsOnly,
+  reviewFindings,
+  markFake,
+  applyProfile,
+} from './sanitizer.js';
+import { failure, internalFailure } from './detection.js';
+import { projectText } from './projection.js';
+import { decodedRuns } from './encoded.js';
+import {
+  structuredBatches,
+  mapStructuredFindings,
+  verifyStructured,
+} from './structured.js';
+
+let upstream;
+const configurations = new WeakMap();
+const streamPreparers = new WeakMap();
+export const sanitizerConfiguration = (engine) => configurations.get(engine);
+export const prepareStreamEngine = (engine, text, nativeFindings) =>
+  streamPreparers.get(engine)?.(text, nativeFindings);
+async function secretlintDetect(text) {
+  upstream ??= Promise.all([
+    import('@secretlint/core'),
+    import('@secretlint/secretlint-rule-preset-recommend'),
+    import('@secretlint/profiler'),
+  ]);
+  const [core, preset, profiler] = await upstream;
+  profiler.secretLintProfiler.setEnabled(false);
+  // Log text is untrusted: embedded disable comments cannot suppress results.
+  const rules = preset.rules.filter(
+    (rule) => rule.meta.id !== '@secretlint/secretlint-rule-filter-comments'
+  );
+  const result = await core.lintSource({
+    source: {
+      filePath: '/virtual/input.txt',
+      content: text,
+      contentType: 'text',
+    },
+    options: {
+      noPhysicFilePath: true,
+      maskSecrets: true,
+      config: { rules: rules.map((rule) => ({ id: rule.meta.id, rule })) },
+    },
+  });
+  return result.messages.map((message) => ({
+    start: message.range?.[0],
+    end: message.range?.[1],
+    category: 'credential',
+    type: 'SECRET',
+    rule: 'secretlint',
+  }));
+}
+
+async function detect(engine, text, limit) {
+  try {
+    const found = await engine.detect(text);
+    if (!Array.isArray(found)) {
+      throw failure('ERR_ENGINE');
+    }
+    if (found.length > limit) {
+      throw failure('ERR_LIMIT');
+    }
+    return found.map((finding) => sanitizeFinding(finding, text.length));
+  } catch {
+    throw failure('ERR_ENGINE');
+  }
+}
+
+async function engineFindings(engine, projection, options, emit) {
+  const limit = options.maxFindings ?? 100000;
+  if (projection.changed) {
+    for (const f of await detect(engine, projection.text, limit)) {
+      emit({
+        ...f,
+        start: projection.starts[f.start],
+        end: projection.ends[f.end - 1],
+      });
+    }
+  }
+  if (options.decode === false) {
+    return;
+  }
+  for (const run of decodedRuns(projection.text)) {
+    if (run.limited) {
+      continue;
+    }
+    const view = projectText(run.text).text;
+    const decoded = markFake(
+      view,
+      await detect(engine, view, limit),
+      options
+    ).filter((f) => actionable(f, options));
+    if (!decoded.length) {
+      continue;
+    }
+    emit({
+      start: projection.changed ? projection.starts[run.start] : run.start,
+      end: projection.changed ? projection.ends[run.end - 1] : run.end,
+      type: 'ENCODED_SENSITIVE',
+      category: decoded.some((f) => f.category === 'credential')
+        ? 'credential'
+        : 'personal',
+      rule: 'encoded-engine',
+    });
+  }
+}
+
+async function analyzeStructured(
+  text,
+  initial,
+  options,
+  nativeFindings,
+  analyze
+) {
+  const findings = [];
+  for (const batch of structuredBatches(text, options)) {
+    for (const finding of mapStructuredFindings(
+      batch,
+      await analyze(
+        batch.text,
+        [],
+        true,
+        options,
+        nativeFindings === undefined ? undefined : []
+      ),
+      options
+    )) {
+      findings.push(finding);
+    }
+  }
+  // Native structured inspection validates, locates and reapplies the
+  // same fake-value/public policy to source offsets from all engines.
+  return inspectFindingsOnly(text, [...findings, ...(nativeFindings ?? [])], {
+    ...options,
+    findings: initial,
+  });
+}
+
+export function createSanitizer(options = {}) {
+  options = applyProfile(options);
+  validateOptions(options);
+  for (const detector of options.detectors ?? []) {
+    if (!detector || typeof detector.detect !== 'function') {
+      throw failure('ERR_CONFIG');
+    }
+  }
+  const engines =
+    options.secretlint === false ? [] : [{ detect: secretlintDetect }];
+  engines.push(...(options.detectors ?? []));
+  let prepared;
+  async function analyze(
+    text,
+    initial = options.findings ?? [],
+    plain = false,
+    scopedOptions = options,
+    nativeFindings
+  ) {
+    const options = scopedOptions;
+    validateInput(text, options);
+    if (options.structured && !plain) {
+      return analyzeStructured(text, initial, options, nativeFindings, analyze);
+    }
+    const findings = [];
+    const emit = (finding) => {
+      if (findings.length >= (options.maxFindings ?? 100000)) {
+        throw failure('ERR_LIMIT');
+      }
+      findings.push(sanitizeFinding(finding, text.length));
+    };
+    for (const finding of initial) {
+      emit(finding);
+    }
+    const projection = projectText(text);
+    for (const engine of engines) {
+      for (const finding of await detect(
+        engine,
+        text,
+        options.maxFindings ?? 100000
+      )) {
+        emit(finding);
+      }
+      await engineFindings(engine, projection, options, emit);
+    }
+    const detected =
+      nativeFindings === undefined
+        ? inspect(text, { ...options, structured: undefined, findings })
+        : reviewFindings(text, [...nativeFindings, ...findings], options);
+    return options.verifyPublic
+      ? verifyFindings(text, detected, options.verifyPublic)
+      : detected;
+  }
+
+  const sanitizer = {
+    inspect: async (...args) => {
+      try {
+        return await analyze(...args);
+      } catch (error) {
+        throw internalFailure(error);
+      }
+    },
+    sanitize: async (...args) => {
+      try {
+        return await sanitizeText(...args);
+      } catch (error) {
+        throw internalFailure(error);
+      }
+    },
+    sanitizeJsonl: (text) =>
+      createSanitizer({ ...options, structured: 'jsonl' }).sanitize(text),
+  };
+  configurations.set(sanitizer, options);
+  streamPreparers.set(sanitizer, async (text, nativeFindings) => {
+    const findings = await analyze(
+      text,
+      nativeFindings === undefined ? (options.findings ?? []) : [],
+      false,
+      options,
+      nativeFindings
+    );
+    prepared = { text, findings };
+    return findings;
+  });
+  return sanitizer;
+  async function sanitizeText(
+    text,
+    depth = 0,
+    fullRedaction,
+    plain,
+    scopedOptions = options,
+    round = 0
+  ) {
+    const options = scopedOptions;
+    const findings =
+      depth === 0 && prepared?.text === text
+        ? prepared.findings
+        : await analyze(text, options.findings ?? [], plain, options);
+    if (depth === 0) {
+      prepared = undefined;
+    }
+    const fullOptions = {
+      ...options,
+      transformation: undefined,
+      transformations: undefined,
+      identityMask: false,
+      preserveEncoding: false,
+    };
+    const result = await redactResultAsync(
+      text,
+      findings,
+      fullRedaction ? fullOptions : options,
+      (decoded, recursiveOptions) =>
+        depth < 2
+          ? sanitizeText(
+              decoded,
+              depth + 1,
+              fullRedaction,
+              true,
+              recursiveOptions
+            )
+          : Promise.resolve({ text: '[REDACTED]' })
+    );
+    const sanitized = fullRedaction
+      ? result.text
+      : (
+          await redactResultAsync(
+            text,
+            findings,
+            fullOptions,
+            (decoded, recursiveOptions) =>
+              depth < 2
+                ? sanitizeText(decoded, depth + 1, true, true, recursiveOptions)
+                : Promise.resolve({ text: '[REDACTED]' })
+          )
+        ).text;
+    // Verification at the publication boundary runs every enabled engine.
+    if (
+      (await analyze(sanitized, [], plain, options)).some((f) =>
+        actionable(f, options)
+      )
+    ) {
+      return redactAgain(result, findings, options, round, (next, nextRound) =>
+        sanitizeText(result.text, depth, fullRedaction, plain, next, nextRound)
+      );
+    }
+    if (options.structured && !plain) {
+      verifyStructured(result.text, options);
+    }
+    return { ...result, findings: auditFaked(findings, options, result) };
+  }
+}
+
+function eligiblePublic(text, finding, findings) {
+  return (
+    finding.category !== 'credential' &&
+    !['known-personal', 'confirmed-personal'].includes(finding.rule) &&
+    ['PERSON', 'ORGANIZATION', 'EMAIL'].includes(finding.type) &&
+    !findings.some(
+      (f) =>
+        (f.category === 'credential' ||
+          ['known-personal', 'confirmed-personal'].includes(f.rule)) &&
+        f.start < finding.end &&
+        f.end > finding.start
+    ) &&
+    !/(?:patient|customer|employee|my name|пациент|клиент)\s*[:=]?\s*$/iu.test(
+      text.slice(Math.max(0, finding.start - 64), finding.start)
+    )
+  );
+}
+async function verifyFindings(text, findings, verifier) {
+  const retained = [];
+  for (const finding of findings) {
+    if (!eligiblePublic(text, finding, findings)) {
+      retained.push(finding);
+      continue;
+    }
+    let verified;
+    try {
+      verified = await verifier({
+        type: finding.type,
+        value: text.slice(finding.start, finding.end),
+      });
+    } catch {
+      throw failure('ERR_PUBLIC_VERIFICATION');
+    }
+    if (!verified) {
+      retained.push(finding);
+      continue;
+    }
+    validatePublic(verified);
+    if (
+      verified.type !== finding.type ||
+      verified.value !== text.slice(finding.start, finding.end)
+    ) {
+      throw failure('ERR_PUBLIC_VERIFICATION');
+    }
+  }
+  return retained;
+}

package/src/entities.js

@@ -0,0 +1,166 @@
+import google from './vendor/entities/google.json' with { type: 'json' };
+import azure from './vendor/entities/azure.json' with { type: 'json' };
+import aws from './vendor/entities/aws.json' with { type: 'json' };
+import { escapePattern, collectMatches } from './detection.js';
+import { numericNoise, epochNumber } from './confidence.js';
+import { valueRange, personalValueAllowed } from './context.js';
+export const entityCatalogs = { google, azure, aws };
+const canonical = (name) =>
+  name.replace(/([a-z])([A-Z])/g, '$1_$2').toUpperCase();
+const entries = new Map();
+for (const [vendor, catalog] of Object.entries(entityCatalogs)) {
+  for (const name of catalog.names) {
+    const normalized = canonical(name);
+    const type =
+      {
+        NAME: 'PERSON',
+        EMAIL_ADDRESS: 'EMAIL',
+        PHONE_NUMBER: 'PHONE',
+        PERSON_NAME: 'PERSON',
+        USER_NAME: 'USERNAME',
+      }[normalized] ?? normalized;
+    if (type.length > 64) {
+      continue;
+    }
+    const entry = {
+      type,
+      rule: `catalog/${vendor}`,
+      category:
+        /PASSWORD|SECRET|TOKEN|AUTH|ACCESS_KEY|PRIVATE_KEY|CREDENTIAL|CONNECTION.*STRING|^PIN$|CVV|CVC/.test(
+          type
+        )
+          ? 'credential'
+          : 'personal',
+    };
+    entries.set(normalized, entry);
+    entries.set(name.toUpperCase(), entry);
+  }
+}
+// All documented names are native labelled recognizers; automatic format
+// recognizers are separate and do not claim the vendors' model behavior.
+const prefix = new RegExp(
+  `(?<![\\p{L}\\p{N}_])(?:["'])?(${[...entries.keys()].map(escapePattern).join('|')})(?:["'])?[ \\t]*[:=：][ \\t]*`,
+  'giu'
+);
+export function detectCatalog(text, emit) {
+  // Every catalog label requires one of these separators. Avoid running the
+  // large Unicode label alternation across ordinary unlabelled log records.
+  if (/[:=：]/.test(text)) {
+    detectCatalogLabels(text, emit);
+  }
+  detectFormats(text, emit);
+}
+function detectCatalogLabels(text, emit) {
+  for (const match of text.matchAll(prefix)) {
+    const entry = entries.get(match[1].toUpperCase());
+    if (entry.type === 'DATE') {
+      continue;
+    }
+    const mode = /[?&]$/.test(text.slice(0, match.index))
+      ? 'query'
+      : match[0].includes('=') || /["']\s*[:：]/.test(match[0])
+        ? 'token'
+        : 'line';
+    const range = valueRange(
+      text,
+      match.index + match[0].length,
+      mode,
+      entry.category === 'personal' ? 512 : Infinity
+    );
+    if (
+      range &&
+      (entry.category === 'credential' ||
+        personalValueAllowed(text, range, entry.type, match.index)) &&
+      text.slice(range.start, range.end).trim() &&
+      !text.slice(range.start, range.end).startsWith('[REDACTED]')
+    ) {
+      emit({ ...range, ...entry, confidence: 0.95 });
+    }
+  }
+}
+function detectFormats(text, emit) {
+  for (const [type, pattern, accept] of formatRules) {
+    collectMatches(
+      text,
+      pattern,
+      (finding) => {
+        if (type === 'US_HEALTHCARE_NPI') {
+          const context = /\b(?:NPI|provider|healthcare|patient)\b/i.test(
+            text.slice(Math.max(0, finding.start - 64), finding.end + 32)
+          );
+          if (
+            !context &&
+            (numericNoise(text, finding.start, finding.end) ||
+              epochNumber(text.slice(finding.start, finding.end)))
+          ) {
+            return;
+          }
+          finding.confidence = context ? 0.95 : 0.35;
+        }
+        emit(finding);
+      },
+      { type, category: 'personal', rule: 'entity-format', confidence: 0.85 },
+      0,
+      accept
+    );
+  }
+}
+function cpf(value) {
+  const d = value.replace(/\D/g, '');
+  if (/^(\d)\1+$/.test(d)) {
+    return false;
+  }
+  for (const n of [9, 10]) {
+    let sum = 0;
+    for (let i = 0; i < n; i++) {
+      sum += Number(d[i]) * (n + 1 - i);
+    }
+    if (((sum * 10) % 11) % 10 !== Number(d[n])) {
+      return false;
+    }
+  }
+  return true;
+}
+function npi(value) {
+  if (!/^[12]\d{9}$/.test(value)) {
+    return false;
+  }
+  const digits = `80840${value}`;
+  let sum = 0;
+  for (
+    let i = digits.length - 1, doubled = false;
+    i >= 0;
+    i--, doubled = !doubled
+  ) {
+    let d = Number(digits[i]);
+    if (doubled) {
+      d = d * 2 > 9 ? d * 2 - 9 : d * 2;
+    }
+    sum += d;
+  }
+  return sum % 10 === 0;
+}
+const formatRules = [
+  ['BRAZIL_CPF_NUMBER', /\b\d{3}\.\d{3}\.\d{3}-\d{2}\b/g, cpf],
+  [
+    'INDIA_PAN_INDIVIDUAL',
+    /\b[A-Z]{3}[ABCFGHLJPT][A-Z]\d{4}[A-Z]\b/g,
+    () => true,
+  ],
+  [
+    'UK_NATIONAL_INSURANCE_NUMBER',
+    /\b(?!BG|GB|NK|KN|TN|NT|ZZ)[A-CEGHJ-PR-TW-Z]{2}[ ]?\d{2}[ ]?\d{2}[ ]?\d{2}[ ]?[A-D]\b/g,
+    () => true,
+  ],
+  [
+    'SPAIN_DNI_NUMBER',
+    /\b\d{8}[A-Z]\b/g,
+    (v) => 'TRWAGMYFPDXBNJZSQVHLCKE'[Number(v.slice(0, 8)) % 23] === v.at(-1),
+  ],
+  [
+    'CRYPTO_WALLET',
+    /\b(?:0x[a-fA-F0-9]{40}|[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[ac-hj-np-z02-9]{25,62})\b/g,
+    () => true,
+  ],
+  ['US_HEALTHCARE_NPI', /\b[12]\d{9}\b/g, npi],
+];

package/src/escape-spans.js

@@ -0,0 +1,20 @@
+// Never cut a JSON escape in half, including when plain text contains JSON.
+// Each endpoint is visited once after sorting, even for overlapping findings.
+export function alignEscapes(text, findings) {
+  const result = findings.map((finding) => ({ ...finding }));
+  for (const boundary of ['start', 'end']) {
+    const ordered = [...result].sort((a, b) => a[boundary] - b[boundary]);
+    let index = 0;
+    for (const match of text.matchAll(/\\(?:u[\da-fA-F]{4}|["\\/bfnrt])/g)) {
+      const start = match.index,
+        end = start + match[0].length;
+      while (index < ordered.length && ordered[index][boundary] <= start) {
+        index++;
+      }
+      while (index < ordered.length && ordered[index][boundary] < end) {
+        ordered[index++][boundary] = boundary === 'start' ? start : end;
+      }
+    }
+  }
+  return result;
+}

package/src/eslint-plugin.d.ts

@@ -0,0 +1,10 @@
+declare const plugin: {
+  rules: Record<
+    string,
+    {
+      meta: object;
+      create(context: any): Record<string, (...args: any[]) => void>;
+    }
+  >;
+};
+export default plugin;