@link-foundation/sensitive-data-sanitizer 0.0.0-stage → 0.12.0

package/src/structured.js

@@ -0,0 +1,278 @@
+import { failure } from './detection.js';
+
+// Session metadata is opt-in at the CLI; library callers choose their own
+// fields. These names do not grant exemptions to provider/known credentials.
+export const sessionStructuralFields = ['id', 'type', 'name', 'description'];
+
+// Advance once through each character. A regexp string matcher can repeatedly
+// restart at escaped quotes in a damaged token and become quadratic.
+function* lexemes(text) {
+  for (let index = 0; index < text.length; index++) {
+    const char = text[index];
+    if (char === '"') {
+      const start = index;
+      index++;
+      while (index < text.length && text[index] !== '"') {
+        index += text[index] === '\\' ? 2 : 1;
+      }
+      if (index >= text.length) {
+        throw failure('ERR_JSON');
+      }
+      let next = index + 1;
+      while (next < text.length && /\s/.test(text[next])) {
+        next++;
+      }
+      yield {
+        raw: text.slice(start, index + 1),
+        index: start,
+        key: text[next] === ':',
+      };
+    } else if ('{}[],:'.includes(char)) {
+      yield { raw: char, index, key: false };
+    }
+  }
+}
+
+function tokens(text, offset = 0) {
+  const strings = [];
+  const stack = [{ field: '' }];
+  for (const match of lexemes(text)) {
+    const raw = match.raw,
+      frame = stack.at(-1);
+    if (raw[0] === '"') {
+      const key = match.key;
+      const value = JSON.parse(raw);
+      if (key) {
+        frame.field = value;
+      }
+      strings.push({
+        raw,
+        start: offset + match.index,
+        field: key ? '' : frame.field,
+        key,
+      });
+    } else if (raw === '{' || raw === '[') {
+      stack.push({ field: frame.field });
+    } else if (raw === '}' || raw === ']') {
+      stack.pop();
+    }
+  }
+  return strings;
+}
+export function structuredBatches(text, options) {
+  const leaves = [];
+  try {
+    if (options.structured === 'json') {
+      JSON.parse(text);
+      for (const leaf of tokens(text)) {
+        leaves.push(leaf);
+      }
+    } else {
+      let offset = 0;
+      for (const line of text.split('\n')) {
+        if (line.trim()) {
+          JSON.parse(line);
+          for (const leaf of tokens(line, offset)) {
+            leaves.push(leaf);
+          }
+        }
+        offset += line.length + 1;
+      }
+    }
+  } catch {
+    throw failure('ERR_JSON');
+  }
+  const batches = [];
+  let parts = [],
+    entries = [],
+    length = 0;
+  const limit = Math.min(
+    256 * 1024,
+    options.maxInputLength ?? 10 * 1024 * 1024
+  );
+  for (const leaf of leaves) {
+    // Keeping the original escaped token gives exact source offsets without
+    // exposing values in finding metadata. Separate keys are scanned too.
+    const prefix = leaf.field
+      ? `${JSON.stringify(leaf.field.slice(0, 128))}: `
+      : '';
+    const part = `${prefix + leaf.raw}\n`;
+    if (length && length + part.length > limit) {
+      batches.push({ text: parts.join('').slice(0, -1), entries });
+      parts = [];
+      entries = [];
+      length = 0;
+    }
+    entries.push({ ...leaf, batchStart: length + prefix.length });
+    parts.push(part);
+    length += part.length;
+  }
+  if (parts.length) {
+    batches.push({ text: parts.join('').slice(0, -1), entries });
+  }
+  return batches;
+}
+
+export function structuredValueRanges(text, options) {
+  return structuredBatches(text, options).flatMap((batch) =>
+    batch.entries
+      .filter((entry) => !entry.key)
+      .map((entry) => ({
+        start: entry.start + 1,
+        end: entry.start + entry.raw.length - 1,
+      }))
+  );
+}
+
+function weakStructuralFinding(finding, entry, batchText) {
+  if (
+    ['known-secret', 'known-personal', 'confirmed-personal'].includes(
+      finding.rule
+    )
+  ) {
+    return false;
+  }
+  if (finding.category === 'personal') {
+    return ['PERSON', 'ORGANIZATION', 'USERNAME', 'ID'].includes(finding.type);
+  }
+  if (
+    ['quoted-entropy', 'entropy', 'gitleaks/generic-api-key'].includes(
+      finding.rule
+    )
+  ) {
+    return true;
+  }
+  if (finding.rule === 'context') {
+    const before = batchText.slice(
+      Math.max(entry.batchStart, finding.start - 80),
+      finding.start
+    );
+    // Generic "secret: ..." prose in skill descriptions is metadata; actual
+    // password/auth context remains sensitive even in declared fields.
+    return /\bsecret\s*[:=]\s*["']?$/i.test(before);
+  }
+  return false;
+}
+function ordinaryPersonalKey(entry, finding) {
+  return (
+    entry.key &&
+    /^[A-Za-z_$][\w$]*$/.test(JSON.parse(entry.raw)) &&
+    finding.category === 'personal' &&
+    finding.rule !== 'known-personal'
+  );
+}
+// Propagation runs again at source offsets. Apply the same key policy after it
+// so a name confirmed in a value cannot rename an ordinary identifier key.
+export function preserveIdentifierKeys(text, findings, options) {
+  if (!options.structured) {
+    return findings;
+  }
+  const keys = structuredBatches(text, options).flatMap((batch) =>
+    batch.entries
+      .filter((entry) => ordinaryPersonalKey(entry, { category: 'personal' }))
+      .map((entry) => ({
+        start: entry.start,
+        end: entry.start + entry.raw.length,
+      }))
+  );
+  return findings.filter((finding) => {
+    if (finding.category !== 'personal' || finding.rule === 'known-personal') {
+      return true;
+    }
+    let low = 0,
+      high = keys.length;
+    while (low < high) {
+      const mid = (low + high) >>> 1;
+      if (keys[mid].end <= finding.start) {
+        low = mid + 1;
+      } else {
+        high = mid;
+      }
+    }
+    return low === keys.length || keys[low].start >= finding.end;
+  });
+}
+function preserveStructuredEntry(entry, finding, text, options) {
+  return (
+    (entry.key &&
+      ['confirmed-personal', 'document-propagation'].includes(finding.rule)) ||
+    ordinaryPersonalKey(entry, finding) ||
+    (!entry.key &&
+      options.structuralFields?.includes(entry.field) &&
+      weakStructuralFinding(finding, entry, text))
+  );
+}
+export function mapStructuredFindings(batch, findings, options) {
+  const result = [];
+  for (const finding of findings) {
+    let low = 0,
+      high = batch.entries.length;
+    while (low < high) {
+      const mid = (low + high) >>> 1,
+        entry = batch.entries[mid];
+      if (entry.batchStart + entry.raw.length - 1 <= finding.start) {
+        low = mid + 1;
+      } else {
+        high = mid;
+      }
+    }
+    for (let i = low; i < batch.entries.length; i++) {
+      const entry = batch.entries[i];
+      const lo = entry.batchStart + 1,
+        hi = entry.batchStart + entry.raw.length - 1;
+      if (lo >= finding.end) {
+        break;
+      }
+      if (finding.end <= lo || finding.start >= hi) {
+        continue;
+      }
+      if (preserveStructuredEntry(entry, finding, batch.text, options)) {
+        continue;
+      }
+      const start = Math.max(lo, finding.start),
+        end = Math.min(hi, finding.end);
+      if (end <= start) {
+        continue;
+      }
+      // A detector that crosses a token boundary cannot safely describe a
+      // partial string. Redact that string's complete content instead.
+      const crosses = finding.start < lo || finding.end > hi;
+      result.push({
+        ...finding,
+        start: entry.start + (crosses ? 1 : start - entry.batchStart),
+        end:
+          entry.start +
+          (crosses ? entry.raw.length - 1 : end - entry.batchStart),
+      });
+    }
+  }
+  return result;
+}
+export function verifyStructured(text, options) {
+  structuredBatches(text, options);
+  // JSON.parse silently overwrites duplicate keys; prevent loss of meaning
+  // when distinct personal keys collapse to the same redaction marker.
+  const records =
+    options.structured === 'json'
+      ? [text]
+      : text.split('\n').filter((line) => line.trim());
+  for (const record of records) {
+    const stack = [];
+    for (const match of lexemes(record)) {
+      if (match.raw === '{') {
+        stack.push(new Set());
+      } else if (match.raw === '[') {
+        stack.push(undefined);
+      } else if (match.raw === '}' || match.raw === ']') {
+        stack.pop();
+      } else if (match.key) {
+        const key = JSON.parse(match.raw),
+          keys = stack.at(-1);
+        if (keys?.has(key)) {
+          throw failure('ERR_JSON_COLLISION');
+        }
+        keys?.add(key);
+      }
+    }
+  }
+}

package/src/transforms.js

@@ -0,0 +1,192 @@
+import { realisticFake } from './fake.js';
+import { createHmac } from 'node:crypto';
+import { failure, REDACTED } from './detection.js';
+import { encodedValue, encodeRun } from './encoded.js';
+import { isIdentityType } from './identity.js';
+
+const modes = new Set([
+  'redact',
+  'fake',
+  'hive-mask',
+  'mask',
+  'pseudonym',
+  'format-preserving',
+  'date-shift',
+  'bucket',
+]);
+export function validateTransforms(options) {
+  if (
+    options.preserveEncoding !== undefined &&
+    typeof options.preserveEncoding !== 'boolean'
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  if (
+    options.transformations !== undefined &&
+    (!options.transformations ||
+      typeof options.transformations !== 'object' ||
+      Array.isArray(options.transformations))
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  for (const transform of [
+    options.transformation,
+    ...Object.values(options.transformations ?? {}),
+  ].filter((t) => t !== undefined)) {
+    validateTransform(transform);
+  }
+}
+function positiveInteger(value) {
+  return Number.isSafeInteger(value) && value > 0;
+}
+function validateTransform(transform) {
+  if (!transform || !modes.has(transform.mode)) {
+    throw failure('ERR_CONFIG');
+  }
+  const checks = {
+    fake: () =>
+      typeof transform.key === 'string' &&
+      Buffer.byteLength(transform.key) >= 16 &&
+      (transform.credentials === undefined ||
+        typeof transform.credentials === 'boolean') &&
+      (transform.mrzCountry === undefined || transform.mrzCountry === 'UTO'),
+    pseudonym: () =>
+      typeof transform.key === 'string' &&
+      Buffer.byteLength(transform.key) >= 16,
+    'format-preserving': () =>
+      typeof transform.key === 'string' &&
+      Buffer.byteLength(transform.key) >= 16,
+    'date-shift': () =>
+      Number.isSafeInteger(transform.days) &&
+      Math.abs(transform.days) <= 365000,
+    bucket: () => positiveInteger(transform.size),
+    mask: () =>
+      [transform.keepStart ?? 0, transform.keepEnd ?? 0].every(
+        (value) => Number.isSafeInteger(value) && value >= 0
+      ) &&
+      (transform.minLength === undefined ||
+        positiveInteger(transform.minLength)) &&
+      (transform.marker === undefined ||
+        (typeof transform.marker === 'string' &&
+          transform.marker.length > 0 &&
+          transform.marker.length <= 64)),
+  };
+  if (checks[transform.mode] && !checks[transform.mode]()) {
+    throw failure('ERR_CONFIG');
+  }
+}
+
+function keyed(value, type, key, suffix = '') {
+  return createHmac('sha256', key)
+    .update(`${type}\0${value}\0${suffix}`)
+    .digest();
+}
+function formatPseudonym(value, type, key) {
+  const result = Array.from(value, (char, i) => {
+    const alphabet = /\d/.test(char)
+      ? '0123456789'
+      : /[A-Z]/.test(char)
+        ? 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+        : /[a-z]/.test(char)
+          ? 'abcdefghijklmnopqrstuvwxyz'
+          : /\p{L}/u.test(char)
+            ? 'abcdefghijklmnopqrstuvwxyz'
+            : undefined;
+    return alphabet
+      ? alphabet[keyed(value, type, key, String(i))[0] % alphabet.length]
+      : char;
+  }).join('');
+  return result === value ? REDACTED : result;
+}
+function shiftedDate(value, days) {
+  const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
+  if (!match) {
+    return REDACTED;
+  }
+  const date = new Date(`${value}T00:00:00.000Z`);
+  if (
+    Number.isNaN(date.getTime()) ||
+    date.toISOString().slice(0, 10) !== value
+  ) {
+    return REDACTED;
+  }
+  date.setUTCDate(date.getUTCDate() + days);
+  return date.getUTCFullYear() < 0 || date.getUTCFullYear() > 9999
+    ? REDACTED
+    : date.toISOString().slice(0, 10);
+}
+function masked(value, finding, transform) {
+  const chars = Array.from(value),
+    start = transform.keepStart ?? 0,
+    end = transform.keepEnd ?? 0;
+  return chars.length <
+    (transform.minLength ?? (isIdentityType(finding.type) ? 5 : 0)) ||
+    start + end >= chars.length
+    ? REDACTED
+    : `${chars.slice(0, start).join('')}${transform.marker ?? '*'.repeat(chars.length - start - end)}${end ? chars.slice(-end).join('') : ''}`;
+}
+function transformed(value, finding, transform) {
+  if (!transform || transform.mode === 'redact') {
+    return REDACTED;
+  }
+  if (transform.mode === 'hive-mask') {
+    const chars = Array.from(value);
+    return chars.length > 12
+      ? `${chars.slice(0, 3).join('')}…${chars.slice(-3).join('')}`
+      : REDACTED;
+  }
+  if (transform.mode === 'fake') {
+    return realisticFake(value, finding, transform);
+  }
+  if (finding.category === 'credential') {
+    return REDACTED;
+  }
+  if (transform.mode === 'mask') {
+    return masked(value, finding, transform);
+  }
+  if (transform.mode === 'pseudonym') {
+    return `[PSEUDONYM_${keyed(value, finding.type, transform.key).toString('hex').slice(0, 24)}]`;
+  }
+  if (transform.mode === 'format-preserving') {
+    return formatPseudonym(value, finding.type, transform.key);
+  }
+  if (transform.mode === 'date-shift') {
+    return shiftedDate(value, transform.days);
+  }
+  const number = Number(value);
+  if (!Number.isSafeInteger(number)) {
+    return REDACTED;
+  }
+  const lower = Math.floor(number / transform.size) * transform.size;
+  return `${lower}–${lower + transform.size - 1}`;
+}
+export function renderReplacement(text, finding, options, encodedSanitize) {
+  const transform =
+    options.transformations?.[finding.type] ??
+    options.transformation ??
+    (options.identityMask && isIdentityType(finding.type)
+      ? { mode: 'mask', keepStart: 2, keepEnd: 2, marker: '***', minLength: 5 }
+      : undefined);
+  if (finding.type === 'ENCODED_SENSITIVE') {
+    const raw = text.slice(finding.start, finding.end);
+    const run = encodedValue(raw);
+    if (run && (options.preserveEncoding || run.encoding === 'json-content')) {
+      return encodeRun(run, encodedSanitize(run.text).text);
+    }
+    if (options.preserveEncoding) {
+      throw failure('ERR_ENCODING');
+    }
+  }
+  let raw = text.slice(finding.start, finding.end);
+  const jsonQuoted =
+    text[finding.start - 1] === '"' && text[finding.end] === '"';
+  if (jsonQuoted) {
+    try {
+      raw = JSON.parse(`"${raw}"`);
+    } catch {
+      return REDACTED;
+    }
+  }
+  const result = transformed(raw, finding, transform);
+  return jsonQuoted ? JSON.stringify(result).slice(1, -1) : result;
+}

package/src/transliteration.js

@@ -0,0 +1,75 @@
+import { firstNames } from './names.js';
+// ICAO 9303 spelling and common passport/GOST variants. Transliteration is
+// lossy: aliases are detection candidates, never proof of a person's identity.
+const cyrillic = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя';
+const tables = [
+  'a b v g d e e zh z i i k l m n o p r s t u f kh ts ch sh shch ie y - e iu ia',
+  'a b v g d e yo zh z i y k l m n o p r s t u f kh ts ch sh shch - y - e yu ya',
+  'a b v g d e jo zh z i j k l m n o p r s t u f x cz ch sh shh - y - eh ju ja',
+  'a b v g d e ë ž z i j k l m n o p r s t u f h c č š ŝ ʺ y ʹ è û â',
+].map((t) =>
+  Object.fromEntries(
+    [...cyrillic].map((c, i) => [c, t.split(' ')[i].replace('-', '')])
+  )
+);
+const inverseTables = tables.map((table) =>
+  [...Object.entries(table), ['кс', 'x']]
+    .filter(([, spelling]) => spelling)
+    .sort((a, b) => b[1].length - a[1].length)
+);
+function spellings(lower) {
+  const values = new Set([lower]);
+  for (const table of tables) {
+    const latin = [...lower].map((c) => table[c] ?? c).join('');
+    values.add(latin);
+    values.add(latin.replaceAll('ks', 'x'));
+  }
+  return values;
+}
+const equivalents = new Map();
+for (const name of `${firstNames.ru} Олеся`.toLowerCase().split(' ')) {
+  const variants = spellings(name);
+  for (const variant of variants) {
+    equivalents.set(variant, variants);
+  }
+}
+export const transliteratedGivenNames = [...equivalents.keys()].filter((v) =>
+  /^[a-z]+$/.test(v)
+);
+export function nameAliases(value) {
+  const lower = value.normalize('NFC').toLowerCase();
+  const values = new Set([lower]);
+  if (/\p{Script=Cyrillic}/u.test(lower)) {
+    for (const spelling of spellings(lower)) {
+      values.add(spelling);
+    }
+  } else if (
+    /^[\p{Script=Latin}ʺʹ]+(?:[ _-][\p{Script=Latin}ʺʹ]+)*$/u.test(lower)
+  ) {
+    // Greedy inverse for each passport/GOST table, with bounded ambiguity.
+    for (const entries of inverseTables) {
+      let inverse = '',
+        index = 0;
+      while (index < lower.length) {
+        const entry = entries.find(
+          ([c, s]) =>
+            (s !== 'y' ||
+              c === (/[aeiou]/.test(lower[index - 1] ?? '') ? 'й' : 'ы')) &&
+            lower.startsWith(s, index)
+        );
+        inverse += entry?.[0] ?? lower[index];
+        index += entry?.[1].length ?? 1;
+      }
+      for (const spelling of spellings(inverse)) {
+        values.add(spelling);
+      }
+    }
+  }
+  // The same table-derived equivalences cover the whole Cyrillic gazetteer.
+  for (const value of [...values]) {
+    for (const variant of equivalents.get(value) ?? []) {
+      values.add(variant);
+    }
+  }
+  return [...values].filter(Boolean);
+}

package/src/utf8.js

@@ -0,0 +1,60 @@
+import { failure } from './detection.js';
+
+export function sequenceWidth(first) {
+  if (first < 0x80) {
+    return 1;
+  }
+  if (first >= 0xc2 && first <= 0xdf) {
+    return 2;
+  }
+  if (first >= 0xe0 && first <= 0xef) {
+    return 3;
+  }
+  if (first >= 0xf0 && first <= 0xf4) {
+    return 4;
+  }
+  return 0;
+}
+function secondByteAllowed(first, byte) {
+  const limits = {
+    0xe0: [0xa0, 0xbf],
+    0xed: [0x80, 0x9f],
+    0xf0: [0x90, 0xbf],
+    0xf4: [0x80, 0x8f],
+  };
+  const [low, high] = limits[first] ?? [0x80, 0xbf];
+  return byte >= low && byte <= high;
+}
+export function validSequence(bytes, index, width) {
+  if (!width || index + width > bytes.length) {
+    return false;
+  }
+  for (let next = 1; next < width; next++) {
+    const byte = bytes[index + next];
+    if (
+      byte < 0x80 ||
+      byte > 0xbf ||
+      (next === 1 && !secondByteAllowed(bytes[index], byte))
+    ) {
+      return false;
+    }
+  }
+  return true;
+}
+// Called only after fatal UTF-8 decoding fails. Diagnostics contain positions,
+// never input bytes or personal values. Offsets identify the sequence start.
+export function encodingFailure(bytes, record, baseOffset) {
+  let index = 0;
+  while (index < bytes.length) {
+    const width = sequenceWidth(bytes[index]);
+    if (!validSequence(bytes, index, width)) {
+      break;
+    }
+    index += width;
+  }
+  return failure('ERR_ENCODING', {
+    record,
+    offset: baseOffset + index,
+    recordOffset: index,
+  });
+}

package/src/vendor/entities/aws.json

@@ -0,0 +1,42 @@
+{
+  "source": "https://docs.aws.amazon.com/comprehend/latest/dg/how-pii.html",
+  "sha256": "0ac728632444f43e09b9079c709558c8237d732e7bcb3c8199c4dd3443f9c6a0",
+  "names": [
+    "ADDRESS",
+    "AGE",
+    "AWS_ACCESS_KEY",
+    "AWS_SECRET_KEY",
+    "BANK_ACCOUNT_NUMBER",
+    "BANK_ROUTING",
+    "CA_HEALTH_NUMBER",
+    "CA_SOCIAL_INSURANCE_NUMBER",
+    "CREDIT_DEBIT_CVV",
+    "CREDIT_DEBIT_EXPIRY",
+    "CREDIT_DEBIT_NUMBER",
+    "DATE_TIME",
+    "DRIVER_ID",
+    "EMAIL",
+    "INTERNATIONAL_BANK_ACCOUNT_NUMBER",
+    "IN_AADHAAR",
+    "IN_NREGA",
+    "IN_PERMANENT_ACCOUNT_NUMBER",
+    "IN_VOTER_NUMBER",
+    "IP_ADDRESS",
+    "LICENSE_PLATE",
+    "MAC_ADDRESS",
+    "NAME",
+    "PASSPORT_NUMBER",
+    "PASSWORD",
+    "PHONE",
+    "PIN",
+    "SSN",
+    "SWIFT_CODE",
+    "UK_NATIONAL_HEALTH_SERVICE_NUMBER",
+    "UK_NATIONAL_INSURANCE_NUMBER",
+    "UK_UNIQUE_TAXPAYER_REFERENCE_NUMBER",
+    "URL",
+    "USERNAME",
+    "US_INDIVIDUAL_TAX_IDENTIFICATION_NUMBER",
+    "VEHICLE_IDENTIFICATION_NUMBER"
+  ]
+}