@link-foundation/sensitive-data-sanitizer 0.0.0-stage → 0.11.31

package/src/gitleaks.js

@@ -0,0 +1,194 @@
+import { RE2 } from 're2-wasm';
+import { RE2JS } from 're2js';
+import { performance } from 'node:perf_hooks';
+import r0 from './vendor/gitleaks/rules-0.json' with { type: 'json' };
+import r1 from './vendor/gitleaks/rules-1.json' with { type: 'json' };
+import r2 from './vendor/gitleaks/rules-2.json' with { type: 'json' };
+import r3 from './vendor/gitleaks/rules-3.json' with { type: 'json' };
+import r4 from './vendor/gitleaks/rules-4.json' with { type: 'json' };
+import r5 from './vendor/gitleaks/rules-5.json' with { type: 'json' };
+import { entropy, REDACTED } from './detection.js';
+import { keywordMatcher } from './keywords.js';
+
+export const gitleaksRules = [...r0, ...r1, ...r2, ...r3, ...r4, ...r5];
+const compiled = new Map();
+const largePatterns = new Map();
+const keywordIndex = keywordMatcher(
+  gitleaksRules.flatMap((r) =>
+    (r.keywords ?? []).map((word) => word.toLowerCase())
+  )
+);
+
+export function detectGitleaks(text, emit, trace) {
+  const keywords = keywordIndex(text.toLowerCase());
+  const large = Buffer.byteLength(text) > 4 * 1024 * 1024;
+  for (const rule of gitleaksRules) {
+    if (
+      !rule.regex ||
+      (rule.keywords?.length &&
+        !rule.keywords.some((word) => keywords.has(word.toLowerCase())))
+    ) {
+      continue;
+    }
+    const started = trace ? performance.now() : 0;
+    for (const view of ruleViews(text, rule)) {
+      scanRule(view.text, view.start, rule, emit, large);
+    }
+    trace?.({
+      rule: rule.id,
+      length: text.length,
+      ms: performance.now() - started,
+    });
+  }
+}
+
+function* ruleViews(text, rule) {
+  // These bounded rules require a listed keyword within 300 characters of
+  // every match. Local views avoid RE2's expensive search/copy over unrelated
+  // multi-megabyte image data. Keep the unbounded catalog rules intact.
+  const boundedPrefix = {
+    'authress-service-client-access-key': '\\b(?:sc|ext|scauth|authress)_',
+    'atlassian-api-token': '(?:atlassian|confluence|jira)',
+    'etsy-access-token': 'etsy',
+    'new-relic-user-api-key': '(?:new-relic|newrelic|new_relic)',
+    'okta-access-token': 'okta',
+    'snyk-api-token': 'snyk[_.-]?(?:(?:api|oauth)[_.-]?)?(?:key|token)',
+  }[rule.id];
+  if (boundedPrefix) {
+    yield* boundedViews(text, boundedPrefix, rule.id);
+    return;
+  }
+  const targeted = {
+    'generic-api-key':
+      /(?:access|auth|api|credential|creds|key|passw(?:or)?d|secret|token)[ \t\w.-]{0,20}[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)/gi,
+    'hashicorp-tf-password': /administrator_login_password|password/gi,
+    'sonar-api-token': /sonar[_.-]?(?:login|token)/gi,
+  }[rule.id];
+  if (!targeted) {
+    yield { text, start: 0 };
+    return;
+  }
+  // These vendored rules have <=50 prefix characters and <=31 characters
+  // between their required keyword and secret. Scan near each keyword instead
+  // of recopying an entire session into WASM for every match. Extend the only
+  // unbounded secret alternative to its real terminator; never truncate it.
+  let coveredEnd = -1;
+  for (const match of text.matchAll(targeted)) {
+    // A keyword inside an already scanned value must not copy/rescan its
+    // complete suffix. Base64 can contain thousands of incidental keywords.
+    if (match.index + 384 <= coveredEnd || coveredEnd === text.length) {
+      continue;
+    }
+    let start = Math.max(0, match.index - 50),
+      end = Math.min(text.length, match.index + 384);
+    if (/[\uDC00-\uDFFF]/.test(text[start] ?? '')) {
+      start--;
+    }
+    while (end < text.length && /[\w.=/+-]/.test(text[end])) {
+      end++;
+    }
+    end = Math.min(text.length, end + 3);
+    coveredEnd = end;
+    yield { text: text.slice(start, end), start };
+  }
+}
+
+function* boundedViews(text, prefix, id) {
+  const words = new RegExp(
+    id === 'authress-service-client-access-key'
+      ? prefix
+      : `${prefix}[ \\t\\w.-]{0,20}[\\s'"]{0,3}(?:=|>|:{1,3}=|\\|\\||:|=>|\\?=|,)${id === 'atlassian-api-token' ? '|ATATT3' : ''}`,
+    'gi'
+  );
+  let covered = -1;
+  for (const match of text.matchAll(words)) {
+    if (match.index + 512 <= covered) {
+      continue;
+    }
+    let start = Math.max(0, match.index - 512);
+    let end = Math.min(text.length, match.index + 1024);
+    if (/[\uDC00-\uDFFF]/.test(text[start] ?? '')) {
+      start--;
+    }
+    if (/[\uDC00-\uDFFF]/.test(text[end] ?? '')) {
+      end++;
+    }
+    covered = end;
+    yield { text: text.slice(start, end), start };
+  }
+}
+
+function scanRule(text, base, rule, emit, large) {
+  // re2-wasm 1.0.2 returns codepoint indices despite its RegExp-like API.
+  // Convert once per view, and keep its search cursor in codepoints too.
+  let offsets;
+  if (!large && /[\uD800-\uDBFF]/.test(text)) {
+    offsets = [0];
+    let offset = 0;
+    for (const char of text) {
+      offset += char.length;
+      offsets.push(offset);
+    }
+  }
+  for (const match of ruleMatches(text, rule, large)) {
+    const secret = capturedSecret(rule, match);
+    if (!secret) {
+      continue;
+    }
+    // RE2's wrapper exposes full-match offsets, not subgroup indices. If the
+    // span includes each occurrence when a captured substring is repeated.
+    const offset = match[0].indexOf(secret),
+      last = match[0].lastIndexOf(secret);
+    emit({
+      start: base + (offsets ? offsets[match.index] : match.index) + offset,
+      end:
+        base +
+        (offsets ? offsets[match.index] : match.index) +
+        last +
+        secret.length,
+      type: 'SECRET',
+      category: 'credential',
+      rule: `gitleaks/${rule.id}`,
+      confidence: rule.entropy ? 0.85 : 0.95,
+    });
+  }
+}
+
+function* ruleMatches(text, rule, large) {
+  if (large) {
+    // re2-wasm has a fixed 16 MiB heap. RE2JS implements the same linear-time
+    // syntax without copying the input into WASM. Its indices are UTF-16.
+    let linear = largePatterns.get(rule.id);
+    if (!linear) {
+      linear = RE2JS.compile(rule.regex);
+      largePatterns.set(rule.id, linear);
+    }
+    if (linear.test(text)) {
+      yield* linear.matchAll(text);
+    }
+    return;
+  }
+  let pattern = compiled.get(rule.id);
+  if (!pattern) {
+    pattern = new RE2(rule.regex, 'gu');
+    compiled.set(rule.id, pattern);
+  }
+  pattern.lastIndex = 0;
+  let match;
+  while ((match = pattern.exec(text)) !== null) {
+    pattern.lastIndex =
+      match.index + Array.from(match[0]).length || pattern.lastIndex + 1;
+    yield match;
+  }
+}
+
+function capturedSecret(rule, match) {
+  const secret = rule.secretGroup
+    ? match[rule.secretGroup]
+    : (match.slice(1).find(Boolean) ?? match[0]);
+  return !secret ||
+    secret === REDACTED ||
+    (rule.entropy && entropy(secret) < rule.entropy)
+    ? undefined
+    : secret;
+}

package/src/history-rewrite.js

@@ -0,0 +1,339 @@
+import { execFileSync } from 'node:child_process';
+import { mkdir, writeFile, readFile, rm, chmod } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { createSanitizer } from './engines.js';
+import { auditGitHistory } from './history.js';
+import { decodeText } from './files.js';
+import { failure } from './detection.js';
+
+function git(repository, args) {
+  try {
+    return execFileSync('git', ['-C', repository, ...args], {
+      maxBuffer: 100 * 1024 * 1024,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+  } catch {
+    throw failure('ERR_GIT');
+  }
+}
+async function identity(line, engine) {
+  const match = /^(.*) <([^>]*)> (\d+ [+-]\d{4})$/.exec(line);
+  if (!match) {
+    throw failure('ERR_GIT');
+  }
+  const name = await engine.sanitize(match[1]),
+    email = await engine.sanitize(match[2]);
+  const date = (await engine.inspect(match[3])).length ? '0 +0000' : match[3];
+  return {
+    name: name.text,
+    email: email.redactions ? 'REDACTED' : email.text,
+    date,
+    originalName: match[1],
+    originalEmail: match[2],
+  };
+}
+function regexp(value) {
+  return value
+    .replace(/[\\^$.*+?()[\]{}|]/g, '\\$&')
+    .replaceAll('\n', '\\n')
+    .replaceAll('\r', '\\r');
+}
+async function buildPlan(repository, audit, engine, maxPaths) {
+  const data = { blobs: {}, commits: {}, tags: {}, paths: {}, refs: {} },
+    replacements = new Set(),
+    mailmap = new Set();
+  for (const entry of audit.findings) {
+    const text = decodeText(
+      git(repository, ['cat-file', entry.type, entry.object])
+    );
+    for (const f of entry.findings) {
+      const value = text.slice(f.start, f.end);
+      replacements.add(
+        `regex:${regexp(value).replaceAll('==>', '\\x3d\\x3d\\x3e')}==>[REDACTED]`
+      );
+    }
+    if (entry.type === 'blob') {
+      data.blobs[entry.object] = Buffer.from(
+        (await engine.sanitize(text)).text
+      ).toString('base64');
+      continue;
+    }
+    const split = text.indexOf('\n\n');
+    if (split < 0) {
+      throw failure('ERR_GIT');
+    }
+    const details = {
+      message: (await engine.sanitize(text.slice(split + 2))).text,
+    };
+    for (const field of entry.type === 'commit'
+      ? ['author', 'committer']
+      : ['tagger']) {
+      const line = text
+        .slice(0, split)
+        .split('\n')
+        .find((record) => record.startsWith(`${field} `));
+      if (!line) {
+        continue;
+      }
+      const person = await identity(line.slice(field.length + 1), engine);
+      details[field] = {
+        name: person.name,
+        email: person.email,
+        date: person.date,
+      };
+      if (
+        person.name !== person.originalName ||
+        person.email !== person.originalEmail
+      ) {
+        mailmap.add(
+          `${person.name} <${person.email}> ${person.originalName} <${person.originalEmail}>`
+        );
+      }
+    }
+    data[entry.type === 'commit' ? 'commits' : 'tags'][entry.object] = details;
+  }
+  const paths = new Set();
+  for (const commit of git(repository, ['rev-list', '--all'])
+    .toString()
+    .trim()
+    .split('\n')
+    .filter(Boolean)) {
+    for (const path of git(repository, [
+      'ls-tree',
+      '-r',
+      '-z',
+      '--name-only',
+      commit,
+    ])
+      .toString('utf8')
+      .split('\0')
+      .filter(Boolean)) {
+      paths.add(path);
+      if (paths.size > maxPaths) {
+        throw failure('ERR_LIMIT');
+      }
+    }
+  }
+  await pathMappings(paths, data.paths, engine, false);
+  const refs = git(repository, ['for-each-ref', '--format=%(refname)'])
+    .toString('utf8')
+    .trim()
+    .split('\n')
+    .filter(Boolean);
+  await pathMappings(refs, data.refs, engine, true);
+  return {
+    data,
+    replacements: `${[...replacements].join('\n')}\n`,
+    mailmap: `${[...mailmap].join('\n')}\n`,
+  };
+}
+async function pathMappings(paths, mappings, engine, ref) {
+  const targets = new Map();
+  for (const path of paths) {
+    const result = await engine.sanitize(path),
+      replacement = ref
+        ? result.text.replaceAll('[REDACTED]', 'REDACTED')
+        : result.text;
+    if (targets.has(replacement) && targets.get(replacement) !== path) {
+      throw failure('ERR_HISTORY_COLLISION');
+    }
+    targets.set(replacement, path);
+    if (result.redactions) {
+      mappings[path] = replacement;
+    }
+  }
+}
+const callbacks = `import base64, json
+from pathlib import Path
+plan = json.loads(Path(__file__).with_name('replacements.json').read_text())
+def blob(blob):
+    value = plan['blobs'].get(blob.original_id.decode())
+    if value is not None: blob.data = base64.b64decode(value)
+def metadata(obj, group):
+    value = plan[group].get(obj.original_id.decode())
+    if value is None: return
+    obj.message = value['message'].encode()
+    for role in ('author', 'committer', 'tagger'):
+        if role in value:
+            for key in ('name', 'email', 'date'):
+                setattr(obj, role + '_' + key, value[role][key].encode())
+def filename(value):
+    return plan['paths'].get(value.decode(), value.decode()).encode()
+def refname(value):
+    return plan['refs'].get(value.decode(), value.decode()).encode()
+`;
+/** Preview by default; --apply affects only the newly created mirror clone. */
+export async function rewriteGitHistory(
+  source,
+  destination,
+  {
+    apply = false,
+    sanitizer,
+    sanitizerOptions = {},
+    filterRepoCommand = 'git-filter-repo',
+    maxPaths = 10000,
+    ...auditOptions
+  } = {}
+) {
+  validateRewriteOptions(apply, maxPaths, filterRepoCommand);
+  const directory = resolve(destination),
+    repository = join(directory, 'repository.git');
+  sanitizer ??= createSanitizer(sanitizerOptions);
+  try {
+    await mkdir(directory, { mode: 0o700 });
+  } catch {
+    throw failure('ERR_OUTPUT');
+  }
+  try {
+    execFileSync(
+      'git',
+      ['clone', '--mirror', '--no-local', '--', resolve(source), repository],
+      { maxBuffer: 4 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'] }
+    );
+    await chmod(repository, 0o700);
+    const before = await auditGitHistory(repository, {
+      ...auditOptions,
+      sanitizer,
+    });
+    if (!before.complete) {
+      throw failure('ERR_HISTORY_INCOMPLETE');
+    }
+    const plan = await buildPlan(repository, before, sanitizer, maxPaths);
+    for (const [name, content] of Object.entries({
+      'replacements.json': JSON.stringify(plan.data),
+      'replace-text.txt': plan.replacements,
+      mailmap: plan.mailmap,
+      'sanitizer_plan.py': callbacks,
+    })) {
+      await writeFile(join(directory, name), content, {
+        flag: 'wx',
+        mode: 0o600,
+      });
+    }
+    if (apply) {
+      runFilter(repository, directory, filterRepoCommand);
+      return await completedRewrite(
+        { repository, directory, before, plan },
+        sanitizer,
+        auditOptions,
+        maxPaths
+      );
+    }
+    git(repository, ['remote', 'remove', 'origin']);
+    return {
+      applied: false,
+      repository,
+      before,
+      changedPaths: Object.keys(plan.data.paths).length,
+      changedRefs: Object.keys(plan.data.refs).length,
+    };
+  } catch (error) {
+    await rm(directory, { recursive: true, force: true });
+    throw failure(
+      /^ERR_[A-Z_]+$/.test(error.code) ? error.code : 'ERR_HISTORY_REWRITE'
+    );
+  }
+}
+function runFilter(repository, directory, command) {
+  try {
+    execFileSync(
+      command,
+      [
+        '--blob-callback',
+        'import sanitizer_plan; sanitizer_plan.blob(blob)',
+        '--commit-callback',
+        "import sanitizer_plan; sanitizer_plan.metadata(commit, 'commits')",
+        '--tag-callback',
+        "import sanitizer_plan; sanitizer_plan.metadata(tag, 'tags')",
+        '--filename-callback',
+        'import sanitizer_plan; return sanitizer_plan.filename(filename)',
+        '--refname-callback',
+        'import sanitizer_plan; return sanitizer_plan.refname(refname)',
+      ],
+      {
+        cwd: repository,
+        env: { ...process.env, PYTHONPATH: directory },
+        maxBuffer: 4 * 1024 * 1024,
+        stdio: ['ignore', 'pipe', 'pipe'],
+      }
+    );
+  } catch {
+    throw failure('ERR_HISTORY_REWRITE');
+  }
+}
+async function checkPaths(repository, sanitizer, maxPaths) {
+  const plan = await buildPlan(
+    repository,
+    { findings: [] },
+    sanitizer,
+    maxPaths
+  );
+  if (
+    Object.keys(plan.data.paths).length ||
+    Object.keys(plan.data.refs).length
+  ) {
+    throw failure('ERR_HISTORY_RESIDUAL');
+  }
+}
+async function removePrivatePlan(directory) {
+  for (const name of [
+    'replacements.json',
+    'replace-text.txt',
+    'mailmap',
+    'sanitizer_plan.py',
+    '__pycache__',
+  ]) {
+    await rm(join(directory, name), { recursive: true, force: true });
+  }
+  const config = await readFile(
+    join(directory, 'repository.git', 'config'),
+    'utf8'
+  );
+  if (config.includes('[remote "origin"]')) {
+    git(join(directory, 'repository.git'), ['remote', 'remove', 'origin']);
+  }
+}
+
+async function completedRewrite(
+  { repository, directory, before, plan },
+  sanitizer,
+  auditOptions,
+  maxPaths
+) {
+  const after = await auditGitHistory(repository, {
+    ...auditOptions,
+    sanitizer,
+  });
+  if (!after.complete || after.findings.length) {
+    throw failure('ERR_HISTORY_RESIDUAL');
+  }
+  await checkPaths(repository, sanitizer, maxPaths);
+  git(repository, ['fsck', '--full']);
+  await removePrivatePlan(directory);
+  const result = {
+    applied: true,
+    repository,
+    before,
+    after,
+    changedPaths: Object.keys(plan.data.paths).length,
+    changedRefs: Object.keys(plan.data.refs).length,
+  };
+  await writeFile(
+    join(directory, 'manifest.json'),
+    `${JSON.stringify(result, null, 2)}\n`,
+    { mode: 0o600 }
+  );
+  return result;
+}
+
+function validateRewriteOptions(apply, maxPaths, command) {
+  if (
+    typeof apply !== 'boolean' ||
+    !Number.isSafeInteger(maxPaths) ||
+    maxPaths <= 0 ||
+    typeof command !== 'string' ||
+    !command
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+}

package/src/history.js

@@ -0,0 +1,103 @@
+import { execFileSync } from 'node:child_process';
+import { failure } from './detection.js';
+import { decodeText } from './files.js';
+import { createSanitizer } from './engines.js';
+
+export async function auditGitHistory(
+  repository,
+  {
+    sanitizer,
+    maxObjects = 10000,
+    maxBytes = 10 * 1024 * 1024,
+    maxTotalBytes = 100 * 1024 * 1024,
+  } = {}
+) {
+  sanitizer ??= createSanitizer();
+  validateLimits([maxObjects, maxBytes, maxTotalBytes]);
+  const git = (args, input) => {
+    try {
+      return execFileSync('git', ['-C', repository, ...args], {
+        input,
+        maxBuffer: Math.max(maxBytes + 1024, 4 * 1024 * 1024),
+        stdio: ['pipe', 'pipe', 'pipe'],
+      });
+    } catch {
+      throw failure('ERR_GIT');
+    }
+  };
+  const objects = new Set(
+    git(['rev-list', '--objects', '--all', '--reflog'])
+      .toString('utf8')
+      .trim()
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => line.split(' ')[0])
+  );
+  if (objects.size > maxObjects) {
+    throw failure('ERR_LIMIT');
+  }
+  const metadata = git(
+    ['cat-file', '--batch-check=%(objectname) %(objecttype) %(objectsize)'],
+    `${[...objects].join('\n')}\n`
+  )
+    .toString('utf8')
+    .trim()
+    .split('\n')
+    .filter(Boolean);
+  const findings = [],
+    skipped = [];
+  let scanned = 0,
+    total = 0;
+  for (const record of metadata) {
+    const { object, type, size } = parseRecord(record);
+    if (!['blob', 'commit', 'tag'].includes(type)) {
+      continue;
+    }
+    total += size;
+    if (size > maxBytes || total > maxTotalBytes) {
+      throw failure('ERR_LIMIT');
+    }
+    const bytes = git(['cat-file', type, object]);
+    const decoded = decodeObject(bytes);
+    if (decoded.reason) {
+      skipped.push({ object, reason: decoded.reason });
+      continue;
+    }
+    const text = decoded.text;
+    const detected = await sanitizer.inspect(text);
+    scanned++;
+    if (detected.length) {
+      findings.push({ object, type, findings: detected });
+    }
+  }
+  return { scanned, findings, skipped, complete: skipped.length === 0 };
+}
+
+function parseRecord(record) {
+  const [object, type, sizeText] = record.split(' ');
+  const size = Number(sizeText);
+  if (
+    !/^[a-f0-9]{40,64}$/.test(object) ||
+    !Number.isSafeInteger(size) ||
+    size < 0
+  ) {
+    throw failure('ERR_GIT');
+  }
+  return { object, type, size };
+}
+
+function decodeObject(bytes) {
+  try {
+    return { text: decodeText(bytes) };
+  } catch (error) {
+    return { reason: error.code };
+  }
+}
+
+function validateLimits(limits) {
+  for (const limit of limits) {
+    if (!Number.isSafeInteger(limit) || limit <= 0) {
+      throw failure('ERR_CONFIG');
+    }
+  }
+}