@link-foundation/sensitive-data-sanitizer 0.0.0-stage → 0.11.31

package/src/jsonl-stream.js

@@ -0,0 +1,198 @@
+import { TextDecoder } from 'node:util';
+import { Worker } from 'node:worker_threads';
+import { URL } from 'node:url';
+import { failure } from './detection.js';
+import { encodingFailure } from './utf8.js';
+import { workerExecArgv } from './worker-options.js';
+import { prepareStreamEngine } from './engines.js';
+import { inspect } from './sanitizer.js';
+
+async function* records(source, { maxRecordBytes, maxTotalBytes }) {
+  const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
+  let parts = [],
+    length = 0,
+    total = 0,
+    record = 1,
+    offset = 0;
+  function decode() {
+    try {
+      return decoder.decode(Buffer.concat(parts, length));
+    } catch {
+      throw encodingFailure(Buffer.concat(parts, length), record, offset);
+    }
+  }
+  for await (const chunk of source) {
+    const bytes = Buffer.from(chunk);
+    total += bytes.length;
+    if (total > maxTotalBytes || bytes.includes(0)) {
+      throw failure(bytes.includes(0) ? 'ERR_BINARY' : 'ERR_LIMIT');
+    }
+    let start = 0;
+    while (start < bytes.length) {
+      const newline = bytes.indexOf(10, start);
+      const end = newline < 0 ? bytes.length : newline + 1;
+      length += end - start;
+      if (length > maxRecordBytes) {
+        throw failure('ERR_LIMIT');
+      }
+      // Copy a fragment so a small retained tail never holds a huge input buffer.
+      parts.push(Buffer.from(bytes.subarray(start, end)));
+      if (newline >= 0) {
+        yield decode();
+        record++;
+        offset += length;
+        parts = [];
+        length = 0;
+      }
+      start = end;
+    }
+  }
+  if (length) {
+    yield decode();
+  }
+}
+async function* batches(source, config) {
+  let parts = [],
+    length = 0;
+  for await (const record of records(source, config)) {
+    const bytes = Buffer.byteLength(record);
+    if (length && length + bytes > config.batchBytes) {
+      yield parts.join('');
+      parts = [];
+      length = 0;
+    }
+    parts.push(record);
+    length += bytes;
+    if (length >= config.batchBytes) {
+      yield parts.join('');
+      parts = [];
+      length = 0;
+    }
+  }
+  if (length) {
+    yield parts.join('');
+  }
+}
+function workerSlot(options, heap, timeout) {
+  let worker;
+  try {
+    worker = new Worker(new URL('./record-worker.js', import.meta.url), {
+      workerData: options,
+      execArgv: workerExecArgv(),
+      resourceLimits: { maxOldGenerationSizeMb: heap, stackSizeMb: 4 },
+    });
+  } catch {
+    throw failure('ERR_CONFIG');
+  }
+  let pending,
+    failed = false;
+  function settle(result) {
+    if (!pending) {
+      return;
+    }
+    clearTimeout(pending.timer);
+    pending.resolve(result);
+    pending = undefined;
+  }
+  worker.on('message', settle);
+  worker.on('error', () => {
+    failed = true;
+    settle({ error: 'ERR_WORKER' });
+  });
+  worker.on('exit', () => {
+    failed = true;
+    settle({ error: 'ERR_WORKER' });
+  });
+  return {
+    run(text, confirmedPersonal, nativeFindings) {
+      if (failed) {
+        return Promise.resolve({ error: 'ERR_WORKER' });
+      }
+      return new Promise((resolve) => {
+        pending = {
+          resolve,
+          timer: setTimeout(() => settle({ error: 'ERR_WORKER' }), timeout),
+        };
+        worker.postMessage({ text, confirmedPersonal, nativeFindings });
+      });
+    },
+    async close() {
+      settle({ error: 'ERR_WORKER' });
+      await worker.terminate();
+    },
+  };
+}
+function workerCount(options, config) {
+  const callbacks =
+    options.sanitizer ||
+    config.sanitizerOptions.debug ||
+    config.sanitizerOptions.verifyPublic ||
+    config.sanitizerOptions.detectors?.length;
+  const count =
+    options.workers ?? (callbacks || typeof Deno !== 'undefined' ? 1 : 2);
+  if (
+    !Number.isSafeInteger(count) ||
+    count < 1 ||
+    count > 16 ||
+    (callbacks && count > 1)
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  // Deno's Node compatibility does not implement resource-limited workers.
+  return typeof Deno !== 'undefined' ? 1 : count;
+}
+export async function* sanitizeJsonlStream(source, options, config) {
+  const count = config.collecting ? 1 : workerCount(options, config);
+  if (count === 1) {
+    for await (const batch of batches(source, config)) {
+      config.identity.prepare(
+        batch,
+        await prepareStreamEngine(config.engine, batch)
+      );
+      if (!config.collecting) {
+        yield (await config.engine.sanitize(batch)).text;
+      }
+    }
+    return;
+  }
+  const heap = options.workerHeapMb ?? 256,
+    timeout = options.workerTimeoutMs ?? 60000;
+  if (
+    !Number.isSafeInteger(heap) ||
+    heap < 32 ||
+    !Number.isSafeInteger(timeout) ||
+    timeout <= 0
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  const slots = [],
+    queue = [];
+  let sequence = 0;
+  async function next() {
+    const result = await queue.shift();
+    if (result.error) {
+      throw failure(result.error, result);
+    }
+    return result.value;
+  }
+  try {
+    for (let i = 0; i < count; i++) {
+      slots.push(workerSlot(config.sanitizerOptions, heap, timeout));
+    }
+    for await (const batch of batches(source, config)) {
+      const nativeFindings = inspect(batch, config.sanitizerOptions);
+      const confirmedPersonal = config.identity.prepare(batch, nativeFindings);
+      queue.push(
+        slots[sequence++ % count].run(batch, confirmedPersonal, nativeFindings)
+      );
+      if (queue.length === count) {
+        yield await next();
+      }
+    }
+    while (queue.length) {
+      yield await next();
+    }
+  } finally {
+    await Promise.all(slots.map((slot) => slot.close()));
+  }
+}

package/src/keywords.js

@@ -0,0 +1,45 @@
+// Aho-Corasick keyword prefilter: equivalent to checking every literal with
+// includes(), including overlapping keywords, but scans a detection view once.
+export function keywordMatcher(words) {
+  const nodes = [{ next: new Map(), fail: 0, output: [] }];
+  for (const word of new Set(words)) {
+    let state = 0;
+    for (let i = 0; i < word.length; i++) {
+      const char = word[i];
+      if (!nodes[state].next.has(char)) {
+        nodes[state].next.set(char, nodes.length);
+        nodes.push({ next: new Map(), fail: 0, output: [] });
+      }
+      state = nodes[state].next.get(char);
+    }
+    nodes[state].output.push(word);
+  }
+  const queue = [...nodes[0].next.values()];
+  for (let index = 0; index < queue.length; index++) {
+    const state = queue[index];
+    for (const [char, child] of nodes[state].next) {
+      let fallback = nodes[state].fail;
+      while (fallback && !nodes[fallback].next.has(char)) {
+        fallback = nodes[fallback].fail;
+      }
+      nodes[child].fail = nodes[fallback].next.get(char) ?? 0;
+      nodes[child].output.push(...nodes[nodes[child].fail].output);
+      queue.push(child);
+    }
+  }
+  return (text) => {
+    const found = new Set();
+    let state = 0;
+    for (let i = 0; i < text.length; i++) {
+      const char = text[i];
+      while (state && !nodes[state].next.has(char)) {
+        state = nodes[state].fail;
+      }
+      state = nodes[state].next.get(char) ?? 0;
+      for (const word of nodes[state].output) {
+        found.add(word);
+      }
+    }
+    return found;
+  };
+}

package/src/known.js

@@ -0,0 +1,164 @@
+import { failure } from './detection.js';
+import { readTextFile } from './files.js';
+
+/** Explicitly load every host/account token from gh's local YAML config. */
+export async function knownSecretsFromGitHubConfig({ path } = {}) {
+  if (path !== undefined && (typeof path !== 'string' || !path)) {
+    throw failure('ERR_CONFIG');
+  }
+  const { lstat } = await import('node:fs/promises');
+  path ??= await gitHubConfigPath();
+  try {
+    await lstat(path);
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      return [];
+    }
+    throw failure('ERR_AUTH');
+  }
+  try {
+    const { parseDocument } = await import('yaml');
+    const document = parseDocument(await readTextFile(path, 1024 * 1024), {
+      logLevel: 'silent',
+      prettyErrors: false,
+    });
+    if (document.errors.length) {
+      throw failure('ERR_AUTH');
+    }
+    return configTokens(document.toJS({ maxAliasCount: 0 }));
+  } catch {
+    throw failure('ERR_AUTH');
+  }
+}
+
+async function gitHubConfigPath() {
+  const { join } = await import('node:path');
+  const { homedir } = await import('node:os');
+  const directory =
+    process.env.GH_CONFIG_DIR ||
+    (process.env.XDG_CONFIG_HOME
+      ? join(process.env.XDG_CONFIG_HOME, 'gh')
+      : process.platform === 'win32' && process.env.AppData
+        ? join(process.env.AppData, 'GitHub CLI')
+        : join(homedir(), '.config', 'gh'));
+  return join(directory, 'hosts.yml');
+}
+
+function configTokens(config) {
+  const values = new Set(),
+    pending = [config];
+  let count = 0;
+  while (pending.length) {
+    const entry = pending.pop();
+    if (!entry || typeof entry !== 'object') {
+      continue;
+    }
+    if (++count > 10000) {
+      throw failure('ERR_LIMIT');
+    }
+    for (const [key, value] of Object.entries(entry)) {
+      if (
+        ['oauth_token', 'api_token'].includes(key) &&
+        typeof value === 'string' &&
+        value.trim()
+      ) {
+        values.add(value);
+      } else if (value && typeof value === 'object') {
+        pending.push(value);
+      }
+    }
+  }
+  return [...values];
+}
+
+const secretEnvName =
+  /(?:^|_)(?:TOKEN|(?:API|ACCESS|PRIVATE)?KEY|SECRET|PASSWORD|PASSWD|CREDENTIALS?|AUTH|PAT)(?:_|$)/i;
+/** Opt-in literal values from credential-named keys, or an explicit selection. */
+export function knownSecretsFromEnv(env = process.env, names) {
+  if (names === undefined) {
+    names =
+      env && typeof env === 'object'
+        ? Object.keys(env).filter((name) => secretEnvName.test(name))
+        : [];
+  }
+  if (
+    !env ||
+    typeof env !== 'object' ||
+    !Array.isArray(names) ||
+    names.some((name) => typeof name !== 'string')
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  return [
+    ...new Set(
+      names
+        .map((name) => env[name])
+        .filter((value) => typeof value === 'string' && value.length)
+    ),
+  ];
+}
+
+/** Date and numeric spellings from documents-processing, with no inferred transliteration. */
+export function personalVariants(entries) {
+  if (!Array.isArray(entries)) {
+    throw failure('ERR_CONFIG');
+  }
+  return entries.flatMap((entry) => {
+    if (
+      !entry ||
+      typeof entry.value !== 'string' ||
+      !entry.value ||
+      !/^[A-Z][A-Z0-9_]{0,63}$/.test(entry.type)
+    ) {
+      throw failure('ERR_CONFIG');
+    }
+    const values = new Set([entry.value]);
+    const numeric = entry.value.replace(/\D/g, '');
+    if (numeric.length >= 6 && /^[+\d ()/.-]+$/.test(entry.value)) {
+      values.add(numeric);
+    }
+    const iso = entry.value.match(/^(\d{4})-(\d{2})-(\d{2})$/);
+    if (iso) {
+      for (const separator of ['/', '.', '-']) {
+        values.add(`${iso[3]}${separator}${iso[2]}${separator}${iso[1]}`);
+      }
+    }
+    const local = entry.value.match(/^(\d{2})[./-](\d{2})[./-](\d{4})$/);
+    if (local) {
+      values.add(`${local[3]}-${local[2]}-${local[1]}`);
+    }
+    return [...values].map((value) => ({ type: entry.type, value }));
+  });
+}
+
+/** Explicitly load the current local gh credential without logging the value. */
+export async function knownSecretsFromGitHubAuth({
+  command = 'gh',
+  hostname,
+  timeoutMs = 5000,
+} = {}) {
+  const { execFile } = await import('node:child_process');
+  if (
+    typeof command !== 'string' ||
+    !command ||
+    (hostname !== undefined && !/^[a-z0-9.-]+$/i.test(hostname)) ||
+    !Number.isSafeInteger(timeoutMs) ||
+    timeoutMs <= 0
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  return new Promise((accept, reject) => {
+    execFile(
+      command,
+      ['auth', 'token', ...(hostname ? ['--hostname', hostname] : [])],
+      { timeout: timeoutMs, maxBuffer: 65536, encoding: 'utf8' },
+      (error, stdout) => {
+        if (error || !stdout.trim()) {
+          reject(failure('ERR_AUTH'));
+          return;
+        }
+        accept([stdout.trim()]);
+      }
+    );
+  });
+}

package/src/legacy.d.ts

@@ -0,0 +1,3 @@
+export declare const add: (a: number, b: number) => number;
+export declare const multiply: (a: number, b: number) => number;
+export declare const delay: (ms: number) => Promise<void>;

package/src/legacy.js

@@ -0,0 +1,2 @@
+/** Arithmetic helpers for the retained template examples. */
+export { add, multiply, delay } from './arithmetic.js';

package/src/locales.js

@@ -0,0 +1,217 @@
+/** Authored vocabulary for contextual personal fields and credential prose. */
+export const localeVocabulary = [
+  [
+    'en',
+    'password',
+    'name',
+    'address',
+    'date of birth',
+    'passport',
+    'phone',
+    'email',
+    'organization',
+    'username',
+  ],
+  [
+    'es',
+    'contraseña',
+    'nombre',
+    'dirección',
+    'fecha de nacimiento',
+    'pasaporte',
+    'teléfono',
+    'correo electrónico',
+    'organización',
+    'usuario',
+  ],
+  [
+    'fr',
+    'mot de passe',
+    'nom',
+    'adresse',
+    'date de naissance',
+    'passeport',
+    'téléphone',
+    'courriel',
+    'organisation',
+    'identifiant',
+  ],
+  [
+    'de',
+    'passwort',
+    'name',
+    'adresse',
+    'geburtsdatum',
+    'reisepass',
+    'telefon',
+    'e-mail',
+    'organisation',
+    'benutzername',
+  ],
+  [
+    'pt',
+    'senha',
+    'nome',
+    'endereço',
+    'data de nascimento',
+    'passaporte',
+    'telefone',
+    'correio eletrônico',
+    'organização',
+    'usuário',
+  ],
+  [
+    'ru',
+    'пароля',
+    'имя',
+    'адрес',
+    'дата рождения',
+    'паспорт',
+    'телефон',
+    'электронная почта',
+    'организация',
+    'имя пользователя',
+  ],
+  [
+    'ja',
+    'パスワード',
+    '氏名',
+    '住所',
+    '生年月日',
+    '旅券',
+    '電話番号',
+    'メール',
+    '組織',
+    'ユーザー名',
+  ],
+  [
+    'zh',
+    '密码',
+    '姓名',
+    '地址',
+    '出生日期',
+    '身份证',
+    '电话号码',
+    '邮箱',
+    '组织',
+    '用户名',
+  ],
+  [
+    'zh-Hant',
+    '密碼',
+    '姓名',
+    '地址',
+    '出生日期',
+    '身分證',
+    '電話號碼',
+    '郵箱',
+    '組織',
+    '用戶名',
+  ],
+  [
+    'ko',
+    '비밀번호',
+    '이름',
+    '주소',
+    '생년월일',
+    '여권',
+    '전화번호',
+    '이메일',
+    '조직',
+    '사용자 이름',
+  ],
+  [
+    'ar',
+    'كلمة المرور',
+    'الاسم',
+    'العنوان',
+    'تاريخ الميلاد',
+    'جواز السفر',
+    'الهاتف',
+    'البريد الإلكتروني',
+    'المؤسسة',
+    'اسم المستخدم',
+  ],
+  [
+    'he',
+    'סיסמה',
+    'שם',
+    'כתובת',
+    'תאריך לידה',
+    'דרכון',
+    'טלפון',
+    'דואר אלקטרוני',
+    'ארגון',
+    'שם משתמש',
+  ],
+  [
+    'hi',
+    'पासवर्ड',
+    'नाम',
+    'पता',
+    'जन्म तिथि',
+    'पासपोर्ट',
+    'फ़ोन',
+    'ईमेल',
+    'संगठन',
+    'उपयोगकर्ता नाम',
+  ],
+  [
+    'tr',
+    'şifre',
+    'ad soyad',
+    'adres',
+    'doğum tarihi',
+    'pasaport',
+    'telefon',
+    'e-posta',
+    'kuruluş',
+    'kullanıcı adı',
+  ],
+  [
+    'id',
+    'kata sandi',
+    'nama',
+    'alamat',
+    'tanggal lahir',
+    'paspor',
+    'telepon',
+    'surel',
+    'organisasi',
+    'nama pengguna',
+  ],
+  [
+    'vi',
+    'mật khẩu',
+    'họ tên',
+    'địa chỉ',
+    'ngày sinh',
+    'hộ chiếu',
+    'điện thoại',
+    'thư điện tử',
+    'tổ chức',
+    'tên người dùng',
+  ],
+  [
+    'th',
+    'รหัสผ่าน',
+    'ชื่อ',
+    'ที่อยู่',
+    'วันเกิด',
+    'หนังสือเดินทาง',
+    'โทรศัพท์',
+    'อีเมล',
+    'องค์กร',
+    'ชื่อผู้ใช้',
+  ],
+];
+export const personalFieldTypes = [
+  'PERSON',
+  'ADDRESS',
+  'DATE_OF_BIRTH',
+  'ID',
+  'PHONE',
+  'EMAIL',
+  'ORGANIZATION',
+  'USERNAME',
+];