@link-foundation/sensitive-data-sanitizer 0.19.3 → 0.21.0

package/src/gitleaks.js

@@ -7,22 +7,46 @@ import r2 from './vendor/gitleaks/rules-2.json' with { type: 'json' };
 import r3 from './vendor/gitleaks/rules-3.json' with { type: 'json' };
 import r4 from './vendor/gitleaks/rules-4.json' with { type: 'json' };
 import r5 from './vendor/gitleaks/rules-5.json' with { type: 'json' };
+import b0 from './vendor/betterleaks/rules-0.json' with { type: 'json' };
+import b1 from './vendor/betterleaks/rules-1.json' with { type: 'json' };
+import b2 from './vendor/betterleaks/rules-2.json' with { type: 'json' };
+import b3 from './vendor/betterleaks/rules-3.json' with { type: 'json' };
+import b4 from './vendor/betterleaks/rules-4.json' with { type: 'json' };
+import b5 from './vendor/betterleaks/rules-5.json' with { type: 'json' };
 import { entropy, REDACTED } from './detection.js';
 import { keywordMatcher } from './keywords.js';
+import { tokenChecksum } from './token-checksum.js';
 
 export const gitleaksRules = [...r0, ...r1, ...r2, ...r3, ...r4, ...r5];
+// Betterleaks rules whose ids are absent from the frozen Gitleaks set; see
+// scripts/sync-betterleaks.py for the translated filter subset.
+export const betterleaksRules = [...b0, ...b1, ...b2, ...b3, ...b4, ...b5];
+const upstreamRules = [
+  ...gitleaksRules.map((rule) => ({ rule, name: `gitleaks/${rule.id}` })),
+  ...betterleaksRules.map((rule) => ({
+    rule,
+    name: `betterleaks/${rule.id}`,
+  })),
+];
 const compiled = new Map();
 const linearPatterns = new Map();
 const keywordIndex = keywordMatcher(
-  gitleaksRules.flatMap((r) =>
-    (r.keywords ?? []).map((word) => word.toLowerCase())
+  upstreamRules.flatMap(({ rule }) =>
+    (rule.keywords ?? []).map((word) => word.toLowerCase())
   )
 );
+const confidence = { high: 0.95, medium: 0.85, low: 0.75 };
+// re2-wasm has a fixed 16 MiB heap shared by every compiled rule and every
+// subject copy, and an allocation failure aborts the module for the rest of
+// the process. All bundled rules compiled take about 7 MiB, and each exec
+// copies its subject, so keep subjects well below the remaining space and
+// scan larger texts with RE2JS (experiments/issue-128/re2-big-input.mjs).
+export const WASM_SUBJECT_LIMIT = 1024 * 1024;
 
 export function detectGitleaks(text, emit, trace) {
   const keywords = keywordIndex(text.toLowerCase());
-  const large = Buffer.byteLength(text) > 4 * 1024 * 1024;
-  for (const rule of gitleaksRules) {
+  const large = Buffer.byteLength(text) > WASM_SUBJECT_LIMIT;
+  for (const { rule, name } of upstreamRules) {
     if (
       !rule.regex ||
       (rule.keywords?.length &&
@@ -32,10 +56,10 @@ export function detectGitleaks(text, emit, trace) {
     }
     const started = trace ? performance.now() : 0;
     for (const view of ruleViews(text, rule)) {
-      scanRule(view.text, view.start, rule, emit, large);
+      scanRule(view.text, view.start, { rule, name }, emit, large);
     }
     trace?.({
-      rule: rule.id,
+      rule: name.startsWith('gitleaks/') ? rule.id : name,
       length: text.length,
       ms: performance.now() - started,
     });
@@ -118,8 +142,8 @@ function* boundedViews(text, prefix, id) {
   }
 }
 
-function scanRule(text, base, rule, emit, large) {
-  for (const match of ruleMatches(text, rule, large)) {
+function scanRule(text, base, { rule, name }, emit, large) {
+  for (const match of ruleMatches(text, name, rule, large)) {
     const secret = capturedSecret(rule, match);
     if (!secret) {
       continue;
@@ -133,17 +157,25 @@ function scanRule(text, base, rule, emit, large) {
       end: base + match.index + last + secret.length,
       type: 'SECRET',
       category: 'credential',
-      rule: `gitleaks/${rule.id}`,
-      confidence: rule.entropy ? 0.85 : 0.95,
+      rule: name,
+      confidence: ruleConfidence(rule, secret),
     });
   }
 }
 
-function* ruleMatches(text, rule, large) {
+// A token whose embedded CRC32 checksum matches is structurally valid offline.
+function ruleConfidence(rule, secret) {
+  if (tokenChecksum(secret)) {
+    return 0.99;
+  }
+  return confidence[rule.confidence] ?? (rule.entropy ? 0.85 : 0.95);
+}
+
+function* ruleMatches(text, name, rule, large) {
   if (large) {
-    // re2-wasm has a fixed 16 MiB heap. RE2JS implements the same linear-time
-    // syntax without copying the input into WASM. Its indices are UTF-16.
-    yield* linearMatches(text, rule);
+    // RE2JS implements the same linear-time syntax without copying the input
+    // into WASM. Its indices are UTF-16.
+    yield* linearMatches(text, name, rule.regex);
     return;
   }
   // re2-wasm 1.0.2 returns codepoint indices despite its RegExp-like API.
@@ -157,11 +189,7 @@ function* ruleMatches(text, rule, large) {
       offsets.push(offset);
     }
   }
-  let pattern = compiled.get(rule.id);
-  if (!pattern) {
-    pattern = new RE2(rule.regex, 'gu');
-    compiled.set(rule.id, pattern);
-  }
+  const pattern = compiledRule(name, rule.regex);
   pattern.lastIndex = 0;
   for (let attempts = 0; attempts < 8; attempts++) {
     const match = pattern.exec(text);
@@ -177,16 +205,32 @@ function* ruleMatches(text, rule, large) {
   // Keep the original subject: slicing would change anchors and boundaries.
   const resume = offsets ? offsets[pattern.lastIndex] : pattern.lastIndex;
   if (resume !== undefined && resume <= text.length) {
-    yield* linearMatches(text, rule, resume);
+    yield* linearMatches(text, name, rule.regex, resume);
+  }
+}
+
+// One cached re2-wasm object per rule name. re2-wasm objects are never freed,
+// so every caller, tests included, shares this cache.
+export function compiledRule(name, source) {
+  let pattern = compiled.get(name);
+  if (!pattern) {
+    pattern = new RE2(source, 'gu');
+    compiled.set(name, pattern);
   }
+  return pattern;
 }
 
-function* linearMatches(text, rule, resume = 0) {
-  let linear = linearPatterns.get(rule.id);
+function linearPattern(key, source) {
+  let linear = linearPatterns.get(key);
   if (!linear) {
-    linear = RE2JS.compile(rule.regex);
-    linearPatterns.set(rule.id, linear);
+    linear = RE2JS.compile(source);
+    linearPatterns.set(key, linear);
   }
+  return linear;
+}
+
+function* linearMatches(text, name, source, resume = 0) {
+  const linear = linearPattern(name, source);
   if (linear.test(text)) {
     for (const match of linear.matchAll(text)) {
       if (match.index >= resume) {
@@ -200,9 +244,20 @@ function capturedSecret(rule, match) {
   const secret = rule.secretGroup
     ? match[rule.secretGroup]
     : (match.slice(1).find(Boolean) ?? match[0]);
-  return !secret ||
-    secret === REDACTED ||
-    (rule.entropy && entropy(secret) < rule.entropy)
+  return !secret || secret === REDACTED || filtered(rule, secret)
     ? undefined
     : secret;
 }
+
+// Secret-only Betterleaks filter clauses, translated by the sync script.
+function filtered(rule, secret) {
+  const score = rule.entropy ? entropy(secret) : Infinity;
+  const test = (source) =>
+    linearPattern(`filter:${source}`, source).test(secret);
+  return (
+    score < rule.entropy ||
+    (rule.entropyInclusive && score === rule.entropy) ||
+    (rule.reject ?? []).some(test) ||
+    !(rule.require ?? []).every(test)
+  );
+}

package/src/html-tags.js

@@ -0,0 +1,8 @@
+// HTML formatting tags split tokens (`SECRET_</span><span>REST`). Inline tags
+// are dropped and block tags become line breaks. Only presentation attributes
+// are allowed, so other attribute values (URLs, data-*) stay in the view.
+export const inlineTags =
+  'abbr|b|cite|code|del|em|font|i|ins|kbd|mark|q|s|samp|small|span|strong|sub|sup|tt|u|var';
+const blockTags =
+  'blockquote|br|dd|div|dt|h[1-6]|hr|li|ol|p|pre|table|tbody|td|th|thead|tr|ul';
+export const htmlTag = String.raw`<\/?(?:${inlineTags}|${blockTags})(?:\s+(?:class|dir|lang|style)="[^"<>=]*")*\s*\/?>`;

package/src/identifier-detector.js

@@ -0,0 +1,36 @@
+import { failure } from './detection.js';
+import { detectNational } from './national.js';
+
+const validRule = (rule) =>
+  rule &&
+  typeof rule.type === 'string' &&
+  /^[A-Z][A-Z0-9_]{0,63}$/.test(rule.type) &&
+  rule.pattern instanceof RegExp &&
+  rule.pattern.global &&
+  rule.context instanceof RegExp &&
+  !rule.context.global &&
+  !rule.context.sticky &&
+  typeof rule.checksum === 'function' &&
+  ['plausible', 'valid'].every(
+    (key) => rule[key] === undefined || typeof rule[key] === 'function'
+  );
+
+// Adapts identifier rules (the `nationalRules` shape) to the Detector port, so
+// callers can add or replace identifiers without forking the built-in table.
+export function createIdentifierDetector(
+  rules,
+  { id = 'identifier-rules' } = {}
+) {
+  if (!Array.isArray(rules) || rules.length === 0 || !rules.every(validRule)) {
+    throw failure('ERR_CONFIG');
+  }
+  const frozen = rules.map((rule) => Object.freeze({ ...rule }));
+  return {
+    id,
+    detect(text) {
+      const findings = [];
+      detectNational(text, (finding) => findings.push(finding), frozen);
+      return findings;
+    },
+  };
+}

package/src/identity-types.js

@@ -0,0 +1,63 @@
+// Identity-document finding types, shared by the transforms and the
+// identity detectors without loading the detectors themselves.
+export const identityTypes = new Set([
+  'SSN',
+  'US_SSN',
+  'BRAZIL_CPF_NUMBER',
+  'INDIA_PAN_INDIVIDUAL',
+  'UK_NATIONAL_INSURANCE_NUMBER',
+  'SPAIN_DNI_NUMBER',
+  'US_HEALTHCARE_NPI',
+  'DRIVER_LICENSE',
+  'DRIVERS_LICENSE',
+  'NATIONAL_ID',
+  'CHINA_RESIDENT_ID_NUMBER',
+  'HONG_KONG_ID_NUMBER',
+  'SOUTH_AFRICA_ID_NUMBER',
+  'TAIWAN_ID_NUMBER',
+  'TURKEY_ID_NUMBER',
+  'GOVERNMENT_ID',
+  'DOD_ID_NUMBER',
+  'JAPAN_INDIVIDUAL_NUMBER',
+  'INDIA_GST_INDIVIDUAL',
+  'US_MEDICARE_BENEFICIARY_ID_NUMBER',
+  'ID',
+  'PASSPORT_NUMBER',
+  'PASSPORT_MRZ',
+  'BOOKING_REFERENCE',
+  'TICKET_NUMBER',
+  'VISA_NUMBER',
+  'UK_NHS',
+  'KR_RRN',
+  'IT_FISCAL_CODE',
+  'SG_NRIC_FIN',
+  'PL_PESEL',
+  'SE_PERSONNUMMER',
+  'ZA_ID_NUMBER',
+  'AU_TFN',
+  'CA_SIN',
+  'RU_SNILS',
+  'RU_INN',
+  'US_ITIN',
+  'US_DRIVER_LICENSE',
+  'UK_DRIVING_LICENCE',
+  'ES_NIE',
+  'IN_AADHAAR',
+  'IN_VOTER',
+  'IN_PASSPORT',
+  'FI_PERSONAL_IDENTITY_CODE',
+  'TH_TNIN',
+  'AU_MEDICARE',
+  'DE_TAX_ID',
+  'AE_EMIRATES_ID',
+  'SA_NATIONAL_ID',
+  'IL_ID_NUMBER',
+]);
+export function isIdentityType(type) {
+  return (
+    identityTypes.has(type) ||
+    /PASSPORT|(?:DRIVERS?|DRIVING)_?LICENSE|IDENTITY|NATIONAL_?(?:ID|IDENTIFICATION)|PERSONAL_?(?:ID|IDENTIFICATION)|UNIQUE_?IDENTIFICATION|SOCIAL_?(?:SECURITY|INSURANCE|IDENTIFICATION)|RESIDENT_?REGISTRATION|(?:TAX|TAXPAYER).*?(?:IDENTIFICATION|NUMBER|REFERENCE)|AADHAAR|FISCAL_CODE|PESEL|RRN|NHS|SNILS|INN_NUMBER/.test(
+      type
+    )
+  );
+}

package/src/identity.js

@@ -1,52 +1,8 @@
 import { collectMatches } from './detection.js';
 
-export const identityTypes = new Set([
-  'SSN',
-  'US_SSN',
-  'BRAZIL_CPF_NUMBER',
-  'INDIA_PAN_INDIVIDUAL',
-  'UK_NATIONAL_INSURANCE_NUMBER',
-  'SPAIN_DNI_NUMBER',
-  'US_HEALTHCARE_NPI',
-  'DRIVER_LICENSE',
-  'DRIVERS_LICENSE',
-  'NATIONAL_ID',
-  'CHINA_RESIDENT_ID_NUMBER',
-  'HONG_KONG_ID_NUMBER',
-  'SOUTH_AFRICA_ID_NUMBER',
-  'TAIWAN_ID_NUMBER',
-  'TURKEY_ID_NUMBER',
-  'GOVERNMENT_ID',
-  'DOD_ID_NUMBER',
-  'JAPAN_INDIVIDUAL_NUMBER',
-  'INDIA_GST_INDIVIDUAL',
-  'US_MEDICARE_BENEFICIARY_ID_NUMBER',
-  'ID',
-  'PASSPORT_NUMBER',
-  'PASSPORT_MRZ',
-  'BOOKING_REFERENCE',
-  'TICKET_NUMBER',
-  'VISA_NUMBER',
-  'UK_NHS',
-  'KR_RRN',
-  'IT_FISCAL_CODE',
-  'SG_NRIC_FIN',
-  'PL_PESEL',
-  'SE_PERSONNUMMER',
-  'ZA_ID_NUMBER',
-  'AU_TFN',
-  'CA_SIN',
-  'RU_SNILS',
-  'RU_INN',
-]);
-export function isIdentityType(type) {
-  return (
-    identityTypes.has(type) ||
-    /PASSPORT|(?:DRIVERS?|DRIVING)_?LICENSE|IDENTITY|NATIONAL_?(?:ID|IDENTIFICATION)|PERSONAL_?(?:ID|IDENTIFICATION)|UNIQUE_?IDENTIFICATION|SOCIAL_?(?:SECURITY|INSURANCE|IDENTIFICATION)|RESIDENT_?REGISTRATION|(?:TAX|TAXPAYER).*?(?:IDENTIFICATION|NUMBER|REFERENCE)|AADHAAR|FISCAL_CODE|PESEL|RRN|NHS|SNILS|INN_NUMBER/.test(
-      type
-    )
-  );
-}
+import { isIdentityType } from './identity-types.js';
+
+export { identityTypes, isIdentityType } from './identity-types.js';
 const personal = (type, rule, confidence = 0.85) => ({
   type,
   rule,

package/src/index.d.ts

@@ -17,6 +17,8 @@ export interface Finding {
   kept?: 'fake';
   /** Returned finding replaced with, or recognized as, a generated fake. */
   faked?: true;
+  /** For `JSON_VALUE`: the field's personal type, e.g. `AGE`, when known. */
+  valueType?: string;
 }
 export interface PersonalValue {
   type: string;
@@ -31,7 +33,9 @@ export type Vocabulary = Partial<
     | 'organizationCues'
     | 'educationCues'
     | 'personalLabels'
-    | 'personalFields',
+    | 'personalFields'
+    /** Word stems for the strict special-category rule; needs `strict: true`. */
+    | 'personalAttributes',
     VocabularyList
   >
 >;
@@ -100,6 +104,33 @@ export interface SanitizerOptions {
   minConfidence?: number;
   /** Alias for minConfidence; specifying both requires equal values. */
   threshold?: number;
+  /** Per-type thresholds (e.g. `{ PERSON: 0.8 }`) that override minConfidence. */
+  minConfidenceByType?: Record<string, number>;
+  /**
+   * Values kept as-is: a string must equal the whole detected value and a
+   * RegExp must fully match it. Declared knownPersonal/knownSecrets values and
+   * credentials (unless allowCredentialDisable) are never allow-listed.
+   */
+  allowList?: (string | RegExp)[];
+  /**
+   * Honour scanner suppression comments (`gitleaks:allow`, `pragma: allowlist
+   * secret`, `pragma: allowlist nextline secret`, `NOSONAR`, ...) under the
+   * same rules as allowList.
+   */
+  inlineAllow?: boolean;
+  /**
+   * Keeps a finding the caller's policy knows is not sensitive. Runs after
+   * detection and before transformation, once per finding, with the text the
+   * offsets refer to; it must return a boolean (`true` keeps the span as-is).
+   * Verification of the output consults it again per span, so the same value
+   * elsewhere is still caught. Declared knownSecrets/knownPersonal/
+   * confirmedPersonal values and JSON_VALUE denials are never offered.
+   * createSyncSanitizer and sanitize() require a synchronous hook.
+   */
+  exempt?: (
+    finding: Readonly<Finding & { value: string }>,
+    text: string
+  ) => boolean | Promise<boolean>;
   /** Explicit network/callback verification; credentials never qualify. */
   verifyPublic?: (candidate: {
     type: PublicEntity['type'];
@@ -122,6 +153,7 @@ export type DebugEvent =
   | { event: 'detected'; count: number }
   | { event: 'document-experience' | 'proper-name-prose'; count: number }
   | { event: 'residual-redaction'; round: number }
+  | { event: 'template-value'; rule: string }
   | { event: 'two-pass-confirmation'; type: string }
   | { event: 'propagation-skipped'; reason: 'length'; type: string }
   | { event: 'gitleaks-rule'; rule: string; length: number; ms: number };
@@ -228,7 +260,11 @@ export declare function fromOffsetReport(
 ): Finding[];
 export declare function knownSecretsFromEnv(
   env?: Record<string, string | undefined>,
-  names?: string[]
+  names?: string[],
+  options?: {
+    /** Skip values shorter than this many characters (default 1). */
+    minLength?: number;
+  }
 ): string[];
 /** Reversible UTF-8 view: invalid bytes use unpaired U+DC80..U+DCFF. */
 export declare function decodeLogBytes(bytes: Uint8Array): string;
@@ -285,6 +321,12 @@ export type Transformation =
       map?: RestoreMap | false;
       format?: string;
       credentials?: boolean;
+      /**
+       * Tokens of the namespace already in the input: `'error'` (default)
+       * throws `ERR_PLACEHOLDER_COLLISION`; `'keep'` leaves them as they are
+       * and numbers new values after the highest kept number per type.
+       */
+      existing?: 'error' | 'keep';
     }
   | {
       mode: 'mask';
@@ -292,6 +334,17 @@ export type Transformation =
       keepEnd?: number;
       marker?: string;
       minLength?: number;
+      /** One masking character, default `*`; not combinable with marker. */
+      char?: string;
+      /** Characters left unmasked, e.g. `'-'` for `****-****-1111`. */
+      ignore?: string;
+      ignoreClasses?: (
+        'digits' | 'upper' | 'lower' | 'punctuation' | 'whitespace'
+      )[];
+      /** Mask at most this many maskable characters. */
+      count?: number;
+      /** Apply `count` from the end instead of the start. */
+      fromEnd?: boolean;
     }
   | { mode: 'pseudonym' | 'format-preserving'; key: string }
   | {
@@ -308,7 +361,9 @@ export type Transformation =
       >;
     }
   | { mode: 'date-shift'; days: number }
-  | { mode: 'bucket'; size: number };
+  | { mode: 'bucket'; size: number }
+  /** Deterministic AES-256-GCM `[ENCRYPTED:TYPE:…]`; key >= 16 UTF-8 bytes. */
+  | { mode: 'encrypt'; key: string };
 
 /** Contains original personal data; never publish this object or its files. */
 export interface RestoreMap {
@@ -350,6 +405,18 @@ export declare function writeRestoreMap(
 ): Promise<void>;
 /** Exact original-document restoration; derived text uses canonical surfaces. */
 export declare function restore(text: string, map: RestoreMap): string;
+/** Sorted, frozen list of the types built-in rules turn into `Type-n` tokens. */
+export declare function reservedPlaceholderTypes(): readonly string[];
+/** Whether `text` is one whole token of the placeholder namespace. */
+export declare function isPlaceholderToken(
+  text: string,
+  options?: {
+    /** Placeholder format, default `'{Type}-{n}'`. */
+    format?: string;
+    /** Extra custom types, e.g. those of `createPatternDetector`. */
+    types?: readonly string[];
+  }
+): boolean;
 export interface StreamOptions extends SanitizerOptions {
   /** Treat NUL as a log byte; default false. Incompatible with JSON/JSONL. */
   allowBinary?: boolean;
@@ -392,6 +459,93 @@ export declare const entityCatalogs: Record<
   'google' | 'azure' | 'aws',
   { source: string; sha256: string; names: string[] }
 >;
+/** Built-in identifier rule; `context` is tested near each `pattern` match. */
+export interface IdentifierRule {
+  type: string;
+  /** Global regular expression for candidate values. */
+  pattern: RegExp;
+  /** Non-global label/context expression, 64 characters before to 32 after. */
+  context: RegExp;
+  /** Checksum or issuing-format check that raises confidence. */
+  checksum: (value: string) => boolean;
+  /** Distinctive shape that is reported without context. */
+  plausible?: (value: string) => boolean;
+  /** Mandatory gate; candidates failing it are never reported. */
+  valid?: (value: string) => boolean;
+}
+export declare const nationalRules: readonly IdentifierRule[];
+/** Format and checksum validators; they never prove issuance. */
+export declare const identifierChecks: Readonly<
+  Record<string, (value: string, ...args: number[]) => boolean>
+> & {
+  digitsOf(value: string): string;
+};
+/** Wrap identifier rules as a Detector for `detectors` options. */
+export declare function createIdentifierDetector(
+  rules: readonly IdentifierRule[],
+  options?: { id?: string }
+): Detector;
+/** Restore `[ENCRYPTED:TYPE:…]` tokens; throws ERR_MAP on a wrong key. */
+export declare function decrypt(
+  text: string,
+  key: string,
+  options?: { json?: boolean }
+): string;
+export interface PatternSpec {
+  type: string;
+  category?: Category;
+  /** Finding rule id, default `custom-pattern`. */
+  rule?: string;
+  /** Default base confidence, 0.6. */
+  confidence?: number;
+  patterns?: (
+    | RegExp
+    | string
+    | { regex: RegExp | string; confidence?: number; group?: number }
+  )[];
+  /** Whole-word terms (case-insensitive unless caseSensitive). */
+  denyList?: string[];
+  caseSensitive?: boolean;
+  denyListConfidence?: number;
+  /** Hotwords that adjust confidence when found near a match. */
+  context?: string[];
+  /** Characters searched for context, default `{ before: 64, after: 16 }`. */
+  contextWindow?: number | { before?: number; after?: number };
+  /** Added when context is found (default 0.35); negative lowers it. */
+  contextBoost?: number;
+  /** false rejects the match; a number in 0..1 sets its confidence. */
+  validate?: (value: string) => boolean | number | undefined;
+}
+/** Declarative regex/deny-list recognizers (Presidio/Google custom infoType style). */
+export declare function createPatternDetector(
+  specs: PatternSpec | readonly PatternSpec[],
+  options?: { id?: string }
+): Detector;
+export interface SarifLog {
+  $schema: string;
+  version: '2.1.0';
+  runs: unknown[];
+}
+/** SARIF 2.1.0 log of rules and locations; never contains matched values. */
+export declare function toSarif(
+  report:
+    | {
+        complete?: boolean;
+        results: { path: string; findings: Finding[] }[];
+        skipped?: { path: string; reason: string }[];
+      }
+    | { path: string; findings: Finding[] }[]
+    | Finding[],
+  options?: { version?: string; path?: string }
+): SarifLog;
+/**
+ * Offline CRC32/base62 check of GitHub (gh[opsur]_) and npm (npm_) tokens:
+ * true when the checksum matches, false when it does not, undefined for
+ * other formats.
+ */
+export declare function tokenChecksum(value: string): boolean | undefined;
+/** Builds a checksum-valid token from a prefix ending in `_` and a 30-character body. */
+export declare function tokenWithChecksum(prefix: string, body: string): string;
 export declare function createWikidataVerifier(options?: {
   fetch?: typeof fetch;
   language?: string;

package/src/index.js

@@ -1,5 +1,7 @@
 /** Sensitive text detection and verified publication APIs. */
 
+import './personal-features.js';
+
 export { inspect, sanitize, redact } from './sanitizer.js';
 export { createSanitizer } from './engines.js';
 export { compileKnown } from './compiled-known.js';
@@ -26,6 +28,13 @@ export {
 } from './external.js';
 
 export { entityCatalogs } from './entities.js';
+export { nationalRules } from './national.js';
+export * as identifierChecks from './national-checks.js';
+export { createIdentifierDetector } from './identifier-detector.js';
+export { decrypt } from './encrypt.js';
+export { createPatternDetector } from './pattern-detector.js';
+export { toSarif } from './sarif.js';
+export { tokenChecksum, tokenWithChecksum } from './token-checksum.js';
 export {
   sanitizeStream,
   sanitizeStreamToFile,
@@ -51,3 +60,7 @@ export {
   readRestoreMap,
   writeRestoreMap,
 } from './restore-map.js';
+export {
+  reservedPlaceholderTypes,
+  isPlaceholderToken,
+} from './placeholder-types.js';

package/src/known.js

@@ -71,28 +71,44 @@ function configTokens(config) {
   return [...values];
 }
 
+// Credential keywords between `_` boundaries, plus suffixes that are written
+// without one (`PGPASSWORD`, `GITHUBTOKEN`, `SESSIONCOOKIE`).
 const secretEnvName =
-  /(?:^|_)(?:TOKEN|(?:API|ACCESS|PRIVATE)?KEY|SECRET|PASSWORD|PASSWD|CREDENTIALS?|AUTH|PAT)(?:_|$)/i;
-// Configuration suffixes describe how authentication works. Explicit names
-// bypass this filter; credential suffixes such as AUTH_MODE_TOKEN still win.
-const authConfigurationName =
-  /(?:^|_)AUTH_(?:TRANSPORT|MODE|METHOD|TYPE|URL|URI|HOST|HOSTNAME|PATH|ENABLED|DISABLED|PORT|PROTOCOL|SCHEME)$/i;
+  /(?:^|_)(?:TOKEN|(?:API|ACCESS|PRIVATE)?KEY|SECRET|PASSWORD|PASSWD|CREDENTIALS?|O?AUTH|PAT|COOKIE)(?:_|$)|(?:PASSWORD|PASSWD|TOKEN|COOKIE)$/i;
+// Configuration suffixes describe a credential (how authentication works,
+// counters, limits, locations, names, identifiers and guard flags), not the
+// credential itself. Explicit names bypass this filter; credential suffixes
+// such as AUTH_MODE_TOKEN still win.
+const configurationName =
+  /_(?:TRANSPORT|MODE|METHOD|TYPE|URL|URI|HOST|HOSTNAME|PATH|ENABLED|DISABLED|PORT|PROTOCOL|SCHEME|GUARD|COUNT|LIMIT|FILE|FILENAME|DIR|DIRECTORY|NAME|ID)$|^GIT_CONFIG_KEY_\d+$|^GOOGLE_APPLICATION_CREDENTIALS$/i;
+// `GIT_CONFIG_KEY_<n>` names a git setting; `GIT_CONFIG_VALUE_<n>` holds the
+// value, which is a credential for headers, passwords and tokens.
+const gitCredentialSetting =
+  /(?:^|\.)(?:extraheader|password|token|secret|cookiefile|auth[a-z]*)$/i;
+
+function discoveredNames(env) {
+  return Object.keys(env).filter((name) => {
+    const git = name.match(/^GIT_CONFIG_VALUE_(\d+)$/);
+    if (git) {
+      return gitCredentialSetting.test(env[`GIT_CONFIG_KEY_${git[1]}`] ?? '');
+    }
+    return secretEnvName.test(name) && !configurationName.test(name);
+  });
+}
+
 /** Opt-in literal values from credential-named keys, or an explicit selection. */
-export function knownSecretsFromEnv(env = process.env, names) {
+export function knownSecretsFromEnv(env = process.env, names, options = {}) {
+  const { minLength = 1 } = options ?? {};
   if (names === undefined) {
-    names =
-      env && typeof env === 'object'
-        ? Object.keys(env).filter(
-            (name) =>
-              secretEnvName.test(name) && !authConfigurationName.test(name)
-          )
-        : [];
+    names = env && typeof env === 'object' ? discoveredNames(env) : [];
   }
   if (
     !env ||
     typeof env !== 'object' ||
     !Array.isArray(names) ||
-    names.some((name) => typeof name !== 'string')
+    names.some((name) => typeof name !== 'string') ||
+    !Number.isSafeInteger(minLength) ||
+    minLength < 1
   ) {
     throw failure('ERR_CONFIG');
   }
@@ -100,7 +116,9 @@ export function knownSecretsFromEnv(env = process.env, names) {
     ...new Set(
       names
         .map((name) => env[name])
-        .filter((value) => typeof value === 'string' && value.length)
+        .filter(
+          (value) => typeof value === 'string' && value.length >= minLength
+        )
     ),
   ];
 }