@discretion/core 0.1.0

package/src/detect/detectors/vat/euVat.ts

@@ -0,0 +1,280 @@
+/**
+ * VAT_NUMBER — EU VAT identification numbers, all 27 member states.
+ *
+ * SPEC.md: "EU VAT numbers — per-country structure and checksum for all
+ * member states." A VAT number is quoted WITH its country prefix (Greece
+ * uses EL, not GR), which is what makes a single detector precise: the
+ * prefix selects the national rule, and every country that publishes a
+ * checksum has it verified here — many reusing the shared library (DE and
+ * HR are ISO 7064 MOD 11,10; EL is the AFM power-of-two rule; IT and SE
+ * close with Luhn; PL is the NIP weights; PT the NIF weights; IE the
+ * PPS-style mod-23 letter). Countries whose personal-number fallbacks are
+ * weaker (CZ 9-digit) accept those structurally, marked in metadata.
+ *
+ * Adding a member state (or successor scheme) is one entry in RULES.
+ */
+
+import {
+  toDigits,
+  isAllDigits,
+  weightedSum,
+  weightedMod,
+  weightedModBy,
+  cyclicWeightedMod,
+  luhnValid,
+  mod11_10Valid,
+  modString,
+} from '../../../checksums/index.js';
+import { registerDetector } from '../../registry.js';
+import { CONFIDENCE, invalid, valid } from '../../types.js';
+import type { ValidationContext, ValidationResult } from '../../types.js';
+
+type VatCheck = (body: string) => boolean | 'structural';
+
+const DNI_LETTERS = 'TRWAGMYFPDXBNJZSQVHLCKE';
+const MOD23 = 'WABCDEFGHIJKLMNOPQRSTUV';
+
+/** Spanish NIF/NIE/CIF check, shared by the ES entry. */
+function esVatValid(body: string): boolean {
+  if (/^\d{8}[A-Z]$/.test(body)) {
+    return body[8] === DNI_LETTERS[Number(body.slice(0, 8)) % 23];
+  }
+  if (/^[XYZ]\d{7}[A-Z]$/.test(body)) {
+    const prefix = { X: '0', Y: '1', Z: '2' }[body[0] as 'X' | 'Y' | 'Z'];
+    return body[8] === DNI_LETTERS[Number(`${prefix}${body.slice(1, 8)}`) % 23];
+  }
+  if (/^[A-HJ-NP-SUVW]\d{7}[0-9A-J]$/.test(body)) {
+    let sum = 0;
+    for (let i = 1; i <= 7; i++) {
+      let v = Number(body[i]);
+      if (i % 2 === 1) {
+        v *= 2;
+        if (v > 9) v -= 9;
+      }
+      sum += v;
+    }
+    const value = (10 - (sum % 10)) % 10;
+    return body[8] === String(value) || body[8] === 'JABCDEFGHI'[value];
+  }
+  return false;
+}
+
+const RULES: Readonly<Record<string, VatCheck>> = {
+  AT: (b) => {
+    // 'U' + 8 digits; c = (10 − (s + 4) mod 10) mod 10 with alternating
+    // 1,2 weights and digit-sum folding. Verified against ATU13585627.
+    if (!/^U\d{8}$/.test(b)) return false;
+    const d = toDigits(b.slice(1))!;
+    let s = 0;
+    for (let i = 0; i < 7; i++) {
+      const product = d[i]! * (i % 2 === 0 ? 1 : 2);
+      s += product > 9 ? product - 9 : product;
+    }
+    return (10 - ((s + 4) % 10)) % 10 === d[7];
+  },
+  BE: (b) => {
+    if (!/^[01]\d{9}$/.test(b)) return false;
+    return Number(b.slice(8)) === 97 - (Number(b.slice(0, 8)) % 97);
+  },
+  BG: (b) => {
+    if (/^\d{9}$/.test(b)) {
+      const d = toDigits(b)!;
+      let r = weightedModBy(d.slice(0, 8), (i) => i + 1, 11)!;
+      if (r === 10) {
+        r = weightedModBy(d.slice(0, 8), (i) => i + 3, 11)!;
+        if (r === 10) r = 0;
+      }
+      return r === d[8];
+    }
+    if (/^\d{10}$/.test(b)) {
+      const d = toDigits(b)!;
+      // Personal numbers: the PNF weight vector closes most; EGN-style
+      // numbers are also legal — accept either closing rule.
+      const pnf = weightedMod(d.slice(0, 9), [21, 19, 17, 13, 11, 9, 7, 3, 1], 10)!;
+      if (pnf === d[9]) return true;
+      let egn = weightedMod(d.slice(0, 9), [2, 4, 8, 5, 10, 9, 7, 3, 6], 11)!;
+      if (egn === 10) egn = 0;
+      return egn === d[9];
+    }
+    return false;
+  },
+  CY: (b) => {
+    if (!/^\d{8}[A-Z]$/.test(b)) return false;
+    const TRANSFORM = [1, 0, 5, 7, 9, 13, 15, 17, 19, 21];
+    let sum = 0;
+    for (let i = 0; i < 8; i++) {
+      const d = Number(b[i]);
+      sum += i % 2 === 0 ? TRANSFORM[d]! : d;
+    }
+    return b[8] === String.fromCharCode(65 + (sum % 26));
+  },
+  CZ: (b) => {
+    if (/^\d{8}$/.test(b)) {
+      const d = toDigits(b)!;
+      const r = weightedModBy(d.slice(0, 7), (i) => 8 - i, 11)!;
+      const c = r === 0 ? 1 : r === 1 ? 0 : 11 - r;
+      return c === d[7];
+    }
+    if (/^\d{10}$/.test(b)) return Number(b) % 11 === 0; // rodné číslo form
+    if (/^\d{9}$/.test(b)) return 'structural'; // pre-1954 personal, no check
+    return false;
+  },
+  DE: (b) => /^\d{9}$/.test(b) && mod11_10Valid(b.slice(0, 8), b[8]!),
+  DK: (b) => {
+    if (!/^\d{8}$/.test(b)) return false;
+    return weightedMod(toDigits(b)!, [2, 7, 6, 5, 4, 3, 2, 1], 11) === 0;
+  },
+  EE: (b) => {
+    if (!/^\d{9}$/.test(b)) return false;
+    const d = toDigits(b)!;
+    const sum = cyclicWeightedMod(d.slice(0, 8), [3, 7, 1], 10)!;
+    return (10 - sum) % 10 === d[8];
+  },
+  EL: (b) => {
+    if (!/^\d{9}$/.test(b) || /^0+$/.test(b)) return false;
+    const d = toDigits(b)!;
+    return weightedModBy(d.slice(0, 8), (i) => 2 ** (8 - i), 11)! % 10 === d[8];
+  },
+  ES: esVatValid,
+  FI: (b) => {
+    if (!/^\d{8}$/.test(b)) return false;
+    const d = toDigits(b)!;
+    const r = weightedMod(d.slice(0, 7), [7, 9, 10, 5, 8, 4, 2], 11)!;
+    if (r === 1) return false;
+    return (r === 0 ? 0 : 11 - r) === d[7];
+  },
+  FR: (b) => {
+    if (!/^[0-9A-Z]{2}\d{9}$/.test(b)) return false;
+    const siren = b.slice(2);
+    if (!luhnValid(siren)) return false;
+    if (isAllDigits(b.slice(0, 2))) {
+      return Number(b.slice(0, 2)) === (12 + 3 * (Number(siren) % 97)) % 97;
+    }
+    return 'structural'; // letter keys: no published arithmetic
+  },
+  HR: (b) => /^\d{11}$/.test(b) && mod11_10Valid(b.slice(0, 10), b[10]!),
+  HU: (b) => {
+    if (!/^\d{8}$/.test(b)) return false;
+    const d = toDigits(b)!;
+    const sum = weightedMod(d.slice(0, 7), [9, 7, 3, 1, 9, 7, 3], 10)!;
+    return (10 - sum) % 10 === d[7];
+  },
+  IE: (b) => {
+    const m = /^(\d{7})([A-W])([AH])?$/.exec(b);
+    if (m === null) return false;
+    let sum = weightedSum(toDigits(m[1]!)!, [8, 7, 6, 5, 4, 3, 2])!;
+    if (m[3] !== undefined) sum += 9 * (m[3].charCodeAt(0) - 64);
+    return m[2] === MOD23[sum % 23];
+  },
+  IT: (b) => /^\d{11}$/.test(b) && !/^0{7}/.test(b) && luhnValid(b),
+  LT: (b) => {
+    const twoPhase = (d: readonly number[], n: number): number => {
+      let r = weightedModBy(d.slice(0, n), (i) => (i % 9) + 1, 11)!;
+      if (r === 10) {
+        r = weightedModBy(d.slice(0, n), (i) => ((i + 2) % 9) + 1, 11)!;
+        if (r === 10) r = 0;
+      }
+      return r;
+    };
+    if (/^\d{9}$/.test(b)) {
+      const d = toDigits(b)!;
+      return twoPhase(d, 8) === d[8];
+    }
+    if (/^\d{12}$/.test(b)) {
+      const d = toDigits(b)!;
+      return twoPhase(d, 11) === d[11];
+    }
+    return false;
+  },
+  LU: (b) => /^\d{8}$/.test(b) && Number(b.slice(0, 6)) % 89 === Number(b.slice(6)),
+  LV: (b) => {
+    if (!/^\d{11}$/.test(b)) return false;
+    const d = toDigits(b)!;
+    let sum = 0;
+    const weights = [9, 1, 4, 8, 3, 10, 2, 5, 7, 6];
+    for (let i = 0; i < 10; i++) sum += d[i]! * weights[i]!;
+    const c = (((3 - sum) % 11) + 11) % 11;
+    if (c === 10) return false;
+    return c === d[10];
+  },
+  MT: (b) => {
+    if (!/^\d{8}$/.test(b)) return false;
+    const d = toDigits(b)!;
+    const r = weightedMod(d.slice(0, 6), [3, 4, 6, 7, 8, 9], 37)!;
+    const expected = 37 - r === 0 ? 37 : 37 - r;
+    return expected === Number(b.slice(6));
+  },
+  NL: (b) => {
+    if (!/^\d{9}B\d{2}$/.test(b)) return false;
+    const d = toDigits(b.slice(0, 9))!;
+    const r = weightedModBy(d.slice(0, 8), (i) => 9 - i, 11)!;
+    if (r !== 10 && r === d[8]) return true;
+    // Post-2020 sole traders: base-36 mod-97 over "NL" + the full body.
+    const numeric = [...`NL${b}`]
+      .map((ch) => (/\d/.test(ch) ? ch : String(ch.charCodeAt(0) - 55)))
+      .join('');
+    return modString(numeric, 97) === 1;
+  },
+  PL: (b) => {
+    if (!/^\d{10}$/.test(b)) return false;
+    const d = toDigits(b)!;
+    const r = weightedMod(d.slice(0, 9), [6, 5, 7, 2, 3, 4, 5, 6, 7], 11)!;
+    return r !== 10 && r === d[9];
+  },
+  PT: (b) => {
+    if (!/^\d{9}$/.test(b)) return false;
+    const d = toDigits(b)!;
+    const r = weightedModBy(d.slice(0, 8), (i) => 9 - i, 11)!;
+    return (r < 2 ? 0 : 11 - r) === d[8];
+  },
+  RO: (b) => {
+    if (!/^\d{2,10}$/.test(b) || b[0] === '0') return false;
+    const padded = b.padStart(10, '0');
+    const d = toDigits(padded)!;
+    const sum = weightedMod(d.slice(0, 9), [7, 5, 3, 2, 1, 7, 5, 3, 2], 11)!;
+    return ((sum * 10) % 11) % 10 === d[9];
+  },
+  SE: (b) => /^\d{10}01$/.test(b) && luhnValid(b.slice(0, 10)),
+  SI: (b) => {
+    if (!/^[1-9]\d{7}$/.test(b)) return false;
+    const d = toDigits(b)!;
+    const r = weightedModBy(d.slice(0, 7), (i) => 8 - i, 11)!;
+    const c = 11 - r;
+    if (c === 10) return false;
+    return (c === 11 ? 0 : c) === d[7];
+  },
+  SK: (b) => /^[1-9]\d[234789]\d{7}$/.test(b) && Number(b) % 11 === 0,
+};
+
+const PREFIXES = Object.keys(RULES).join('|');
+
+registerDetector({
+  id: 'eu-vat',
+  entityType: 'VAT_NUMBER',
+  regions: [
+    'AT', 'BE', 'BG', 'CY', 'CZ', 'DE', 'DK', 'EE', 'GR', 'ES', 'FI', 'FR', 'HR',
+    'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'MT', 'NL', 'PL', 'PT', 'RO', 'SE', 'SI', 'SK',
+  ],
+  pattern: new RegExp(`\\b(${PREFIXES})[ ]?([0-9A-Z]{2,13})\\b`, 'g'),
+  baseConfidence: CONFIDENCE.HIGH,
+  description: 'EU VAT numbers: per-country structure and checksum for all 27 member states.',
+  validate(ctx: ValidationContext): ValidationResult {
+    const country = ctx.match[1]!;
+    const body = ctx.match[2]!;
+    const rule = RULES[country];
+    if (rule === undefined) return invalid('not a member-state prefix');
+
+    const result = rule(body);
+    if (result === false) return invalid('national VAT rule failed');
+
+    return valid({
+      canonical: `${country}${body}`,
+      ...(result === 'structural' ? { confidence: CONFIDENCE.MEDIUM } : {}),
+      metadata: {
+        country: country === 'EL' ? 'GR' : country,
+        ...(result === 'structural' ? { checksum: 'structural-only' } : {}),
+      },
+      validator: result === 'structural' ? 'vat-structure' : 'vat-checksum',
+    });
+  },
+});

package/src/detect/detectors/vat/index.ts

@@ -0,0 +1,4 @@
+/**
+ * VAT family. Side-effect imports only.
+ */
+import './euVat.js';

package/src/detect/entropy.ts

@@ -0,0 +1,60 @@
+/**
+ * Shannon entropy, for the GENERIC_SECRET detector.
+ *
+ * SPEC.md requires GENERIC_SECRET to combine "a Shannon entropy threshold
+ * AND an assignment-context signal (see Stage 3)". This module is the
+ * entropy half; the context half is the Stage 3 hook that keeps the detector
+ * capped at low confidence until M7. Kept separate and pure so the threshold
+ * can be tuned against the eval corpus in M3 with the entropy computation
+ * itself under test.
+ */
+
+/**
+ * Shannon entropy of a string, in bits per character (0 for empty/one-char).
+ *
+ * H = −Σ p(c)·log₂ p(c) over the observed character distribution. A value's
+ * TOTAL entropy is `shannonEntropy(s) * s.length`; the per-character form is
+ * the more stable threshold because it does not grow with length.
+ */
+export function shannonEntropy(s: string): number {
+  if (s.length < 2) return 0;
+  const counts = new Map<string, number>();
+  for (const ch of s) counts.set(ch, (counts.get(ch) ?? 0) + 1);
+  let h = 0;
+  for (const count of counts.values()) {
+    const p = count / s.length;
+    h -= p * Math.log2(p);
+  }
+  return h;
+}
+
+/** Distinct-character count — a cheap complement to entropy. */
+export function distinctChars(s: string): number {
+  return new Set(s).size;
+}
+
+/**
+ * Shapes that clear an entropy bar but are provably NOT secrets. These are
+ * suppressed regardless of context, because SPEC.md names them explicitly as
+ * things that "do not trigger": UUIDs, git SHAs, base64 image data, and
+ * obvious placeholders.
+ */
+export function isKnownNonSecret(value: string): boolean {
+  // UUID (any version).
+  if (/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/.test(value)) {
+    return true;
+  }
+  // Git object name: exactly 40 or 64 lowercase hex (SHA-1 / SHA-256).
+  if (/^[0-9a-f]{40}$/.test(value) || /^[0-9a-f]{64}$/.test(value)) return true;
+  // Placeholder text.
+  if (/^(?:x+|X+|\*+|\.+|-+|0+)$/.test(value)) return true;
+  if (/your[_-]?(?:api[_-]?)?key/i.test(value)) return true;
+  if (/[_-]?here$/i.test(value) || /^(?:changeme|placeholder|redacted|example|sample|dummy|todo)$/i.test(value)) {
+    return true;
+  }
+  // A data: URI or a long run of pure base64 that is almost certainly a blob
+  // (image, file) rather than a credential — decided by the caller via
+  // length, but the data: scheme is unambiguous.
+  if (/^data:/i.test(value)) return true;
+  return false;
+}

package/src/detect/isoCountries.ts

@@ -0,0 +1,30 @@
+/**
+ * ISO 3166-1 alpha-2: the officially assigned codes.
+ *
+ * Shared by detectors that embed a country code in their format (SWIFT BIC,
+ * VAT numbers, MRZ nationality fields). Includes the exceptionally reserved
+ * codes that appear in real identifiers (EU in VAT, XK in IBAN/Kosovo
+ * documents, UK alongside GB) — noted inline.
+ */
+
+const ASSIGNED =
+  'AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ ' +
+  'BA BB BD BE BF BG BH BI BJ BL BM BN BO BQ BR BS BT BV BW BY BZ ' +
+  'CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ ' +
+  'DE DJ DK DM DO DZ EC EE EG EH ER ES ET FI FJ FK FM FO FR ' +
+  'GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY ' +
+  'HK HM HN HR HT HU ID IE IL IM IN IO IQ IR IS IT JE JM JO JP ' +
+  'KE KG KH KI KM KN KP KR KW KY KZ LA LB LC LI LK LR LS LT LU LV LY ' +
+  'MA MC MD ME MF MG MH MK ML MM MN MO MP MQ MR MS MT MU MV MW MX MY MZ ' +
+  'NA NC NE NF NG NI NL NO NP NR NU NZ OM ' +
+  'PA PE PF PG PH PK PL PM PN PR PS PT PW PY QA RE RO RS RU RW ' +
+  'SA SB SC SD SE SG SH SI SJ SK SL SM SN SO SR SS ST SV SX SY SZ ' +
+  'TC TD TF TG TH TJ TK TL TM TN TO TR TT TV TW TZ ' +
+  'UA UG UM US UY UZ VA VC VE VG VI VN VU WF WS YE YT ZA ZM ZW';
+
+/** Exceptionally reserved / user-assigned codes seen in real identifiers. */
+const RESERVED_IN_USE = 'EU UK XK';
+
+export const ISO_COUNTRY_CODES: ReadonlySet<string> = new Set(
+  `${ASSIGNED} ${RESERVED_IN_USE}`.split(' '),
+);

package/src/detect/registry.ts

@@ -0,0 +1,134 @@
+/**
+ * The Stage 1 detector registry.
+ *
+ * SPEC.md: "Adding a new national identifier must require touching exactly
+ * one new file." This module is what makes that literally true. A detector
+ * file declares its `Detector` and calls `registerDetector(...)`; the only
+ * other edit anywhere is the one-line side-effect import in
+ * `detectors/index.ts` that causes the file to be loaded at all.
+ *
+ * WHY SELF-REGISTRATION RATHER THAN A CENTRAL ARRAY. A central array is a
+ * merge-conflict magnet once there are 145 detectors, and it silently permits
+ * two detectors to share an id. Registration validates each detector as it
+ * arrives and fails loudly on a duplicate id or a malformed pattern, at
+ * module-load time, so the problem surfaces in the failing test that imported
+ * it rather than as a mysteriously missing detection later.
+ */
+
+import type { Detector, RegionCode } from './types.js';
+import { GLOBAL_REGION } from './types.js';
+
+/** Every registered detector, keyed by id so duplicates are impossible. */
+const registry = new Map<string, Detector>();
+
+/** Detector ids must be lowercase kebab-case: stable, greppable, url-safe. */
+const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
+
+/** Region codes are ISO 3166-1 alpha-2, or the GLOBAL sentinel. */
+const REGION_PATTERN = /^[A-Z]{2}$/;
+
+/**
+ * Register a detector.
+ *
+ * Throws on any contract violation. Failing at load time is deliberate: a
+ * detector that silently fails to register is a silent hole in detection
+ * coverage, which SPEC.md's fail-closed posture does not tolerate.
+ */
+export function registerDetector(detector: Detector): void {
+  assertValidDetector(detector);
+  if (registry.has(detector.id)) {
+    throw new Error(
+      `Duplicate detector id "${detector.id}". Ids must be unique across the whole registry.`,
+    );
+  }
+  registry.set(detector.id, detector);
+}
+
+/** Every registered detector, in registration order. */
+export function allDetectors(): readonly Detector[] {
+  return [...registry.values()];
+}
+
+/** Look one up by id, or `undefined`. */
+export function getDetector(id: string): Detector | undefined {
+  return registry.get(id);
+}
+
+/**
+ * Detectors applicable to a region.
+ *
+ * Always includes `GLOBAL` detectors. Passing `undefined` returns everything,
+ * which is the default scanning posture: a user pasting a foreign colleague's
+ * national ID should still have it detected.
+ */
+export function detectorsForRegion(region?: RegionCode): readonly Detector[] {
+  if (region === undefined) return allDetectors();
+  return allDetectors().filter(
+    (d) => d.regions.includes(GLOBAL_REGION) || d.regions.includes(region),
+  );
+}
+
+/** Detectors emitting a given entity type. */
+export function detectorsForEntityType(entityType: Detector['entityType']): readonly Detector[] {
+  return allDetectors().filter((d) => d.entityType === entityType);
+}
+
+/** How many detectors are registered. Used by the coverage test. */
+export function detectorCount(): number {
+  return registry.size;
+}
+
+/**
+ * Remove every registration.
+ *
+ * Exists ONLY for tests that need an isolated registry. Production code never
+ * calls this — the registry is populated once, at module load, and is
+ * effectively immutable thereafter.
+ */
+export function resetRegistryForTesting(): void {
+  registry.clear();
+}
+
+/**
+ * Validate a detector's declared contract.
+ *
+ * Every check here corresponds to a mistake that would otherwise produce a
+ * subtly wrong detection rather than an obvious crash.
+ */
+function assertValidDetector(d: Detector): void {
+  if (!ID_PATTERN.test(d.id)) {
+    throw new Error(`Detector id "${d.id}" must be lowercase kebab-case.`);
+  }
+
+  if (d.regions.length === 0) {
+    throw new Error(`Detector "${d.id}" declares no regions; use [GLOBAL_REGION] if universal.`);
+  }
+  for (const region of d.regions) {
+    if (region !== GLOBAL_REGION && !REGION_PATTERN.test(region)) {
+      throw new Error(
+        `Detector "${d.id}" declares region "${region}"; expected an ISO 3166-1 alpha-2 code or "${GLOBAL_REGION}".`,
+      );
+    }
+  }
+
+  // A non-global pattern silently resumes from a stale lastIndex on its second
+  // use, which shows up as "the detector works in isolation but misses matches
+  // in a full scan" — precisely the bug class worth making impossible.
+  if (!d.pattern.global) {
+    throw new Error(`Detector "${d.id}" pattern must carry the /g flag.`);
+  }
+
+  if (!(d.baseConfidence > 0 && d.baseConfidence <= 1)) {
+    throw new Error(
+      `Detector "${d.id}" baseConfidence must be in (0, 1]; got ${String(d.baseConfidence)}.`,
+    );
+  }
+
+  if (typeof d.validate !== 'function') {
+    throw new Error(`Detector "${d.id}" must supply a validate() function.`);
+  }
+
+  if (d.description.trim().length === 0) {
+    throw new Error(`Detector "${d.id}" must supply a non-empty description.`);
+  }
+}