@discretion/core 0.1.0

package/src/detect/detectors/secrets/apiKey.ts

@@ -0,0 +1,115 @@
+/**
+ * API_KEY — known-prefix provider tokens, driven by the data-file table.
+ *
+ * SPEC.md wants providers addable "without code changes", so this single
+ * detector reads @discretion/data's SECRET_PROVIDERS. The pattern is the
+ * alternation of every prefix; the validator resolves the longest matching
+ * prefix, checks the body charset and length, and runs the provider's
+ * checksum where one exists (GitHub's base62 CRC32).
+ *
+ * Stripe TEST keys (sk_test_, pk_test_) are handled as a non-sensitive
+ * sibling of the live prefixes: detected so eval sees them, never masked.
+ */
+
+import { SECRET_PROVIDERS, SECRET_CHARSET_PATTERN } from '@discretion/data';
+import type { SecretProvider } from '@discretion/data';
+import { registerDetector } from '../../registry.js';
+import { CONFIDENCE, GLOBAL_REGION, invalid, valid } from '../../types.js';
+import type { ValidationContext, ValidationResult } from '../../types.js';
+
+/** GitHub token CRC32 over the 30-char random body → 6-char base62 check. */
+const BASE62 = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
+
+function crc32(s: string): number {
+  let crc = 0xffffffff;
+  for (let i = 0; i < s.length; i++) {
+    crc ^= s.charCodeAt(i);
+    for (let b = 0; b < 8; b++) {
+      crc = crc & 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
+    }
+  }
+  return (crc ^ 0xffffffff) >>> 0;
+}
+
+/** GitHub's check: last 6 base62 chars encode crc32 of the preceding body. */
+function githubChecksumValid(body: string): boolean {
+  if (body.length < 7) return false;
+  const random = body.slice(0, body.length - 6);
+  const check = body.slice(body.length - 6);
+  let value = crc32(random);
+  let encoded = '';
+  for (let i = 0; i < 6; i++) {
+    encoded = BASE62[value % 62]! + encoded;
+    value = Math.floor(value / 62);
+  }
+  return encoded === check;
+}
+
+/** All prefixes, longest first, each paired with its provider. */
+const PREFIX_INDEX: readonly (readonly [string, SecretProvider])[] = SECRET_PROVIDERS.flatMap((p) =>
+  p.prefixes.map((prefix) => [prefix, p] as const),
+).sort((a, b) => b[0].length - a[0].length);
+
+/** Stripe test-key prefixes: detected, non-sensitive. */
+const STRIPE_TEST_PREFIXES = ['sk_test_', 'pk_test_', 'rk_test_'];
+
+function validateApiKey(ctx: ValidationContext): ValidationResult {
+  const raw = ctx.match[0];
+
+  for (const testPrefix of STRIPE_TEST_PREFIXES) {
+    if (raw.startsWith(testPrefix)) {
+      return valid({
+        canonical: raw,
+        sensitive: false,
+        metadata: { provider: 'stripe', test: true },
+        validator: 'provider-prefix',
+      });
+    }
+  }
+
+  for (const [prefix, provider] of PREFIX_INDEX) {
+    if (!raw.startsWith(prefix)) continue;
+    const body = raw.slice(prefix.length);
+    const [min, max] = provider.bodyLength;
+    if (body.length < min || body.length > max) return invalid('body length outside provider window');
+    if (!SECRET_CHARSET_PATTERN[provider.bodyCharset].test(body.replace(/[.]/g, ''))) {
+      return invalid('body charset mismatch');
+    }
+    if (provider.checksum === 'github' && !githubChecksumValid(body)) {
+      return invalid('GitHub token checksum failed');
+    }
+    return valid({
+      canonical: raw,
+      metadata: { provider: provider.id, ...(provider.checksum !== undefined ? { checksum: provider.checksum } : {}) },
+      validator: provider.checksum !== undefined ? `${provider.id}-checksum` : 'provider-prefix',
+    });
+  }
+
+  return invalid('no known provider prefix');
+}
+
+/** Build one alternation of every prefix, longest first, regex-escaped. */
+const PREFIX_ALTERNATION = PREFIX_INDEX.map(([prefix]) => prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+  .concat(STRIPE_TEST_PREFIXES)
+  .join('|');
+
+registerDetector({
+  id: 'api-key',
+  entityType: 'API_KEY',
+  regions: [GLOBAL_REGION],
+  // The body charset must include `.` - plenty of provider tokens contain one
+  // - but the match may not END with one, or a key at the end of a sentence
+  // swallows the full stop. Found at M12 while running a README example:
+  // `sk_live_...tY6.` matched WITH the period, so the masked text lost the
+  // punctuation and, worse, the vault keyed on a value that differs from the
+  // same key written mid-sentence - which quietly breaks the guarantee that a
+  // value seen twice gets the same surrogate.
+  //
+  // Only `.` is excluded. Trailing `-` and `_` are also unlikely to be part of
+  // a real token, but they were not what this found, and tightening them here
+  // would be an unmeasured change riding on a measured one.
+  pattern: new RegExp(`(?:${PREFIX_ALTERNATION})[A-Za-z0-9._-]{9,129}[A-Za-z0-9_-]`, 'g'),
+  baseConfidence: CONFIDENCE.HIGH,
+  description: 'Provider API tokens from the bundled prefix table; GitHub checksums verified.',
+  validate: validateApiKey,
+});

package/src/detect/detectors/secrets/connectionString.ts

@@ -0,0 +1,71 @@
+/**
+ * CONNECTION_STRING — database URIs with embedded credentials.
+ *
+ * SPEC.md: "database URIs with embedded credentials." The signal is a
+ * database scheme plus a password in the userinfo. A scheme with only a
+ * username (or none) is a connection string but not a SECRET — it leaks no
+ * credential — so it is not emitted here; a bare host:port DSN is likewise
+ * out of scope for a credential detector. The whole URI is the span, for the
+ * same reason as URL_WITH_CREDENTIALS: a partial mask still leaks the host.
+ */
+
+import { registerDetector } from '../../registry.js';
+import { CONFIDENCE, GLOBAL_REGION, invalid, valid } from '../../types.js';
+import type { ValidationContext, ValidationResult } from '../../types.js';
+
+const DB_SCHEMES = new Set([
+  'postgres', 'postgresql', 'mysql', 'mariadb', 'mongodb', 'mongodb+srv',
+  'redis', 'rediss', 'amqp', 'amqps', 'mssql', 'sqlserver', 'jdbc',
+  'cassandra', 'couchbase', 'clickhouse', 'cockroachdb', 'db2', 'oracle',
+]);
+
+function isPlaceholder(value: string): boolean {
+  if (/^\$\{[^}]*\}$/.test(value) || /^\{\{[^}]*\}\}$/.test(value) || /^<[^>]*>$/.test(value)) return true;
+  if (/^(?:x+|\*+|password|passwd|pass|changeme|placeholder)$/i.test(value)) return true;
+  if (/^YOUR[_-]/i.test(value)) return true;
+  return false;
+}
+
+function validateConnectionString(ctx: ValidationContext): ValidationResult {
+  const raw = ctx.match[0];
+  const schemeEnd = raw.indexOf('://');
+  if (schemeEnd < 0) return invalid('no scheme');
+  const scheme = raw.slice(0, schemeEnd).toLowerCase();
+  if (!DB_SCHEMES.has(scheme)) return invalid('not a database scheme');
+
+  const rest = raw.slice(schemeEnd + 3);
+  const authorityEnd = rest.search(/[/?#]/);
+  const authority = authorityEnd < 0 ? rest : rest.slice(0, authorityEnd);
+  const at = authority.lastIndexOf('@');
+  if (at <= 0) return invalid('no userinfo credential');
+  const userinfo = authority.slice(0, at);
+  const colon = userinfo.indexOf(':');
+  if (colon <= 0 || colon === userinfo.length - 1) return invalid('userinfo carries no password');
+
+  const password = userinfo.slice(colon + 1);
+  const host = authority.slice(at + 1);
+  const placeholder = isPlaceholder(password);
+
+  return valid({
+    canonical: raw,
+    sensitive: !placeholder,
+    metadata: { scheme, host },
+    validator: 'connection-string',
+  });
+}
+
+registerDetector({
+  id: 'connection-string',
+  entityType: 'CONNECTION_STRING',
+  regions: [GLOBAL_REGION],
+  // Commas and pipes terminate the URI, for the reason recorded on the
+  // credentialled-URL detector: in CSV rows and pipe-delimited logs they are
+  // field boundaries, and an over-long span makes masking overwrite the
+  // user's adjacent cells. `;` is deliberately NOT excluded — a JDBC
+  // connection string carries its properties after one, and cutting there
+  // would truncate the credential this detector exists to find.
+  pattern: /\b[a-zA-Z][a-zA-Z0-9+]*:\/\/[^\s"'<>,|]*:[^\s"'<>,|@]*@[^\s"'<>,|]{2,300}/g,
+  baseConfidence: CONFIDENCE.HIGH,
+  description: 'Database connection URIs carrying a password in userinfo; whole-URI span.',
+  validate: validateConnectionString,
+});

package/src/detect/detectors/secrets/genericSecret.ts

@@ -0,0 +1,81 @@
+/**
+ * GENERIC_SECRET — high-entropy strings.
+ *
+ * SPEC.md is emphatic and specific: "Require a Shannon entropy threshold AND
+ * an assignment-context signal (see Stage 3) so that 'your-api-key-here',
+ * 'xxxxxxxxxxxx', UUIDs, git SHAs, and base64 image data do not trigger."
+ *
+ * Stage 3 does not exist in M2. So this detector is built in its FINAL shape
+ * but declares `requiresContext: true`, which the runner honours by capping
+ * it at CONFIDENCE.LOW until a context signal is supplied. It therefore
+ * never "fires on entropy alone" — the structural guarantee SPEC.md wants.
+ * The entropy computation and the known-non-secret suppressions are fully
+ * implemented and tested now; only the confidence ceiling is deferred.
+ *
+ * The threshold below is provisional. SPEC.md requires it be "tuned
+ * empirically against the eval corpus" with the precision/recall curve
+ * documented — that is M3 work. Until then it is deliberately conservative
+ * (high), because at LOW confidence a miss is cheap and a false positive is
+ * the thing SPEC.md warns hardest against.
+ */
+
+import { registerDetector } from '../../registry.js';
+import { CONFIDENCE, GLOBAL_REGION, invalid, valid } from '../../types.js';
+import type { ValidationContext, ValidationResult } from '../../types.js';
+import { shannonEntropy, distinctChars, isKnownNonSecret } from '../../entropy.js';
+
+/**
+ * Provisional per-character entropy floor, in bits. A random base62 string
+ * approaches log2(62) ≈ 5.95; english prose sits near 3–4. 3.5 admits
+ * plausible keys while excluding words. Retuned against the corpus in M3.
+ */
+const ENTROPY_FLOOR = 3.5;
+
+/** A secret needs both length and character diversity, not just one. */
+const MIN_LENGTH = 20;
+const MIN_DISTINCT = 12;
+
+function validateGenericSecret(ctx: ValidationContext): ValidationResult {
+  const value = ctx.match[0];
+
+  if (value.length < MIN_LENGTH) return invalid('too short');
+  if (isKnownNonSecret(value)) return invalid('known non-secret shape');
+  if (distinctChars(value) < MIN_DISTINCT) return invalid('too few distinct characters');
+
+  const entropy = shannonEntropy(value);
+  if (entropy < ENTROPY_FLOOR) return invalid('entropy below threshold');
+
+  // A run of a single character class that is all letters is far more likely
+  // an identifier or a word-ish token than a key; require mixed classes.
+  const classes = [/[a-z]/, /[A-Z]/, /[0-9]/, /[^A-Za-z0-9]/].filter((re) => re.test(value)).length;
+  if (classes < 2) return invalid('single character class');
+
+  // Return the natural confidence (baseConfidence, MEDIUM) and let the runner
+  // apply the requiresContext cap: no Stage 3 signal → capped at LOW, a signal
+  // present → allowed up to base. Hardcoding LOW here would defeat the cap by
+  // pinning it even when context arrives, which the test below pins.
+  return valid({
+    // No canonicalization: a secret is matched verbatim.
+    metadata: {
+      entropyBitsPerChar: Number(entropy.toFixed(3)),
+      distinct: distinctChars(value),
+      // Flags for Stage 3 fusion to combine with an assignment signal.
+      awaitingContext: true,
+    },
+    validator: 'shannon-entropy',
+  });
+}
+
+registerDetector({
+  id: 'generic-secret',
+  entityType: 'GENERIC_SECRET',
+  regions: [GLOBAL_REGION],
+  // Long token-shaped runs: letters, digits, and the punctuation common in
+  // keys. Deliberately broad; the validator and the context requirement do
+  // the discriminating.
+  pattern: /\b[A-Za-z0-9+/_=-]{20,120}\b/g,
+  baseConfidence: CONFIDENCE.MEDIUM,
+  requiresContext: true,
+  description: 'High-entropy secrets: entropy gate now, Stage 3 assignment context required for confidence.',
+  validate: validateGenericSecret,
+});

package/src/detect/detectors/secrets/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Secrets & credentials family. Side-effect imports only.
+ */
+import './apiKey.js';
+import './jwt.js';
+import './pemPrivateKey.js';
+import './connectionString.js';
+import './genericSecret.js';

package/src/detect/detectors/secrets/jwt.ts

@@ -0,0 +1,101 @@
+/**
+ * JWT — structural validation.
+ *
+ * SPEC.md: "three base64url segments, header decodes to valid JSON with an
+ * alg field." That is exactly the bar: a JWT's signature cannot be verified
+ * without its key, so structure is all there is — but a well-formed header
+ * with a real `alg` is a strong and specific signal. base64url is decoded
+ * by hand (atob is a DOM global, banned in core).
+ */
+
+import { registerDetector } from '../../registry.js';
+import { CONFIDENCE, GLOBAL_REGION, invalid, valid } from '../../types.js';
+import type { ValidationContext, ValidationResult } from '../../types.js';
+
+/** Registered JWS algorithms plus 'none'; a header alg must be one of these. */
+const JWT_ALGS = new Set([
+  'none',
+  'HS256', 'HS384', 'HS512',
+  'RS256', 'RS384', 'RS512',
+  'ES256', 'ES256K', 'ES384', 'ES512',
+  'PS256', 'PS384', 'PS512',
+  'EdDSA',
+]);
+
+function base64urlDecode(segment: string): string | null {
+  if (!/^[A-Za-z0-9_-]+$/.test(segment)) return null;
+  const b64 = segment.replace(/-/g, '+').replace(/_/g, '/');
+  const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+  let bits = 0;
+  let acc = 0;
+  const bytes: number[] = [];
+  for (const ch of b64) {
+    const v = chars.indexOf(ch);
+    if (v < 0) return null;
+    acc = (acc << 6) | v;
+    bits += 6;
+    if (bits >= 8) {
+      bits -= 8;
+      bytes.push((acc >> bits) & 0xff);
+    }
+  }
+  // Decode UTF-8 bytes to a string.
+  try {
+    let out = '';
+    let i = 0;
+    while (i < bytes.length) {
+      const b0 = bytes[i]!;
+      if (b0 < 0x80) { out += String.fromCharCode(b0); i += 1; }
+      else if (b0 < 0xe0) { out += String.fromCharCode(((b0 & 0x1f) << 6) | (bytes[i + 1]! & 0x3f)); i += 2; }
+      else if (b0 < 0xf0) { out += String.fromCharCode(((b0 & 0x0f) << 12) | ((bytes[i + 1]! & 0x3f) << 6) | (bytes[i + 2]! & 0x3f)); i += 3; }
+      else {
+        const cp = ((b0 & 0x07) << 18) | ((bytes[i + 1]! & 0x3f) << 12) | ((bytes[i + 2]! & 0x3f) << 6) | (bytes[i + 3]! & 0x3f);
+        out += String.fromCodePoint(cp);
+        i += 4;
+      }
+    }
+    return out;
+  } catch {
+    return null;
+  }
+}
+
+function validateJwt(ctx: ValidationContext): ValidationResult {
+  const parts = ctx.match[0].split('.');
+  if (parts.length !== 3) return invalid('not three segments');
+  const [headerSeg, payloadSeg, signatureSeg] = parts as [string, string, string];
+  if (signatureSeg.length === 0) return invalid('empty signature segment');
+
+  const headerJson = base64urlDecode(headerSeg);
+  if (headerJson === null) return invalid('header is not base64url');
+
+  let header: unknown;
+  try {
+    header = JSON.parse(headerJson);
+  } catch {
+    return invalid('header is not JSON');
+  }
+  if (typeof header !== 'object' || header === null) return invalid('header is not a JSON object');
+  const alg = (header as Record<string, unknown>)['alg'];
+  if (typeof alg !== 'string' || !JWT_ALGS.has(alg)) return invalid('header has no valid alg');
+
+  // The payload must at least decode as base64url; it need not be JSON
+  // (though it usually is) — some tokens carry non-JSON claims sets.
+  if (base64urlDecode(payloadSeg) === null) return invalid('payload is not base64url');
+
+  return valid({
+    canonical: ctx.match[0],
+    metadata: { alg },
+    validator: 'jwt-structural',
+  });
+}
+
+registerDetector({
+  id: 'jwt',
+  entityType: 'JWT',
+  regions: [GLOBAL_REGION],
+  pattern: /\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{2,}\b/g,
+  baseConfidence: CONFIDENCE.HIGH,
+  description: 'JSON Web Tokens: three base64url segments, header JSON carrying a valid alg.',
+  validate: validateJwt,
+});

package/src/detect/detectors/secrets/pemPrivateKey.ts

@@ -0,0 +1,59 @@
+/**
+ * PRIVATE_KEY — PEM private-key blocks.
+ *
+ * A whole armored block, header through footer, is matched and the span
+ * covers all of it: a private key with its header stripped is still a
+ * private key, and substitution must replace the entire block. The type is
+ * PRIVATE_KEY (its own EntityType, split from API_KEY in ARCHITECTURE.md D4)
+ * because a leaked private key's blast radius and its surrogate differ from
+ * a token's. Confidence is MAXIMUM: a well-formed PEM private-key block is
+ * essentially never a false positive.
+ *
+ * PUBLIC-key blocks and CERTIFICATE blocks are deliberately excluded — they
+ * are not secret.
+ */
+
+import { registerDetector } from '../../registry.js';
+import { CONFIDENCE, GLOBAL_REGION, invalid, valid } from '../../types.js';
+import type { ValidationContext, ValidationResult } from '../../types.js';
+
+const KEY_LABELS = [
+  'PRIVATE KEY',
+  'RSA PRIVATE KEY',
+  'EC PRIVATE KEY',
+  'DSA PRIVATE KEY',
+  'OPENSSH PRIVATE KEY',
+  'PGP PRIVATE KEY BLOCK',
+  'ENCRYPTED PRIVATE KEY',
+];
+
+function validatePem(ctx: ValidationContext): ValidationResult {
+  const label = ctx.match[1];
+  const footerLabel = ctx.match[3];
+  if (label === undefined || !KEY_LABELS.includes(label)) return invalid('not a private-key label');
+  if (label !== footerLabel) return invalid('header and footer labels differ');
+
+  const body = ctx.match[2] ?? '';
+  // The body between the armor lines must contain real base64 content, not
+  // just whitespace — an empty pair of armor lines is a template, not a key.
+  if (!/[A-Za-z0-9+/]{16,}/.test(body)) return invalid('no key material between armor lines');
+
+  return valid({
+    canonical: ctx.match[0].trim(),
+    confidence: CONFIDENCE.MAXIMUM,
+    metadata: { keyType: label },
+    validator: 'pem-armor',
+  });
+}
+
+registerDetector({
+  id: 'pem-private-key',
+  entityType: 'PRIVATE_KEY',
+  regions: [GLOBAL_REGION],
+  // Header, body, footer. [\s\S] so the body can cross newlines; the label
+  // is captured twice so the validator can require they match.
+  pattern: /-----BEGIN ([A-Z0-9 ]+?)-----([\s\S]*?)-----END ([A-Z0-9 ]+?)-----/g,
+  baseConfidence: CONFIDENCE.MAXIMUM,
+  description: 'PEM private-key blocks (RSA/EC/DSA/OpenSSH/PGP/PKCS8), whole-block span.',
+  validate: validatePem,
+});