@discretion/core 0.1.0

package/src/fuse/calibrate.ts

@@ -0,0 +1,224 @@
+/**
+ * Stage 4 — CALIBRATION.
+ *
+ * SPEC.md: "Combine scores from all stages into a single calibrated confidence
+ * per candidate. Calibrate against the eval corpus so that a confidence of 0.8
+ * empirically means roughly 80% precision. Document the calibration method and
+ * the resulting reliability curve."
+ *
+ * THE METHOD: ISOTONIC REGRESSION, fitted per entity type.
+ *
+ * Isotonic rather than Platt scaling (a fitted logistic) for two reasons that
+ * matter here. First, the raw scores are not a logistic function of the truth
+ * and there is no reason to expect them to be: they are a base confidence plus
+ * a handful of additive Stage 3 contributions, so the shape of the mapping is
+ * unknown and a two-parameter family would impose one. Second, isotonic
+ * regression is EXPLAINABLE — the fitted model is a step function over score
+ * bins whose steps are empirical precisions, so "0.8 means 80%" can be read
+ * directly off the table rather than inferred from coefficients.
+ *
+ * What isotonic buys over raw binned precision is MONOTONICITY. A bin with few
+ * samples can easily show lower precision than the bin below it, which would
+ * make calibrated confidence non-monotonic in the evidence and produce the
+ * absurd result that adding evidence lowers confidence. Pool-adjacent-
+ * violators merges such bins, so the mapping never decreases.
+ *
+ * PER TYPE, because the types are not comparable. A validated IBAN and a
+ * shape-only postal code can carry the same raw score and mean entirely
+ * different things — that non-comparability is stated in the eval's own
+ * header and is exactly what calibration exists to remove. Types with too few
+ * samples to fit fall back to a pooled curve rather than to an overfitted one.
+ *
+ * FITTED AND EVALUATED ON DISJOINT DOCUMENTS. A curve fitted and scored on the
+ * same corpus measures nothing; the split is stated in BENCHMARKS.md.
+ */
+
+import type { EntityType } from '../types.js';
+
+/** One step of a fitted curve: scores in [from, to) map to `precision`. */
+export interface CalibrationBin {
+  readonly from: number;
+  readonly to: number;
+  /** Empirical precision of candidates whose raw score fell in this bin. */
+  readonly precision: number;
+  /** How many candidates the step was fitted on, for reporting weight. */
+  readonly samples: number;
+}
+
+export interface CalibrationCurve {
+  readonly bins: readonly CalibrationBin[];
+  readonly samples: number;
+}
+
+export interface CalibrationModel {
+  /** Per-type curves. Types absent here use `pooled`. */
+  readonly perType: Readonly<Partial<Record<EntityType, CalibrationCurve>>>;
+  /** Fallback for types with too little data to fit their own curve. */
+  readonly pooled: CalibrationCurve;
+  /** Documents the model was fitted on, for the split statement. */
+  readonly fittedOn: string;
+}
+
+/** One observation: a candidate's raw score and whether it was correct. */
+export interface CalibrationSample {
+  readonly type: EntityType;
+  readonly score: number;
+  readonly correct: boolean;
+}
+
+/** Bins per curve. Ten keeps each step readable and populated. */
+const BIN_COUNT = 10;
+
+/**
+ * Minimum samples before a type gets its own curve.
+ *
+ * Below this the per-type curve would be fitted on noise, and a confidently
+ * wrong calibration is worse than an honest pooled one.
+ */
+const MIN_SAMPLES_PER_TYPE = 200;
+
+/**
+ * Pool adjacent violators: the isotonic regression fit.
+ *
+ * Walks left to right merging any bin whose precision is below its left
+ * neighbour, which yields the closest non-decreasing sequence in least
+ * squares. The merge is sample-weighted, so a large well-measured bin is not
+ * dragged by a tiny noisy one.
+ */
+function poolAdjacentViolators(
+  raw: readonly { from: number; to: number; correct: number; total: number }[],
+): CalibrationBin[] {
+  const stack: { from: number; to: number; correct: number; total: number }[] = [];
+
+  for (const bin of raw) {
+    if (bin.total === 0) continue;
+    stack.push({ ...bin });
+    // Merge backwards while the sequence decreases.
+    while (stack.length > 1) {
+      const right = stack[stack.length - 1]!;
+      const left = stack[stack.length - 2]!;
+      if (left.correct / left.total <= right.correct / right.total) break;
+      stack.splice(stack.length - 2, 2, {
+        from: left.from,
+        to: right.to,
+        correct: left.correct + right.correct,
+        total: left.total + right.total,
+      });
+    }
+  }
+
+  return stack.map((b) => ({
+    from: b.from,
+    to: b.to,
+    precision: b.correct / b.total,
+    samples: b.total,
+  }));
+}
+
+function fitCurve(samples: readonly CalibrationSample[]): CalibrationCurve {
+  const raw = Array.from({ length: BIN_COUNT }, (_, i) => ({
+    from: i / BIN_COUNT,
+    to: (i + 1) / BIN_COUNT,
+    correct: 0,
+    total: 0,
+  }));
+
+  for (const s of samples) {
+    const index = Math.min(BIN_COUNT - 1, Math.max(0, Math.floor(s.score * BIN_COUNT)));
+    const bin = raw[index]!;
+    bin.total += 1;
+    if (s.correct) bin.correct += 1;
+  }
+
+  return { bins: poolAdjacentViolators(raw), samples: samples.length };
+}
+
+/** Fit a calibration model from labelled observations. */
+export function fitCalibration(
+  samples: readonly CalibrationSample[],
+  fittedOn: string,
+): CalibrationModel {
+  const byType = new Map<EntityType, CalibrationSample[]>();
+  for (const s of samples) {
+    const list = byType.get(s.type) ?? [];
+    list.push(s);
+    byType.set(s.type, list);
+  }
+
+  const perType: Partial<Record<EntityType, CalibrationCurve>> = {};
+  for (const [type, list] of byType) {
+    if (list.length >= MIN_SAMPLES_PER_TYPE) perType[type] = fitCurve(list);
+  }
+
+  return { perType, pooled: fitCurve(samples), fittedOn };
+}
+
+/**
+ * Map a raw score to a calibrated probability.
+ *
+ * PIECEWISE CONSTANT — the containing step's empirical precision, which is the
+ * standard prediction for an isotonic fit. An earlier revision interpolated
+ * between step midpoints to smooth the output, and that was an embellishment
+ * with no justification behind it: because the step function is coarse where
+ * data is sparse, interpolating systematically pulled predictions toward the
+ * step below and left the model under-confident through 0.7-0.8. Returning the
+ * step's own measured precision is both the standard method and the one whose
+ * meaning is readable — "candidates in this band were right this often".
+ */
+export function calibrate(model: CalibrationModel, type: EntityType, score: number): number {
+  const curve = model.perType[type] ?? model.pooled;
+  if (curve.bins.length === 0) return score;
+
+  const clamped = Math.min(1, Math.max(0, score));
+  const index = curve.bins.findIndex((b) => clamped < b.to);
+  const bin = curve.bins[index === -1 ? curve.bins.length - 1 : index]!;
+  return bin.precision;
+}
+
+/** A reliability-curve row: what the model promised versus what happened. */
+export interface ReliabilityPoint {
+  readonly bucket: string;
+  readonly predicted: number;
+  readonly observed: number;
+  readonly samples: number;
+}
+
+/**
+ * Measure a fitted model against HELD-OUT observations.
+ *
+ * This is the number that means something: a curve scored on the documents it
+ * was fitted on measures memorisation, not calibration.
+ */
+export function reliability(
+  model: CalibrationModel,
+  heldOut: readonly CalibrationSample[],
+  buckets = 10,
+): { readonly points: readonly ReliabilityPoint[]; readonly expectedCalibrationError: number } {
+  const acc = Array.from({ length: buckets }, () => ({ predicted: 0, correct: 0, total: 0 }));
+
+  for (const s of heldOut) {
+    const p = calibrate(model, s.type, s.score);
+    const index = Math.min(buckets - 1, Math.max(0, Math.floor(p * buckets)));
+    const bucket = acc[index]!;
+    bucket.predicted += p;
+    bucket.total += 1;
+    if (s.correct) bucket.correct += 1;
+  }
+
+  const points: ReliabilityPoint[] = [];
+  let error = 0;
+  for (const [i, b] of acc.entries()) {
+    if (b.total === 0) continue;
+    const predicted = b.predicted / b.total;
+    const observed = b.correct / b.total;
+    points.push({
+      bucket: `${(i / buckets).toFixed(1)}–${((i + 1) / buckets).toFixed(1)}`,
+      predicted,
+      observed,
+      samples: b.total,
+    });
+    error += (b.total / heldOut.length) * Math.abs(predicted - observed);
+  }
+
+  return { points, expectedCalibrationError: error };
+}

package/src/fuse/defaultCalibration.ts

@@ -0,0 +1,71 @@
+/**
+ * defaultCalibration.ts — the shipped calibration model, checked rather than cast.
+ *
+ * ─────────────────────────────────────────────────────────────────────────
+ * THE CAST THIS REPLACES
+ *
+ * `@discretion/data` types the fitted model's `perType` as
+ * `Record<string, CalibrationCurveData>`, because it is generated from an eval
+ * run and the generator has no access to the `EntityType` union. Core's
+ * `CalibrationModel` wants `Partial<Record<EntityType, CalibrationCurve>>`.
+ *
+ * Every consumer so far bridged that with `as unknown as CalibrationModel`.
+ * The cast is not merely ugly: it asserts the keys are valid `EntityType`
+ * members without checking, so a model fitted before a type was renamed would
+ * carry a key that matches nothing. `calibrate()` would then fall through to
+ * the pooled curve for that type, forever, with no error and no warning — the
+ * numbers would just be quietly worse than the published ones.
+ *
+ * Converting with a check turns that into something observable.
+ */
+
+import { ALL_ENTITY_TYPES } from '../entityTypes.js';
+import type { EntityType } from '../types.js';
+import type { CalibrationCurve, CalibrationModel } from './calibrate.js';
+
+/**
+ * The generated shape, restated so this module does not depend on data's types.
+ *
+ * Exported because it is `toCalibrationModel`'s parameter type: a consumer who
+ * fits their own model needs to name what the function accepts. It was local
+ * when this file was written, and `public-api.test.ts` caught it on its first
+ * run - which is the whole reason that test exists.
+ */
+export interface GeneratedCalibrationModel {
+  readonly perType: Readonly<Record<string, CalibrationCurve>>;
+  readonly pooled: CalibrationCurve;
+  readonly fittedOn: string;
+}
+
+export interface CalibrationConversion {
+  readonly model: CalibrationModel;
+  /**
+   * Keys in the generated model that are not `EntityType` members.
+   *
+   * Non-empty means the committed model and the union have drifted apart. The
+   * conversion still succeeds — dropping an unusable curve is better than
+   * refusing to calibrate anything — but the caller can now see it, and
+   * `calibration-model.test.ts` asserts it is empty.
+   */
+  readonly unknownTypes: readonly string[];
+}
+
+/**
+ * Convert a generated calibration model to core's typed one, reporting any key
+ * that is not a known entity type instead of assuming there are none.
+ */
+export function toCalibrationModel(generated: GeneratedCalibrationModel): CalibrationConversion {
+  const known = new Set<string>(ALL_ENTITY_TYPES);
+  const perType: Partial<Record<EntityType, CalibrationCurve>> = {};
+  const unknownTypes: string[] = [];
+
+  for (const [key, curve] of Object.entries(generated.perType)) {
+    if (known.has(key)) perType[key as EntityType] = curve;
+    else unknownTypes.push(key);
+  }
+
+  return {
+    model: { perType, pooled: generated.pooled, fittedOn: generated.fittedOn },
+    unknownTypes,
+  };
+}

package/src/fuse/explain.ts

@@ -0,0 +1,114 @@
+/**
+ * Stage 4 — EXPLANATIONS.
+ *
+ * SPEC.md: "Every emitted entity carries an EXPLANATION: which stages fired,
+ * which triggers were found, which validator passed. This drives the review UI
+ * and makes failures diagnosable."
+ *
+ * The last clause is the one that shapes this module. An explanation is not
+ * decoration for the UI — it is how a wrong detection gets diagnosed without
+ * re-running the pipeline under a debugger. So it records what actually
+ * happened rather than a summary of it: which stages contributed, which named
+ * Stage 3 signals fired and with what sign, and what the profile decided.
+ *
+ * NO PLAINTEXT IN EXPLANATIONS, which is why triggers are lexicon terms and
+ * signal names rather than document text. Explanations flow into eval reports,
+ * the review panel, and the popup; a value that reached any of those would
+ * breach SPEC's third non-negotiable. The one exception is the entity's own
+ * text, which the caller already holds and which `DetectedEntity` carries in
+ * its own field.
+ */
+
+import type { DetectedEntity, DetectionStage, EntityExplanation, EntityType } from '../types.js';
+import type { ContextScoredCandidate } from '../context/types.js';
+import type { ProfileDecision } from './profiles.js';
+
+/** What Stage 4 knows about one candidate as it becomes an entity. */
+export interface FusionInput {
+  readonly scored: ContextScoredCandidate;
+  /** Confidence after calibration; comparable across types. */
+  readonly calibratedConfidence: number;
+  /** Whether the entity's span was widened by overlap resolution. */
+  readonly absorbedOverlap?: boolean;
+  /** Types this candidate won an overlap against. */
+  readonly wonAgainst?: readonly EntityType[];
+}
+
+/**
+ * Build the explanation for one entity.
+ *
+ * `triggers` carries the Stage 3 signal names rather than only lexicon
+ * matches, because "why was this reported" is answered as much by
+ * `structure:key-names-API_KEY` as by a trigger word — and a reviewer chasing
+ * a false positive needs the negative signals too, which is why suppression
+ * and penalty signals appear with their sign rather than being filtered out.
+ */
+export function explain(input: FusionInput): EntityExplanation {
+  const { scored } = input;
+  const stages: DetectionStage[] = [scored.candidate.stage];
+
+  // Stage 3 fired if it contributed anything at all.
+  if (scored.contributions.length > 0) stages.push('stage3-context');
+  stages.push('stage4-fusion');
+
+  const triggers = scored.contributions.map((c) => {
+    const sign = c.delta >= 0 ? '+' : '';
+    const detail = c.detail === undefined ? '' : ` (${c.detail})`;
+    return `${c.signal}${detail} ${sign}${c.delta.toFixed(2)}`;
+  });
+
+  if (input.absorbedOverlap === true) {
+    triggers.push('stage4:absorbed-overlapping-span +0.00');
+  }
+  for (const type of input.wonAgainst ?? []) {
+    triggers.push(`stage4:more-specific-than-${type} +0.00`);
+  }
+
+  return {
+    stages,
+    triggers,
+    ...(scored.candidate.validatorPassed !== undefined
+      ? { validatorPassed: scored.candidate.validatorPassed }
+      : {}),
+  };
+}
+
+/** Assemble the entity a consumer actually receives. */
+export function toDetectedEntity(input: FusionInput): DetectedEntity {
+  const { candidate } = input.scored;
+  return {
+    text: candidate.text,
+    type: candidate.type,
+    start: candidate.start,
+    end: candidate.end,
+    calibratedConfidence: input.calibratedConfidence,
+    explanation: explain(input),
+  };
+}
+
+/**
+ * A one-line reason a candidate was NOT reported.
+ *
+ * SPEC's explanation requirement covers emitted entities, but the review UI
+ * has to answer "why did you miss this?" too, and a suppression with no
+ * recorded reason is exactly the failure D18 was written about. Kept in the
+ * same module so the two stay consistent.
+ */
+export function explainOmission(
+  scored: ContextScoredCandidate,
+  decision?: ProfileDecision,
+): string {
+  if (scored.suppressed) {
+    return `suppressed by ${scored.suppressionReason ?? 'an unnamed rule'}`;
+  }
+  switch (decision?.reason) {
+    case 'allowlist':
+      return 'on the user allowlist';
+    case 'type-out-of-profile':
+      return 'entity type is outside the active sensitivity profile';
+    case 'below-threshold':
+      return 'calibrated confidence below the profile threshold';
+    default:
+      return 'not reported';
+  }
+}

package/src/fuse/profiles.ts

@@ -0,0 +1,187 @@
+/**
+ * Stage 4 — SENSITIVITY PROFILES, allowlist and denylist.
+ *
+ * SPEC.md:
+ *   "Apply the active SENSITIVITY PROFILE, which sets thresholds:
+ *      Minimal — secrets and financial identifiers only.
+ *      Balanced — default. Secrets, financial, national IDs, contact details,
+ *        person names.
+ *      Strict — adds health data, addresses, dates of birth, organizations,
+ *        low-confidence candidates.
+ *      Custom — per-entity-type threshold control."
+ *   "Apply user allowlist (never mask …) and denylist (always mask …).
+ *    Denylist beats everything."
+ *
+ * A profile does two things at once and it is worth separating them: it says
+ * WHICH TYPES are in scope, and it says HOW SURE the pipeline must be before
+ * reporting one. The second only became meaningful with calibration — before
+ * a confidence meant something comparable across types, a threshold was an
+ * arbitrary cut. That is why profiles land in the same milestone as the
+ * calibration curve rather than earlier.
+ *
+ * Thresholds are stated as CALIBRATED probabilities, so "0.5" reads as "report
+ * this when it is more likely right than wrong" rather than as a detector-local
+ * number whose meaning varies by type.
+ */
+
+import type { EntityType } from '../types.js';
+
+export type ProfileName = 'minimal' | 'balanced' | 'strict' | 'custom';
+
+/** Broad families, used to state the profiles in SPEC's own vocabulary. */
+const SECRETS: readonly EntityType[] = [
+  'API_KEY',
+  'PRIVATE_KEY',
+  'JWT',
+  'GENERIC_SECRET',
+  'CONNECTION_STRING',
+  'URL_WITH_CREDENTIALS',
+];
+
+const FINANCIAL: readonly EntityType[] = [
+  'CREDIT_CARD',
+  'IBAN',
+  'SWIFT_BIC',
+  'US_ROUTING_NUMBER',
+  'UK_SORT_CODE',
+  'CA_TRANSIT_NUMBER',
+  'AU_BSB',
+  'IN_IFSC',
+  'BR_AGENCIA',
+  'CRYPTO_WALLET',
+  'VAT_NUMBER',
+];
+
+const GOVERNMENT_ID: readonly EntityType[] = [
+  'NATIONAL_ID',
+  'TAX_ID',
+  'PASSPORT_MRZ',
+  'DRIVERS_LICENSE',
+  'VIN',
+  'US_NPI',
+];
+
+const CONTACT: readonly EntityType[] = ['EMAIL', 'PHONE', 'IP_ADDRESS', 'MAC_ADDRESS'];
+
+const LOCATION_TYPES: readonly EntityType[] = ['STREET_ADDRESS', 'POSTAL_CODE', 'COORDINATES', 'LOCATION'];
+
+const HEALTH: readonly EntityType[] = ['HEALTH_DATA'];
+
+const PEOPLE: readonly EntityType[] = ['PERSON'];
+const ORGS: readonly EntityType[] = ['ORG'];
+const DATES: readonly EntityType[] = ['DATE_OF_BIRTH'];
+
+export interface SensitivityProfile {
+  readonly name: ProfileName;
+  /** Types the profile reports at all. */
+  readonly types: ReadonlySet<EntityType>;
+  /**
+   * Minimum CALIBRATED confidence to report, per type, with a default.
+   * Meaningful only because Stage 4 calibrated the scale (D23).
+   */
+  readonly threshold: (type: EntityType) => number;
+}
+
+function profile(
+  name: ProfileName,
+  types: readonly (readonly EntityType[])[],
+  defaultThreshold: number,
+  overrides: Partial<Record<EntityType, number>> = {},
+): SensitivityProfile {
+  return {
+    name,
+    types: new Set(types.flat()),
+    threshold: (type) => overrides[type] ?? defaultThreshold,
+  };
+}
+
+/**
+ * The three named profiles.
+ *
+ * Thresholds differ by INTENT, not by tuning. Minimal exists for a developer
+ * who wants surgical protection and would rather miss a weak signal than see
+ * a spurious one, so it demands high confidence. Strict exists for someone who
+ * would rather over-mask, so it deliberately admits low-confidence candidates
+ * — SPEC names "low-confidence candidates" as part of what Strict adds.
+ */
+export const PROFILES: Readonly<Record<Exclude<ProfileName, 'custom'>, SensitivityProfile>> = {
+  minimal: profile('minimal', [SECRETS, FINANCIAL], 0.7),
+  balanced: profile('balanced', [SECRETS, FINANCIAL, GOVERNMENT_ID, CONTACT, PEOPLE], 0.5, {
+    // A leaked credential is unrecoverable, so it is reported on weaker
+    // evidence than a name, which the user can dismiss at no cost.
+    API_KEY: 0.35,
+    PRIVATE_KEY: 0.35,
+    CONNECTION_STRING: 0.35,
+  }),
+  strict: profile(
+    'strict',
+    [SECRETS, FINANCIAL, GOVERNMENT_ID, CONTACT, PEOPLE, HEALTH, LOCATION_TYPES, ORGS, DATES],
+    0.25,
+    { API_KEY: 0.2, PRIVATE_KEY: 0.2, CONNECTION_STRING: 0.2 },
+  ),
+};
+
+/** A user's own additions to whatever profile is active. */
+export interface UserLists {
+  /**
+   * Never mask these. Matched case- and whitespace-insensitively on the
+   * entity's text — a user's own employer name, say.
+   */
+  readonly allow?: readonly string[];
+  /** Always mask these, whatever the profile or confidence says. */
+  readonly deny?: readonly string[];
+}
+
+function fold(value: string): string {
+  return value.trim().toLowerCase().replace(/\s+/g, ' ');
+}
+
+export interface ProfileDecision {
+  readonly report: boolean;
+  /** Why, in one stable token, for the explanation and for tests. */
+  readonly reason: 'denylist' | 'allowlist' | 'type-out-of-profile' | 'below-threshold' | 'in-profile';
+}
+
+/**
+ * Decide whether one entity is reported under a profile.
+ *
+ * ORDER IS THE WHOLE CONTRACT, and it follows SPEC exactly: "Denylist beats
+ * everything." A denylisted value is reported even when its type is out of
+ * profile and even when confidence is far below threshold — it is the one
+ * override a user can state absolutely. The allowlist is checked next, before
+ * type and threshold, so that suppressing a value the user has vouched for
+ * does not depend on which profile happens to be active.
+ */
+export function decide(
+  entity: { readonly type: EntityType; readonly text: string; readonly calibratedConfidence: number },
+  profileIn: SensitivityProfile,
+  lists: UserLists = {},
+): ProfileDecision {
+  const text = fold(entity.text);
+
+  if ((lists.deny ?? []).some((d) => fold(d) === text)) {
+    return { report: true, reason: 'denylist' };
+  }
+  if ((lists.allow ?? []).some((a) => fold(a) === text)) {
+    return { report: false, reason: 'allowlist' };
+  }
+  if (!profileIn.types.has(entity.type)) {
+    return { report: false, reason: 'type-out-of-profile' };
+  }
+  if (entity.calibratedConfidence < profileIn.threshold(entity.type)) {
+    return { report: false, reason: 'below-threshold' };
+  }
+  return { report: true, reason: 'in-profile' };
+}
+
+/** Build a custom profile from explicit per-type thresholds. */
+export function customProfile(
+  thresholds: Partial<Record<EntityType, number>>,
+  defaultThreshold = 0.5,
+): SensitivityProfile {
+  return {
+    name: 'custom',
+    types: new Set(Object.keys(thresholds) as EntityType[]),
+    threshold: (type) => thresholds[type] ?? defaultThreshold,
+  };
+}