@discretion/core 0.1.0

package/src/ner/engine.ts

@@ -0,0 +1,141 @@
+/**
+ * engine.ts — Stage 2 orchestration: window → classify → align → decode,
+ * under a HARD deadline with fail-closed semantics.
+ *
+ * The deadline covers the whole recognize() call (all windows), not one
+ * inference: SPEC.md's budget is per user action. On exceed the engine
+ * throws DetectionTimeoutError — the same fail-closed contract the Stage 1
+ * runner enforces; callers treat it as "block the send", never as "send
+ * what we found so far", because a partial scan is indistinguishable from
+ * a clean scan on exactly the text that was never reached.
+ *
+ * warmup() runs one tiny inference so the first real interaction does not
+ * absorb model-initialization latency (SPEC: warm at init, not first use).
+ */
+
+import { DetectionTimeoutError } from '../detect/runner.js';
+import { alignPieces } from './align.js';
+import { chunkText } from './chunk.js';
+import { decodeEntities } from './merge.js';
+import { attachGazetteerHits } from './stage2b.js';
+import type { ChunkCache } from './chunkCache.js';
+import type { NerRecognizer, NerSpan, TokenClassifier } from './types.js';
+
+export interface NerEngineOptions {
+  /** Hard deadline for one recognize() call. Default 2000 ms. */
+  readonly timeBudgetMs?: number;
+  /** Window overlap in characters. Default 96. */
+  readonly overlapChars?: number;
+  /**
+   * Stage 2b gazetteer corroboration. Default true.
+   *
+   * Lives here rather than in Stage 3's options because the lookup lives here:
+   * SPEC places the gazetteers in Stage 2, checked in parallel with the model.
+   */
+  readonly useGazetteers?: boolean;
+}
+
+const DEFAULT_BUDGET_MS = 2000;
+const DEFAULT_OVERLAP = 96;
+
+export class NerEngine implements NerRecognizer {
+  private readonly classifier: TokenClassifier;
+  private readonly budgetMs: number;
+  private readonly overlap: number;
+  private readonly useGazetteers: boolean;
+  private warmedUp = false;
+
+  constructor(classifier: TokenClassifier, options: NerEngineOptions = {}) {
+    this.classifier = classifier;
+    this.budgetMs = options.timeBudgetMs ?? DEFAULT_BUDGET_MS;
+    this.overlap = options.overlapChars ?? DEFAULT_OVERLAP;
+    this.useGazetteers = options.useGazetteers ?? true;
+  }
+
+  get id(): string {
+    return this.classifier.id;
+  }
+
+  /** One tiny inference so model init cost is paid before first real use. */
+  async warmup(): Promise<void> {
+    if (this.warmedUp) return;
+    await this.classifier.classify('warmup.');
+    this.warmedUp = true;
+  }
+
+  /**
+   * Recognize entities in (normalized) `text`. Spans are in the text's own
+   * coordinates; runStage2 maps them back through the Stage 0 offset map.
+   */
+  /**
+   * Recognize entities in (normalized) `text`.
+   *
+   * `cache` is passed IN rather than owned here because its keys are the
+   * user's text: it must live and die with one session, and this engine is
+   * shared across every session in the process. See chunkCache.ts.
+   */
+  async recognize(text: string, cache?: ChunkCache): Promise<NerSpan[]> {
+    if (text.length === 0) return [];
+    const started = Date.now();
+    const deadline = started + this.budgetMs;
+
+    const chunks = chunkText(text, this.classifier.maxInputChars, this.overlap);
+    const spans: NerSpan[] = [];
+
+    for (const chunk of chunks) {
+      // A cache hit costs no inference and therefore no deadline: the budget
+      // exists to bound model time, and there is none to bound. Checking the
+      // deadline before the lookup would let a document time out on work it
+      // was never going to do.
+      let local = cache?.get(chunk.text);
+
+      if (local === undefined) {
+        const remaining = deadline - Date.now();
+        if (remaining <= 0) {
+          throw new DetectionTimeoutError(this.budgetMs, Date.now() - started);
+        }
+        const predictions = await withDeadline(
+          this.classifier.classify(chunk.text),
+          remaining,
+          () => new DetectionTimeoutError(this.budgetMs, Date.now() - started),
+        );
+        const aligned = alignPieces(chunk.text, predictions.map((p) => p.piece));
+        // Stored in CHUNK-LOCAL coordinates, before the offset shift below:
+        // the same chunk text can appear at different offsets in different
+        // documents, and a cache keyed on the text must not remember where it
+        // happened to sit the first time.
+        local = decodeEntities(chunk.text, predictions, aligned);
+        cache?.set(chunk.text, local);
+      }
+
+      for (const span of local) {
+        const mid = chunk.offset + (span.start + span.end) / 2;
+        // The owning-core rule from chunk.ts: emit each entity exactly once.
+        if (mid >= chunk.coreStart && mid < chunk.coreEnd) {
+          spans.push({ ...span, start: chunk.offset + span.start, end: chunk.offset + span.end });
+        }
+      }
+    }
+
+    const ordered = spans.sort((a, b) => a.start - b.start);
+    // Stage 2b, on the same side of any process boundary as the model: SPEC
+    // has the gazetteers "checked in parallel with the model", and they can
+    // only corroborate types the model produces.
+    return this.useGazetteers ? attachGazetteerHits(ordered) : ordered;
+  }
+}
+
+/** Race a promise against a deadline; the loser's result is discarded. */
+async function withDeadline<T>(work: Promise<T>, ms: number, onTimeout: () => Error): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      work,
+      new Promise<never>((_, reject) => {
+        timer = setTimeout(() => reject(onTimeout()), ms);
+      }),
+    ]);
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+  }
+}

package/src/ner/merge.ts

@@ -0,0 +1,123 @@
+/**
+ * merge.ts — decode per-token BIO labels into entity spans.
+ *
+ * Handles BOTH tagging conventions the candidate models use:
+ *  • IOB2 (mBERT/distilmBERT NER-HRL): entities start with B-X.
+ *  • IOB1 (XLM-R CoNLL fine-tunes): entities may start with I-X after O
+ *    or at sequence start; B-X appears only to split adjacent same-type
+ *    entities.
+ * Decoding rule that satisfies both: a token starts a NEW entity when its
+ * label is B-X, or when it is I-X and the previous token was not part of
+ * an X entity. Same-type I-X continues; any other label ends the entity.
+ *
+ * Unlocated pieces ([UNK] and friends) inside an entity are BRIDGED: the
+ * entity's span runs from its first located piece to its last located
+ * piece. An entity with no located piece at all is dropped — emitting a
+ * guessed span would violate the exact-offsets contract.
+ *
+ * The entity score is the MINIMUM token score — one uncertain token makes
+ * the whole span uncertain. Scores are uncalibrated model softmax until
+ * Stage 4 (M8) calibrates; the eval buckets measure what they are worth.
+ */
+
+import type { AlignedPiece } from './align.js';
+import type { NerEntityType, NerSpan, TokenPrediction } from './types.js';
+
+/** Model label suffix → Discretion entity type. MISC and friends drop. */
+const DEFAULT_LABEL_TYPES: Readonly<Record<string, NerEntityType>> = {
+  PER: 'PERSON',
+  PERSON: 'PERSON',
+  ORG: 'ORG',
+  LOC: 'LOCATION',
+  LOCATION: 'LOCATION',
+  GPE: 'LOCATION',
+};
+
+interface Open {
+  type: NerEntityType;
+  start: number;
+  end: number;
+  score: number;
+  located: boolean;
+}
+
+function labelParts(label: string): { prefix: 'B' | 'I' | 'O'; type?: NerEntityType } {
+  if (label === 'O' || label === '') return { prefix: 'O' };
+  const dash = label.indexOf('-');
+  const prefix = label.slice(0, dash === -1 ? undefined : dash);
+  const suffix = dash === -1 ? label : label.slice(dash + 1);
+  const type = DEFAULT_LABEL_TYPES[suffix.toUpperCase()];
+  if ((prefix === 'B' || prefix === 'I') && type !== undefined) {
+    return { prefix, type };
+  }
+  return { prefix: 'O' };
+}
+
+/** Decode predictions + aligned spans into merged entity spans. */
+export function decodeEntities(
+  text: string,
+  predictions: readonly TokenPrediction[],
+  aligned: readonly AlignedPiece[],
+): NerSpan[] {
+  const spans: NerSpan[] = [];
+  let open: Open | null = null;
+
+  const close = (): void => {
+    if (open !== null && open.located && open.end > open.start) {
+      spans.push({
+        type: open.type,
+        start: open.start,
+        end: open.end,
+        text: text.slice(open.start, open.end),
+        score: open.score,
+      });
+    }
+    open = null;
+  };
+
+  for (let i = 0; i < predictions.length; i++) {
+    const pred = predictions[i]!;
+    const span = aligned[i];
+    if (span === undefined) break; // defensive: prediction/alignment mismatch
+    const { prefix, type } = labelParts(pred.label);
+
+    if (prefix === 'O' || type === undefined) {
+      close();
+      continue;
+    }
+
+    const continues = open !== null && open.type === type && prefix === 'I';
+    if (continues && open !== null) {
+      if (span.located) {
+        if (!open.located) open.start = span.start; // first located piece
+        open.end = span.end;
+        open.located = true;
+      }
+      open.score = Math.min(open.score, pred.score);
+    } else {
+      close();
+      open = {
+        type,
+        start: span.start,
+        end: span.end,
+        score: pred.score,
+        located: span.located,
+      };
+    }
+  }
+  close();
+
+  return spans.map(trimSpan(text)).filter((s): s is NerSpan => s !== null);
+}
+
+/** Trim surrounding whitespace the model occasionally absorbs into a span. */
+function trimSpan(text: string): (s: NerSpan) => NerSpan | null {
+  return (s) => {
+    let { start, end } = s;
+    while (start < end && /\s/.test(text[start]!)) start += 1;
+    while (end > start && /\s/.test(text[end - 1]!)) end -= 1;
+    if (end <= start) return null;
+    if (start === s.start && end === s.end) return s;
+    return { ...s, start, end, text: text.slice(start, end) };
+  };
+}

package/src/ner/runStage2.ts

@@ -0,0 +1,51 @@
+/**
+ * runStage2.ts — produce Stage 2 candidates in the pipeline's shared shape.
+ *
+ * Mirrors the Stage 1 runner's contract: detection runs on NORMALIZED text
+ * and every candidate carries its exact ORIGINAL span, resolved through the
+ * Stage 0 offset map with mapNormalizedSpan (never raw map indexing — see
+ * the M1 offset-map contract).
+ *
+ * rawConfidence is the model's softmax score, unchanged and explicitly
+ * uncalibrated — the eval measures what those scores are worth per bucket,
+ * and Stage 4 (M8) calibrates. Inventing a mapping here would launder an
+ * unmeasured number into a measured-looking one.
+ */
+
+import type { NormalizationResult } from '../types.js';
+import { mapNormalizedSpan } from '../offsetMap.js';
+import type { NerRecognizer, Stage2Candidate } from './types.js';
+
+/** Collapse interior whitespace so trivial layout variants share a key. */
+function canonicalName(text: string): string {
+  return text.replace(/\s+/g, ' ').trim();
+}
+
+export async function runStage2(
+  normalization: NormalizationResult,
+  engine: NerRecognizer,
+): Promise<Stage2Candidate[]> {
+  const { normalizedText, offsetMap } = normalization;
+  const spans = await engine.recognize(normalizedText);
+
+  return spans.map((span) => {
+    const original = mapNormalizedSpan(offsetMap, span.start, span.end);
+    return {
+      text: span.text,
+      type: span.type,
+      start: span.start,
+      end: span.end,
+      originalStart: original.start,
+      originalEnd: original.end,
+      rawConfidence: span.score,
+      stage: 'stage2-ner' as const,
+      detectorId: `ner:${engine.id}`,
+      sensitive: true,
+      canonical: canonicalName(span.text),
+      metadata: { model: engine.id },
+      // Carried rather than re-looked-up: the hit was computed where the
+      // gazetteers live, which may be a different process from this one.
+      ...(span.gazetteer === undefined ? {} : { gazetteer: span.gazetteer }),
+    };
+  });
+}

package/src/ner/stage2b.ts

@@ -0,0 +1,73 @@
+/**
+ * Stage 2b — gazetteer corroboration, attached to the spans it corroborates.
+ *
+ * ─────────────────────────────────────────────────────────────────────────
+ * WHY THIS FILE EXISTS, WHICH IS A CORRECTION RATHER THAN A NEW FEATURE
+ *
+ * SPEC.md places the gazetteers in STAGE 2: "Bundled compressed lookup sets,
+ * checked in parallel with the model." The lookup had drifted into Stage 3's
+ * scorer, where it ran as one context signal among many.
+ *
+ * That placement was invisible until the extension was built, and then it was
+ * expensive in a way nothing else made visible. Stage 3 runs in the CONTENT
+ * SCRIPT. Its scorer statically imported the gazetteer module, so 3.4 MB of
+ * base64 Bloom filters were linked into a bundle that is parsed on every page
+ * load of all three sites — to serve PERSON, ORG and LOCATION candidates,
+ * which only Stage 2 produces, and which cannot exist at all in a build with
+ * no model attached.
+ *
+ * Moving the lookup here is not a size optimisation. It restores the stage
+ * boundary SPEC drew, and the size follows from that: the gazetteers now live
+ * on the same side of the process boundary as the model, because they answer
+ * questions only the model can ask.
+ *
+ * BEHAVIOUR IS UNCHANGED, and that is checkable rather than asserted: no
+ * Stage 1 detector declares PERSON, ORG or LOCATION (31 types, verified by
+ * enumeration), so no Stage 1 candidate could ever have reached the lookup in
+ * Stage 3. The contribution Stage 3 emits is byte-identical; it is now built
+ * from a hit computed here instead of one computed there.
+ * ─────────────────────────────────────────────────────────────────────────
+ */
+
+import { lookupGazetteer } from '../gazetteer/index.js';
+import type { GazetteerType } from '../gazetteer/index.js';
+import type { NerEntityType, NerSpan } from './types.js';
+
+/**
+ * The model's label set and the gazetteer's are the same three types.
+ *
+ * Asserted at COMPILE time rather than guarded at run time. The obvious
+ * implementation calls `isGazetteerType(span.type)` before each lookup - but
+ * `NerSpan.type` is already exactly those three, so that branch can never be
+ * false. A check that cannot fail reads as a safety property and is not one;
+ * this fails the BUILD if either union ever gains a member the other lacks,
+ * which is the moment the guard would have started mattering.
+ */
+type MissingFromGazetteer = Exclude<NerEntityType, GazetteerType>;
+type MissingFromNer = Exclude<GazetteerType, NerEntityType>;
+// Both must be `never`. A non-never member here is a type the other side does
+// not know about, and it fails the build at exactly the point where the
+// removed runtime guard would have started doing work.
+export type Stage2bTypesAgree = [MissingFromGazetteer, MissingFromNer] extends [never, never]
+  ? true
+  : never;
+const _typesAgree: Stage2bTypesAgree = true;
+void _typesAgree;
+
+/**
+ * Attach gazetteer corroboration to each span.
+ *
+ * A miss attaches nothing rather than attaching an absent hit: "not in the
+ * gazetteer" is not evidence against a name — the sets are large but nowhere
+ * near complete — and a field distinguishing "looked and found nothing" from
+ * "did not look" would invite a caller to treat the first as a negative
+ * signal. `useGazetteers: false` and a genuine miss are meant to be
+ * indistinguishable downstream, because they justify the same conclusion:
+ * nothing was corroborated.
+ */
+export function attachGazetteerHits(spans: readonly NerSpan[]): NerSpan[] {
+  return spans.map((span) => {
+    const hit = lookupGazetteer(span.text, span.type);
+    return hit === undefined ? span : { ...span, gazetteer: hit };
+  });
+}

package/src/ner/transformersClassifier.ts

@@ -0,0 +1,79 @@
+/**
+ * transformersClassifier.ts — the Transformers.js/ONNX-backed
+ * TokenClassifier. Runs identically in Node (onnxruntime-node) and in a
+ * browser Web Worker (onnxruntime-web/WASM); hosting it in a worker is the
+ * consumer's concern (extension, M9), which keeps this module free of any
+ * environment-specific API.
+ *
+ * ZERO NETWORK BY DEFAULT: remote model fetching is opt-in
+ * (`allowRemoteModels`), used only by build-time tooling (today the eval
+ * benchmark and CLI; the extension's bundling step at M9). Production
+ * consumers load from the bundled cache and never touch the network — the
+ * SPEC non-negotiable.
+ *
+ * This module is exported through the dedicated `./ner-transformers` entry
+ * point, NOT the package root, so consumers who never run Stage 2 never
+ * load the ONNX runtime.
+ */
+
+import { env, pipeline } from '@huggingface/transformers';
+import type { TokenClassifier, TokenPrediction } from './types.js';
+
+export interface TransformersClassifierOptions {
+  /** HF repo id, e.g. 'Xenova/distilbert-base-multilingual-cased-ner-hrl'. */
+  readonly model: string;
+  /** Weight precision: 'fp32' | 'fp16' | 'q8' | 'int8' | 'uint8' | …. */
+  readonly dtype?: string;
+  /** Where model files live (bundle dir in production, cache in tooling). */
+  readonly cacheDir?: string;
+  /** Allow downloading missing models. Build-time tooling ONLY. */
+  readonly allowRemoteModels?: boolean;
+  /**
+   * Pinned repo revision (commit hash). HF commits are content-addressed,
+   * so a pinned revision pins the exact model bytes — the integrity story
+   * for build-time bundling. Default 'main' is acceptable only in
+   * exploratory tooling.
+   */
+  readonly revision?: string;
+  /** Short id for candidate metadata; defaults to model@dtype. */
+  readonly id?: string;
+  /**
+   * Per-window character budget. Default 400: the models' 512-token limit
+   * divided by the worst-case one-token-per-character ratio (CJK), with
+   * headroom for specials and subword expansion of unusual codepoints.
+   */
+  readonly maxInputChars?: number;
+}
+
+interface RawTokenOutput {
+  readonly entity: string;
+  readonly score: number;
+  readonly word: string;
+}
+
+const DEFAULT_MAX_INPUT_CHARS = 400;
+
+export async function createTransformersClassifier(
+  options: TransformersClassifierOptions,
+): Promise<TokenClassifier> {
+  if (options.cacheDir !== undefined) env.cacheDir = options.cacheDir;
+  env.allowRemoteModels = options.allowRemoteModels ?? false;
+
+  const dtype = options.dtype ?? 'q8';
+  const nerPipeline = await pipeline('token-classification', options.model, {
+    dtype: dtype as never, // the runtime validates; its literal union lags its own docs
+    ...(options.revision !== undefined ? { revision: options.revision } : {}),
+  });
+
+  const id = options.id ?? `${options.model.split('/').pop() ?? options.model}@${dtype}`;
+  const maxInputChars = options.maxInputChars ?? DEFAULT_MAX_INPUT_CHARS;
+
+  return {
+    id,
+    maxInputChars,
+    async classify(text: string): Promise<readonly TokenPrediction[]> {
+      const raw = (await nerPipeline(text, { ignore_labels: [] })) as readonly RawTokenOutput[];
+      return raw.map((t) => ({ label: t.entity, score: t.score, piece: t.word }));
+    },
+  };
+}

package/src/ner/types.ts

@@ -0,0 +1,99 @@
+/**
+ * Stage 2 (multilingual NER) — shared types.
+ *
+ * The model runtime is INJECTED behind `TokenClassifier`, so the pure parts
+ * of Stage 2 (piece→character alignment, BIO decoding, chunking, timeout
+ * discipline, candidate shaping) are testable without a model and the
+ * engine stays environment-agnostic. The Transformers.js-backed classifier
+ * lives in `transformersClassifier.ts`, exported through its own package
+ * entry point so that consumers who never run NER never load the runtime.
+ */
+
+import type { Stage1Candidate } from '../detect/types.js';
+import type { GazetteerHit } from '../gazetteer/index.js';
+
+/** One model token prediction, in input order, special tokens excluded. */
+export interface TokenPrediction {
+  /** Raw model label, e.g. 'B-PER', 'I-ORG', 'O'. */
+  readonly label: string;
+  /** Model softmax score for that label. */
+  readonly score: number;
+  /**
+   * The token's surface piece as the runtime reports it: WordPiece
+   * continuations keep their '##' prefix; SentencePiece markers ('▁') may
+   * or may not be present — the aligner handles both.
+   */
+  readonly piece: string;
+}
+
+/** The injected model runtime. */
+export interface TokenClassifier {
+  /** Identifies the model in candidate metadata, e.g. 'distilmbert-ner-hrl@q8'. */
+  readonly id: string;
+  /**
+   * Safe per-call input size in CHARACTERS. Text longer than this must be
+   * windowed by the caller: transformer runtimes silently truncate past
+   * their token limit, and silent truncation would be silent fail-open.
+   * The floor of one token per character (CJK) makes chars the safe unit.
+   */
+  readonly maxInputChars: number;
+  /** Classify one window of text. Pieces arrive in input order. */
+  classify(text: string): Promise<readonly TokenPrediction[]>;
+}
+
+export type NerEntityType = 'PERSON' | 'ORG' | 'LOCATION';
+
+/** A merged entity span in NORMALIZED-text coordinates. */
+export interface NerSpan {
+  readonly type: NerEntityType;
+  readonly start: number;
+  readonly end: number;
+  readonly text: string;
+  /** Minimum token score across the entity — the conservative aggregate. */
+  readonly score: number;
+  /**
+   * Stage 2b corroboration, when the gazetteer knows this name.
+   *
+   * Attached HERE rather than looked up in Stage 3 because SPEC places the
+   * gazetteers in Stage 2 — "checked in parallel with the model" — and
+   * because the sets are only meaningful for the types the model produces.
+   * See ner/stage2b.ts for what that placement was costing.
+   *
+   * Absent on a miss. "Not in the gazetteer" is not evidence against a name.
+   */
+  readonly gazetteer?: GazetteerHit;
+}
+
+/**
+ * What Stage 2 needs from whatever performs recognition.
+ *
+ * An interface rather than the concrete `NerEngine` because the engine may not
+ * be in the same PROCESS as the pipeline. The extension runs the model in an
+ * offscreen document - a content script cannot compile WebAssembly under the
+ * host page's policy - so its Stage 2 is a proxy that forwards `recognize`
+ * across a message port and returns the spans that come back.
+ *
+ * Declared here, next to the span it returns, so that `runStage2` and
+ * `detect` can name it without importing `engine.ts`. That matters: engine.ts
+ * pulls in Stage 2b and therefore the gazetteers, and a value import of it
+ * from the pipeline would link 3.4 MB of Bloom filters into every bundle that
+ * runs detection, including the one that has no model at all.
+ */
+export interface NerRecognizer {
+  /** Model identity, recorded on every candidate this produces. */
+  readonly id: string;
+  /** Pay initialization cost before first real use. */
+  warmup(): Promise<void>;
+  /** Spans in the given text's own coordinates. */
+  recognize(text: string): Promise<NerSpan[]>;
+}
+
+/**
+ * Stage 2 candidates share Stage 1's shape (SPEC: one candidate contract
+ * for the whole pipeline), differing only in the stage tag.
+ */
+export interface Stage2Candidate extends Omit<Stage1Candidate, 'stage'> {
+  readonly stage: 'stage2-ner';
+  /** Stage 2b corroboration, carried through to Stage 3's scorer. */
+  readonly gazetteer?: GazetteerHit;
+}

package/src/normalization.ts

@@ -0,0 +1,119 @@
+/**
+ * Stage 0 normalization: the composed transform pipeline.
+ *
+ * Order (each step optional via NormalizationOptions, all on by default):
+ *
+ *   1. strip invisibles      — zero-width & bidi controls are pure noise and
+ *                              interfere with everything downstream
+ *   2. NFKC (per grapheme)   — compatibility folding with an exact map
+ *   3. selective homoglyph   — fold ONLY script-anomalous characters
+ *      folding                 (see homoglyphFold.ts for the full rationale)
+ *   3b. NFKC again, only when folding changed something: a folded character
+ *       must compose with a following combining mark exactly as a native one
+ *       would (e.g. Cyrillic а + U+0301 → folded "a" + U+0301 → "á"),
+ *       otherwise normalizing already-normalized text would not be a no-op
+ *   4. whitespace/punct      — exotic spaces, dashes, quotes → ASCII forms
+ *
+ * Each transform independently produces { text, map }; the final offset map
+ * is the COMPOSITION of the per-stage maps (see offsetMap.ts for the map
+ * format and its invariants). Composing small verified maps is far easier to
+ * get right than building one map through four simultaneous transforms.
+ */
+import type { NormalizationOptions, NormalizationResult, TransformationRecord } from './types.js';
+import {
+  composeMaps,
+  identityMap,
+  buildReverseMap,
+  mapNormalizedSpan,
+  type TransformStepResult,
+} from './offsetMap.js';
+import { detectScripts } from './scripts.js';
+import { stripInvisibles } from './transforms/stripInvisibles.js';
+import { nfkcByGrapheme } from './transforms/nfkc.js';
+import { foldHomoglyphs } from './transforms/homoglyphFold.js';
+import { normalizeWhitespacePunct } from './transforms/whitespacePunct.js';
+import { foldDigits } from './transforms/foldDigits.js';
+
+export function normalize(text: string, options?: NormalizationOptions): NormalizationResult {
+  const opts = {
+    stripInvisibles: true,
+    nfkc: true,
+    homoglyphFold: true,
+    whitespacePunct: true,
+    foldDigits: true,
+    ...options,
+  };
+
+  /** The text after the transforms applied so far. */
+  let current = text;
+  /**
+   * Composed map: current index → ORIGINAL index. null means "identity so
+   * far" — kept symbolic so untouched text (the common case) never pays for
+   * map allocation or composition.
+   */
+  let cumulative: Int32Array | null = null;
+  const transformations: TransformationRecord[] = [];
+
+  const apply = (step: (input: string) => TransformStepResult | null): boolean => {
+    const result = step(current);
+    if (result === null) return false;
+    for (const change of result.changes) {
+      // The step reports ranges in ITS input's coordinates; map them back to
+      // original-text coordinates through the maps of the earlier steps.
+      // mapNormalizedSpan (not plain lookup) so a change inside an earlier
+      // expansion widens to the whole original cluster.
+      const span = cumulative
+        ? mapNormalizedSpan(cumulative, change.start, change.end)
+        : { start: change.start, end: change.end };
+      transformations.push({
+        kind: change.kind,
+        originalStart: span.start,
+        originalEnd: span.end,
+        original: text.slice(span.start, span.end),
+        replacement: change.after,
+      });
+    }
+    cumulative = cumulative ? composeMaps(cumulative, result.map) : result.map;
+    current = result.text;
+    return true;
+  };
+
+  // Per-cluster NFKC must run to a FIXPOINT: one pass's output can contain
+  // clusters that only became composable by being placed next to each other
+  // (the canonical case is Hangul compatibility jamo — ㅍ and ㅓ are separate
+  // clusters, normalize to the conjoining jamo ᄑ + ᅥ, and only THOSE form a
+  // single cluster that composes to 퍼 on the next look). One extra pass
+  // settles it in practice; the cap is a defensive bound, and the fuzz suite
+  // asserts both termination and idempotency.
+  const applyNfkcToFixpoint = (): void => {
+    for (let i = 0; i < 8 && apply(nfkcByGrapheme); i++) {
+      // apply() advanced the pipeline; loop until it reports no change.
+    }
+  };
+
+  if (opts.stripInvisibles) apply(stripInvisibles);
+  if (opts.nfkc) applyNfkcToFixpoint();
+  if (opts.homoglyphFold) {
+    const folded = apply(foldHomoglyphs);
+    // 3b — see the file header. Only needed when a fold actually happened,
+    // and only meaningful when the NFKC step is enabled at all.
+    if (folded && opts.nfkc) applyNfkcToFixpoint();
+  }
+  if (opts.whitespacePunct) apply(normalizeWhitespacePunct);
+  // Last, and after homoglyph folding on purpose: folding digits changes a
+  // token's script census, and doing it earlier would alter the
+  // dominant-script decision homoglyph folding depends on. ASCII digits are
+  // NFKC-stable, so no further normalization pass is owed.
+  if (opts.foldDigits) apply(foldDigits);
+
+  const offsetMap = cumulative ?? identityMap(text.length);
+  return {
+    normalizedText: current,
+    offsetMap,
+    reverseMap: buildReverseMap(offsetMap, text.length),
+    scripts: detectScripts(current),
+    transformations,
+    originalLength: text.length,
+    normalizedLength: current.length,
+  };
+}