@nationaldesignstudio/rampart 0.1.1

package/src/ner/worker.ts

@@ -0,0 +1,130 @@
+/**
+ * Web Worker host for the NER classifier.
+ *
+ * Inference must not jank the chat UI, so the model lives on a worker thread.
+ * The main thread talks to it through {@link createWorkerClassifier}, which
+ * adapts the postMessage round-trip back into the {@link TokenClassifier}
+ * signature the pipeline expects — so the rest of the system is agnostic to
+ * whether detection runs on the main thread or off it.
+ *
+ * Bundle this file as the worker entry (it self-registers `onmessage`). The
+ * client is created on the main thread with `new Worker(new URL(...))`.
+ */
+
+import { detectNer, loadNerClassifier, type NerOptions, type TokenClassifier } from "./classifier";
+
+interface InitMessage {
+  readonly kind: "init";
+  readonly options: NerOptions;
+}
+interface DetectMessage {
+  readonly kind: "detect";
+  readonly id: number;
+  readonly text: string;
+  readonly minScore?: number;
+}
+type InboundMessage = InitMessage | DetectMessage;
+
+type WorkerInboundEvent = {
+  data: InboundMessage;
+};
+
+type WorkerOutboundEvent = {
+  data: { kind: string; id?: number; spans?: unknown; message?: string };
+};
+
+export type WorkerMessagePort = {
+  onmessage: ((event: WorkerInboundEvent) => void) | null;
+  postMessage: (message: unknown) => void;
+};
+
+// --- Worker side (runs inside the worker thread) ---
+
+/** Register the worker message handler. Call from the worker entry module. */
+export function registerNerWorker(scope: WorkerMessagePort): void {
+  let classifierPromise: Promise<TokenClassifier> | null = null;
+
+  scope.onmessage = async (event: WorkerInboundEvent) => {
+    const message = event.data;
+    if (message.kind === "init") {
+      try {
+        classifierPromise = loadNerClassifier(message.options);
+        await classifierPromise;
+        scope.postMessage({ kind: "ready" });
+      } catch (error) {
+        // Surface init failures so the main thread can fail closed instead of
+        // hanging forever on a `ready` message that never arrives.
+        classifierPromise = null;
+        scope.postMessage({ kind: "error", message: String(error) });
+      }
+      return;
+    }
+    if (message.kind === "detect") {
+      try {
+        if (classifierPromise === null) throw new Error("[pii-filter] worker not initialized");
+        const classifier = await classifierPromise;
+        const spans = await detectNer(message.text, classifier, message.minScore);
+        scope.postMessage({ kind: "result", id: message.id, spans });
+      } catch (error) {
+        scope.postMessage({ kind: "error", id: message.id, message: String(error) });
+      }
+    }
+  };
+}
+
+// --- Main-thread side ---
+
+/**
+ * Wrap a worker as a {@link TokenClassifier}-compatible async function. The
+ * detection contract is span-in/span-out, so callers use it exactly like the
+ * in-process classifier. Resolves once the worker reports `ready`.
+ */
+export function createWorkerClassifier(
+  worker: WorkerMessagePort,
+  options: NerOptions,
+): { ready: Promise<void>; detect: (text: string, minScore?: number) => Promise<unknown> } {
+  let nextId = 0;
+  const pending = new Map<number, { resolve: (v: unknown) => void; reject: (e: unknown) => void }>();
+  let resolveReady: () => void = () => {
+    /* replaced synchronously by Promise constructor below */
+  };
+  let rejectReady: (error: unknown) => void = () => {
+    /* replaced synchronously by Promise constructor below */
+  };
+  const ready = new Promise<void>((resolve, reject) => {
+    resolveReady = resolve;
+    rejectReady = reject;
+  });
+
+  worker.onmessage = (event: WorkerOutboundEvent) => {
+    const data = event.data;
+    if (data.kind === "ready") {
+      resolveReady();
+      return;
+    }
+    // Worker reports a pre-init failure (no id) → reject the ready promise so
+    // callers fail loudly instead of hanging on `await ready`.
+    if (data.kind === "error" && data.id === undefined) {
+      rejectReady(new Error(data.message ?? "[pii-filter] worker init failed"));
+      return;
+    }
+    if (data.id === undefined) return;
+    const entry = pending.get(data.id);
+    if (entry === undefined) return;
+    pending.delete(data.id);
+    if (data.kind === "error") entry.reject(new Error(data.message));
+    else entry.resolve(data.spans);
+  };
+
+  worker.postMessage({ kind: "init", options });
+
+  function detect(text: string, minScore?: number): Promise<unknown> {
+    const id = nextId++;
+    return new Promise((resolve, reject) => {
+      pending.set(id, { resolve, reject });
+      worker.postMessage({ kind: "detect", id, text, minScore });
+    });
+  }
+
+  return { ready, detect };
+}

package/src/policy.ts

@@ -0,0 +1,64 @@
+/**
+ * Span reconciliation + default-deny policy.
+ *
+ * Detectors (heuristic + NER) emit overlapping, possibly conflicting spans.
+ * {@link mergeSpans} resolves them into a non-overlapping set, then
+ * {@link applyPolicy} drops anything in the keep-set so only redactable spans
+ * remain. This is Option A: redact detected entities whose label is not kept.
+ */
+
+import { KEEP_LABELS, shouldRedact, type PiiLabel, type Span } from "./types";
+
+/**
+ * Reduce overlapping spans to a disjoint set. When two spans overlap, the
+ * higher-confidence one wins; ties break toward the longer span, then toward
+ * heuristics (validator-backed, so most trustworthy). Biased to *keep* a
+ * redaction rather than drop one — recall over precision, per the threat model.
+ *
+ * On partial overlap (neither span contains the other) the previous span
+ * covers bytes the winner does not. Taking only the winner exposes those
+ * bytes; we take the byte-union under the preferred label so no detected
+ * bytes are silently dropped. Full containment still collapses to the winner.
+ */
+export function mergeSpans(spans: readonly Span[]): Span[] {
+  const sorted = [...spans].sort((a, b) => a.start - b.start || b.end - a.end);
+  const merged: Span[] = [];
+  for (const span of sorted) {
+    const prev = merged[merged.length - 1];
+    if (prev === undefined || span.start >= prev.end) {
+      merged.push(span);
+      continue;
+    }
+    const winner = preferred(prev, span);
+    const prevContains = prev.start <= span.start && prev.end >= span.end;
+    const spanContains = span.start <= prev.start && span.end >= prev.end;
+    if (prevContains || spanContains) {
+      merged[merged.length - 1] = winner;
+    } else {
+      // Partial overlap: union the bytes under the preferred label so the
+      // loser's exclusive range is not silently exposed.
+      const start = Math.min(prev.start, span.start);
+      const end = Math.max(prev.end, span.end);
+      merged[merged.length - 1] = { ...winner, start, end, text: winner.text };
+    }
+  }
+  return merged;
+}
+
+function preferred(a: Span, b: Span): Span {
+  if (a.score !== b.score) return a.score > b.score ? a : b;
+  const aLen = a.end - a.start;
+  const bLen = b.end - b.start;
+  if (aLen !== bLen) return aLen > bLen ? a : b;
+  return a.source === "heuristic" ? a : b;
+}
+
+/**
+ * Apply the keep-set. Returns only spans that must be redacted, sorted right to
+ * left so callers can splice from the end and keep earlier offsets valid.
+ */
+export function applyPolicy(spans: readonly Span[], keepLabels: ReadonlySet<PiiLabel> = KEEP_LABELS): Span[] {
+  return mergeSpans(spans)
+    .filter((s) => shouldRedact(s.label, keepLabels))
+    .sort((a, b) => b.start - a.start);
+}

package/src/premask.ts

@@ -0,0 +1,90 @@
+/**
+ * Pre-mask harness: substitute deterministically-detected structured PII with
+ * neutral sentinels *before* the contextual model runs.
+ *
+ * The heuristic layer (SSN / CREDIT_CARD / IP_ADDRESS / EMAIL / URL) is
+ * validator- or pattern-backed and runs synchronously. Rather than show the
+ * model raw card/SSN/IP digits or email/URL strings — which it never needs to
+ * classify and which only add noise — we replace each heuristic span with a
+ * fixed sentinel token (`[SSN]`, `[CREDIT_CARD]`, `[IP_ADDRESS]`, ...). The model
+ * is trained on text masked the exact same way, so the train-time and
+ * inference-time input distributions match by construction.
+ *
+ * Offsets: the masked string is paired with a per-character map back to the raw
+ * input, so a span the model reports in masked coordinates can be projected to
+ * exact raw offsets. Sentinel characters map onto their source span's raw range,
+ * so if the model happens to label part of a sentinel the projection lands
+ * inside the heuristic span (which then wins the merge on its score-1 weight).
+ *
+ * The heuristic spans themselves are returned with their original raw offsets,
+ * so final redaction and the session table are unaffected: only the text handed
+ * to the model is masked, never the text that gets placeholdered.
+ */
+
+import { mergeSpans } from "./policy";
+import type { PiiLabel, Span } from "./types";
+
+/** A masked copy of the input plus a map from each masked char to raw offsets. */
+export interface PremaskResult {
+  /** Input with heuristic spans replaced by sentinel tokens. */
+  readonly masked: string;
+  /** `rawStart[i]` is the raw offset of the source of `masked[i]`. */
+  readonly rawStart: number[];
+  /** `rawEnd[i]` is the raw offset just past that source. */
+  readonly rawEnd: number[];
+}
+
+/** The sentinel substituted for a premasked label. Stable across train/serve. */
+export function sentinelFor(label: PiiLabel): string {
+  return `[${label}]`;
+}
+
+/**
+ * Replace each (disjoint) heuristic span in `raw` with its label sentinel,
+ * recording per masked-character raw offsets. Spans are merged first so they are
+ * non-overlapping; verbatim runs between spans map 1:1 to raw, and sentinel
+ * characters map onto the whole span they stand in for.
+ */
+export function premask(raw: string, spans: readonly Span[]): PremaskResult {
+  const ordered = mergeSpans(spans).slice().sort((a, b) => a.start - b.start);
+  let masked = "";
+  const rawStart: number[] = [];
+  const rawEnd: number[] = [];
+
+  const copyVerbatim = (from: number, to: number): void => {
+    for (let i = from; i < to; i++) {
+      masked += raw[i];
+      rawStart.push(i);
+      rawEnd.push(i + 1);
+    }
+  };
+
+  let cursor = 0;
+  for (const span of ordered) {
+    if (span.start < cursor) continue; // defensive: skip any residual overlap
+    copyVerbatim(cursor, span.start);
+    const sentinel = sentinelFor(span.label);
+    for (const ch of sentinel) {
+      masked += ch;
+      rawStart.push(span.start);
+      rawEnd.push(span.end);
+    }
+    cursor = span.end;
+  }
+  copyVerbatim(cursor, raw.length);
+
+  return { masked, rawStart, rawEnd };
+}
+
+/**
+ * Project a span reported in masked coordinates back onto the raw input. Returns
+ * `null` for an empty/degenerate span. The raw `text` is sliced from `raw` so
+ * callers always carry the real substring for rehydration.
+ */
+export function projectMaskedSpan(span: Span, raw: string, map: PremaskResult): Span | null {
+  if (span.end <= span.start) return null;
+  const start = map.rawStart[span.start];
+  const end = map.rawEnd[span.end - 1];
+  if (start === undefined || end === undefined || end <= start) return null;
+  return { ...span, start, end, text: raw.slice(start, end) };
+}

package/src/session.ts

@@ -0,0 +1,99 @@
+/**
+ * Session entity table: reversible placeholders for coherent chat.
+ *
+ * Blanket `[REDACTED]` makes the assistant's replies nonsense. Instead each
+ * redacted value gets a stable, typed placeholder (`[GIVEN_NAME_1]`, `[SSN_2]`)
+ * that survives across turns: the same raw value always maps to the same token,
+ * so the model can reason about "GIVEN_NAME_1" and we {@link rehydrate} the real
+ * value back into its reply before display.
+ *
+ * The map lives only on the client. What leaves the device is placeholdered
+ * text; the table never crosses the wire.
+ */
+
+import { applyPolicy } from "./policy";
+import { KEEP_LABELS, type PiiLabel, type Span } from "./types";
+
+/** Result of scrubbing one message. */
+export interface ScrubResult {
+  /** Text with PII replaced by placeholders. Safe to send/log. */
+  readonly text: string;
+  /** Placeholders introduced or reused in this message. */
+  readonly placeholders: readonly string[];
+}
+
+/**
+ * Optional display aliases for placeholder tokens. By default a GIVEN_NAME span
+ * becomes `[GIVEN_NAME_1]`; pass `{ GIVEN_NAME: "NAME" }` to get `[NAME_1]` instead.
+ * Only the visible token changes — detection and policy are unaffected.
+ */
+export type PlaceholderAliases = Partial<Record<PiiLabel, string>>;
+
+/** Token shape minted by the table; also the regex used to find them on the
+ * way back. Kept in one place so scrub and rehydrate can never disagree. */
+export const PLACEHOLDER_PATTERN = /\[[A-Z][A-Z_]*_\d+\]/g;
+
+/**
+ * A per-conversation store mapping raw PII values to stable placeholders.
+ * Keyed by `label + normalized value` so "John" stays `GIVEN_NAME_1` on every turn
+ * and casing/whitespace noise doesn't mint duplicate tokens.
+ */
+export class SessionEntityTable {
+  private readonly forward = new Map<string, string>();
+  private readonly reverse = new Map<string, string>();
+  private readonly counters = new Map<string, number>();
+
+  constructor(
+    private readonly aliases: PlaceholderAliases = {},
+    private readonly keepLabels: ReadonlySet<PiiLabel> = KEEP_LABELS,
+  ) {}
+
+  /** The visible name used in tokens for a label (alias or the label itself). */
+  private displayName(label: PiiLabel): string {
+    return this.aliases[label] ?? label;
+  }
+
+  /** Get or mint the placeholder for a given label+value. Idempotent. */
+  placeholderFor(label: PiiLabel, value: string): string {
+    const key = `${label}:${value.toLowerCase().replace(/\s+/g, " ").trim()}`;
+    const existing = this.forward.get(key);
+    if (existing !== undefined) return existing;
+    const name = this.displayName(label);
+    const next = (this.counters.get(name) ?? 0) + 1;
+    this.counters.set(name, next);
+    const token = `[${name}_${next}]`;
+    this.forward.set(key, token);
+    this.reverse.set(token, value);
+    return token;
+  }
+
+  /**
+   * Replace each redactable span with its placeholder. Spans are pre-sorted
+   * right-to-left by {@link applyPolicy}, so splicing never invalidates an
+   * earlier offset.
+   */
+  scrub(raw: string, spans: readonly Span[]): ScrubResult {
+    const redactable = applyPolicy(spans, this.keepLabels);
+    const placeholders: string[] = [];
+    let text = raw;
+    for (const span of redactable) {
+      const token = this.placeholderFor(span.label, span.text);
+      placeholders.push(token);
+      text = `${text.slice(0, span.start)}${token}${text.slice(span.end)}`;
+    }
+    return { text, placeholders: placeholders.reverse() };
+  }
+
+  /**
+   * Restore real values in an assistant reply. Used on the *outbound* response
+   * so the user sees "John", not "[NAME_1]". Unknown tokens are left intact.
+   */
+  rehydrate(text: string): string {
+    return text.replace(PLACEHOLDER_PATTERN, (token) => this.reverse.get(token) ?? token);
+  }
+
+  /** True if `token` is a placeholder this table can resolve. */
+  knows(token: string): boolean {
+    return this.reverse.has(token);
+  }
+}

package/src/streaming.ts

@@ -0,0 +1,73 @@
+/**
+ * Streaming placeholder reveal.
+ *
+ * The assistant streams tokens, so a placeholder can be split across chunks:
+ *   chunk A: "...thanks [NA"
+ *   chunk B: "ME_1] for..."
+ * A per-chunk regex replace would emit the broken `[NA` and never match. This
+ * buffers the smallest suffix that could still become a placeholder, emits
+ * everything safe before it, and flushes the remainder at stream end.
+ */
+
+import { PLACEHOLDER_PATTERN } from "./session";
+
+/** Resolves a placeholder token to its real value, or null to leave it as-is. */
+export type PlaceholderResolver = (token: string) => string | null;
+
+// A run that *might* grow into a placeholder: '[' then [A-Z_] then optional
+// '_' digits, not yet closed by ']'. If a suffix matches this, hold it.
+const PARTIAL_TOKEN = /\[[A-Z_]*(?:_\d*)?$/;
+
+/**
+ * Stateful reveal for a token stream. Feed chunks in order; `push` returns the
+ * text safe to render now, `flush` returns whatever was held at the end.
+ */
+export class StreamingReveal {
+  private buffer = "";
+
+  constructor(private readonly resolve: PlaceholderResolver) {}
+
+  /** Reveal complete placeholders in `chunk`, holding any partial tail. */
+  push(chunk: string): string {
+    this.buffer += chunk;
+    const revealed = this.replaceComplete(this.buffer);
+    // Find the longest suffix that could still be completing a placeholder.
+    const partial = revealed.match(PARTIAL_TOKEN);
+    if (partial === null) {
+      this.buffer = "";
+      return revealed;
+    }
+    const cut = revealed.length - partial[0].length;
+    this.buffer = revealed.slice(cut);
+    return revealed.slice(0, cut);
+  }
+
+  /** Emit any buffered tail (e.g. a lone `[` that never became a token). */
+  flush(): string {
+    const out = this.replaceComplete(this.buffer);
+    this.buffer = "";
+    return out;
+  }
+
+  private replaceComplete(text: string): string {
+    return text.replace(PLACEHOLDER_PATTERN, (token) => this.resolve(token) ?? token);
+  }
+}
+
+/**
+ * A Web Streams transform that reveals placeholders in a string stream — drop
+ * it into an AI SDK text stream pipeline so the user never sees `[NAME_1]`.
+ */
+export function createRevealTransform(resolve: PlaceholderResolver): TransformStream<string, string> {
+  const reveal = new StreamingReveal(resolve);
+  return new TransformStream<string, string>({
+    transform(chunk, controller) {
+      const out = reveal.push(chunk);
+      if (out) controller.enqueue(out);
+    },
+    flush(controller) {
+      const out = reveal.flush();
+      if (out) controller.enqueue(out);
+    },
+  });
+}

package/src/types.ts

@@ -0,0 +1,74 @@
+/**
+ * Shared vocabulary for the PII filter.
+ *
+ * The whole system is *default-deny*: every detected entity is redacted unless
+ * its label is in {@link KEEP_LABELS} (currently {CITY, STATE, ZIP_CODE}). An
+ * unrecognized span is dropped, never leaked.
+ *
+ * Two layers emit labels:
+ *  - Heuristics (synchronous, validator-backed) detect and premask the
+ *    structured numeric identifiers below before the model ever runs.
+ *  - The NER model learns the contextual "fine set": split names, contact and
+ *    document identifiers, and address components.
+ */
+
+/** Every entity class the heuristics and the NER model can emit. */
+export type PiiLabel =
+  // --- Structured (heuristic-detectable, premasked before the model) ---
+  | "SSN"
+  | "CREDIT_CARD"
+  | "IP_ADDRESS"
+  // --- Contextual (NER model fine set) ---
+  | "GIVEN_NAME"
+  | "SURNAME"
+  | "EMAIL"
+  | "PHONE"
+  | "URL"
+  | "TAX_ID"
+  | "BANK_ACCOUNT"
+  | "ROUTING_NUMBER"
+  | "GOVERNMENT_ID"
+  | "PASSPORT"
+  | "DRIVERS_LICENSE"
+  // --- Address components (NER) ---
+  | "BUILDING_NUMBER"
+  | "STREET_NAME"
+  | "SECONDARY_ADDRESS"
+  | "CITY"
+  | "STATE"
+  | "ZIP_CODE";
+
+/**
+ * Labels that are intentionally preserved (classified but never redacted).
+ * The model still learns these; the JS policy layer simply keeps them in the
+ * text. Public-benefits assistants need broad geography (city / state / ZIP)
+ * for area-median-income (AMI) eligibility, so those are kept while the precise
+ * street line (BUILDING_NUMBER + STREET_NAME) is still redacted.
+ */
+export const KEEP_LABELS: ReadonlySet<PiiLabel> = new Set<PiiLabel>(["CITY", "STATE", "ZIP_CODE"]);
+
+/** A detected entity span over the *original* (raw) text. */
+export interface Span {
+  /** Inclusive start offset into the raw input. */
+  readonly start: number;
+  /** Exclusive end offset into the raw input. */
+  readonly end: number;
+  /** The classified entity type. */
+  readonly label: PiiLabel;
+  /** Detector confidence in [0, 1]. Heuristics with validators report 1. */
+  readonly score: number;
+  /** Which layer produced the span; used for merge tie-breaks and audit. */
+  readonly source: "heuristic" | "ner";
+  /** The raw substring covered, retained for placeholder rehydration. */
+  readonly text: string;
+}
+
+/** Resolve a caller keep-set; omitted → {@link KEEP_LABELS}. */
+export function resolveKeepLabels(keepLabels?: readonly PiiLabel[]): ReadonlySet<PiiLabel> {
+  return keepLabels === undefined ? KEEP_LABELS : new Set(keepLabels);
+}
+
+/** True when a label must be redacted under the default-deny policy. */
+export function shouldRedact(label: PiiLabel, keepLabels: ReadonlySet<PiiLabel> = KEEP_LABELS): boolean {
+  return !keepLabels.has(label);
+}

package/src/validators.ts

@@ -0,0 +1,40 @@
+/**
+ * Validators that turn "a run of digits" into "a *valid* SSN / card / phone".
+ *
+ * These exist to suppress false positives so the filter doesn't mangle numbers
+ * the assistant is allowed to keep (income figures, ages, years). A detector
+ * proposes a span; a validator decides whether it is really that entity.
+ */
+
+/** Luhn checksum — gates CREDIT_CARD so arbitrary 16-digit runs don't match. */
+export function isLuhnValid(digits: string): boolean {
+  let sum = 0;
+  let double = false;
+  for (let i = digits.length - 1; i >= 0; i--) {
+    let d = digits.charCodeAt(i) - 0x30;
+    if (double) {
+      d *= 2;
+      if (d > 9) d -= 9;
+    }
+    sum += d;
+    double = !double;
+  }
+  return sum % 10 === 0;
+}
+
+/**
+ * US SSN structural rules. Area (first 3) cannot be 000, 666, or 900-999;
+ * group (middle 2) cannot be 00; serial (last 4) cannot be 0000. Rejects
+ * obvious non-SSNs like phone numbers padded to nine digits.
+ */
+export function isValidSsn(digits: string): boolean {
+  if (digits.length !== 9) return false;
+  const area = digits.slice(0, 3);
+  const group = digits.slice(3, 5);
+  const serial = digits.slice(5);
+  if (area === "000" || area === "666") return false;
+  if (Number(area) >= 900) return false;
+  if (group === "00") return false;
+  if (serial === "0000") return false;
+  return true;
+}