@nationaldesignstudio/rampart 0.1.1

package/package.json

@@ -0,0 +1,103 @@
+{
+  "name": "@nationaldesignstudio/rampart",
+  "version": "0.1.1",
+  "description": "Rampart — client-side PII redaction for AI assistants: deterministic recognizers + a 14.7 MB ONNX classifier (transformers.js), default-deny policy, and reversible placeholders. Runs entirely in the browser.",
+  "license": "CC-BY-4.0",
+  "homepage": "https://github.com/nationaldesignstudio/rampart#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/nationaldesignstudio/rampart.git"
+  },
+  "bugs": {
+    "url": "https://github.com/nationaldesignstudio/rampart/issues"
+  },
+  "keywords": [
+    "pii",
+    "redaction",
+    "privacy",
+    "llm",
+    "browser",
+    "onnx",
+    "transformers.js"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./dist": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./worker": {
+      "types": "./dist/src/ner/worker.d.ts",
+      "import": "./dist/src/ner/worker.js",
+      "default": "./dist/src/ner/worker.js"
+    },
+    "./dist/worker": {
+      "types": "./dist/src/ner/worker.d.ts",
+      "import": "./dist/src/ner/worker.js",
+      "default": "./dist/src/ner/worker.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "files": [
+    "dist",
+    "eval",
+    "examples",
+    "index.ts",
+    "LICENSE",
+    "src",
+    "MODEL_CARD.md",
+    "README.md",
+    "RELEASE.md",
+    "WHITEPAPER.md",
+    "!eval/bench/webgpu/bundle.js",
+    "!eval/bench/data",
+    "!eval/bench/runs/*/latency.json"
+  ],
+  "scripts": {
+    "build": "bun build ./index.ts ./src/ner/worker.ts --outdir ./dist --format esm --sourcemap --target browser && tsc -p tsconfig.build.json",
+    "eval:public": "bun eval/run-public-eval.ts",
+    "eval:public:strict": "bun eval/run-public-eval.ts --strict",
+    "bench:fetch": "bun eval/bench/fetch.ts",
+    "bench": "bun eval/bench/run.ts",
+    "bench:webgpu": "bun eval/bench/webgpu.ts",
+    "bench:webgpu:wasm": "bun eval/bench/webgpu.ts --device wasm",
+    "export:huggingface": "bun scripts/export-huggingface.ts",
+    "export:huggingface:verify": "bun run export:huggingface && bun scripts/verify-huggingface-export.ts",
+    "publish:huggingface": "bun run export:huggingface:verify && hf repo create nationaldesignstudio/rampart --repo-type model --private --exist-ok && hf upload nationaldesignstudio/rampart hf-export . --repo-type model",
+    "prepack": "bun run build",
+    "prepublishOnly": "bun run verify:public",
+    "redact": "bun cli/redact.ts",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "type-check": "tsc -p tsconfig.json --noEmit",
+    "verify:public": "bun run build && bun test && bun run type-check && bun run eval:public:strict && bun run export:huggingface:verify"
+  },
+  "peerDependencies": {
+    "@huggingface/transformers": ">=3"
+  },
+  "peerDependenciesMeta": {
+    "@huggingface/transformers": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@huggingface/transformers": "3.7.5",
+    "@types/bun": "1.3.14",
+    "playwright": "1.61.1",
+    "typescript": "5.9.2",
+    "vitest": "4.1.8"
+  },
+  "packageManager": "bun@1.3.14"
+}

package/src/guard.ts

@@ -0,0 +1,170 @@
+/**
+ * ChatGuard: per-conversation PII filter for wiring into a chat app.
+ *
+ *   const guard = await createGuard();
+ *   const safe = await guard.protect(userInput);
+ *   const reply = await llm(safe.text);
+ *   guard.reveal(reply);                    // non-streaming
+ *   stream.pipeThrough(guard.revealTransform()); // streaming
+ *
+ * The entity table lives only on the client; the real values never leave the
+ * device. Each guard keeps placeholder identity stable across every turn of a
+ * conversation.
+ *
+ *   user message
+ *     → heuristics (sync, structured PII)
+ *     → NER (optional, async, contextual PII)
+ *     → merge + default-deny policy (keep {city, state, zip})
+ *     → session table: replace with stable placeholders
+ *   assistant reply
+ *     → rehydrate placeholders → render to user
+ */
+
+import { detectHeuristics } from "./heuristics";
+import { loadNerClassifier, detectNer, RAMPART_MODEL_ID, type NerOptions } from "./ner/classifier";
+import { createWorkerClassifier, type WorkerMessagePort } from "./ner/worker";
+import { premask, projectMaskedSpan } from "./premask";
+import { createRevealTransform } from "./streaming";
+import { SessionEntityTable, type PlaceholderAliases, type ScrubResult } from "./session";
+import { resolveKeepLabels, type PiiLabel, type Span } from "./types";
+
+/** An async contextual detector. Matches both the in-process and worker forms. */
+export type NerDetector = (text: string) => Promise<Span[]>;
+
+/**
+ * Default placeholder aliases. Empty: names are split into GIVEN_NAME/SURNAME
+ * and a household may share a surname, so each keeps its own typed token
+ * (`[GIVEN_NAME_1]`, `[SURNAME_1]`) rather than collapsing to a single `NAME`.
+ */
+export const DEFAULT_ALIASES: PlaceholderAliases = {};
+
+export interface GuardOptions {
+  /** Placeholder aliases. Defaults to `{}` — typed tokens like `[GIVEN_NAME_1]`. */
+  readonly aliases?: PlaceholderAliases;
+  /** Labels to preserve; defaults to `{CITY, STATE, ZIP_CODE}`. */
+  readonly keepLabels?: readonly PiiLabel[];
+  /**
+   * When `true`, skip the structured-PII premask before the model. Required for
+   * a model trained without prefilter (no-prefilter ablation) whose classes
+   * include SSN / CREDIT_CARD / IP_ADDRESS. Heuristic spans for those types
+   * still run as a safety net.
+   */
+  readonly noPrefilter?: boolean;
+  /** Pre-built NER detector. When set, `model` is ignored. */
+  readonly ner?: NerDetector;
+  /** When `true`, skip the classifier and run heuristics only. */
+  readonly heuristicsOnly?: boolean;
+  /**
+   * Hugging Face model id or local directory path (q4 ONNX). Defaults to
+   * {@link RAMPART_MODEL_ID}.
+   */
+  readonly model?: string;
+  /** Worker script URL. When set, NER runs off the main thread. */
+  readonly worker?: string | URL;
+  /** Backend. `"wasm"`/`"webgpu"` in browsers; `"cpu"` for Node. */
+  readonly device?: NerOptions["device"];
+  /** Spans below this score are discarded. */
+  readonly minScore?: number;
+}
+
+type ChatGuardConfig = Pick<GuardOptions, "ner" | "aliases" | "keepLabels" | "noPrefilter">;
+
+export class ChatGuard {
+  private readonly table: SessionEntityTable;
+  private readonly ner?: NerDetector;
+  private readonly noPrefilter: boolean;
+
+  constructor(config: ChatGuardConfig = {}) {
+    this.table = new SessionEntityTable(config.aliases, resolveKeepLabels(config.keepLabels));
+    this.ner = config.ner;
+    this.noPrefilter = config.noPrefilter ?? false;
+  }
+
+  private async detect(text: string): Promise<Span[]> {
+    const heuristic = detectHeuristics(text);
+    if (this.ner === undefined) return heuristic;
+    if (this.noPrefilter) {
+      const modelSpans = await this.ner(text);
+      return [...heuristic, ...modelSpans];
+    }
+    const map = premask(text, heuristic);
+    const maskedSpans = await this.ner(map.masked);
+    const contextual: Span[] = [];
+    for (const span of maskedSpans) {
+      const projected = projectMaskedSpan(span, text, map);
+      if (projected !== null) contextual.push(projected);
+    }
+    return [...heuristic, ...contextual];
+  }
+
+  /**
+   * Run this on the user's text *before* handing it to the AI SDK. Returns the
+   * placeholdered text to send plus the placeholders introduced this turn.
+   */
+  async protect(text: string): Promise<ScrubResult> {
+    const spans = await this.detect(text);
+    return this.table.scrub(text, spans);
+  }
+
+  /** Restore real values in a complete (non-streaming) assistant reply. */
+  reveal(reply: string): string {
+    return this.table.rehydrate(reply);
+  }
+
+  /**
+   * A Web Streams transform that reveals placeholders in a streamed reply,
+   * correctly handling placeholders split across chunks. Pipe an AI SDK
+   * `textStream` through it before rendering.
+   */
+  revealTransform(): TransformStream<string, string> {
+    return createRevealTransform((token) => {
+      const restored = this.table.rehydrate(token);
+      return restored === token ? null : restored;
+    });
+  }
+
+  /**
+   * Defense in depth: scrub the model's *output* before logging/persisting it,
+   * since a model can emit PII the user never typed. Returns placeholdered text.
+   */
+  async protectReply(reply: string): Promise<ScrubResult> {
+    const spans = await this.detect(reply);
+    return this.table.scrub(reply, spans);
+  }
+}
+
+type NerLoadOptions = Pick<GuardOptions, "model" | "worker" | "minScore" | "device">;
+
+async function buildNer(options: NerLoadOptions): Promise<NerDetector> {
+  const { model, worker, minScore, device = "wasm" } = options;
+  const modelOptions = { model, device, minScore };
+
+  if (worker !== undefined) {
+    // The DOM `Worker` types `onmessage` with `MessageEvent`; `WorkerMessagePort`
+    // is the deliberately DOM-free structural port the worker module also uses.
+    // They are not mutually assignable, so cast at this single boundary.
+    const port = new Worker(worker, { type: "module" }) as unknown as WorkerMessagePort;
+    const classifier = createWorkerClassifier(port, modelOptions);
+    await classifier.ready;
+    return async (text) => (await classifier.detect(text)) as Span[];
+  }
+
+  const classifier = await loadNerClassifier(modelOptions);
+  return (text) => detectNer(text, classifier, minScore);
+}
+
+/**
+ * Build a conversation guard. Loads the Rampart classifier (q4 ONNX) by default.
+ * Pass `model` for a different Hugging Face id or local path, `heuristicsOnly:
+ * true` to skip NER, or `ner` to inject a custom detector.
+ */
+export async function createGuard(options: GuardOptions = {}): Promise<ChatGuard> {
+  const { aliases = DEFAULT_ALIASES, keepLabels, noPrefilter, ner, heuristicsOnly, ...nerLoad } = options;
+
+  let detector = ner;
+  if (detector === undefined && heuristicsOnly !== true) {
+    detector = await buildNer(nerLoad);
+  }
+
+  return new ChatGuard({ ner: detector, aliases, keepLabels, noPrefilter });
+}

package/src/heuristics.ts

@@ -0,0 +1,141 @@
+/**
+ * Heuristic detectors: the cheap, synchronous, zero-model first pass.
+ *
+ * Digit-bearing PII (SSN, card, phone, routing) is found over the *digit
+ * projection* (see normalize.ts) so every separator variant collapses to one
+ * rule: `888-88-8888`, `888 88 8888`, `888.88.8888`, and `888888888` all match.
+ * Text-shaped PII (email, URL, IP) is matched on the raw string where the
+ * structure lives in the punctuation.
+ */
+
+import type { PiiLabel, Span } from "./types";
+import { isLuhnValid, isValidSsn } from "./validators";
+
+/** A fixed-length digit candidate paired with its label and validator. */
+interface DigitRule {
+  readonly label: PiiLabel;
+  /** Exact digit lengths to try, longest first so cards beat SSN/phone. */
+  readonly lengths: readonly number[];
+  /** Returns true if the digit slice is a real instance of this entity. */
+  readonly validate: (digits: string) => boolean;
+}
+
+// Order matters: longer/stricter entities are tried first so a 16-digit card is
+// never carved into a 9-digit "SSN". Overlap is also resolved later in merge.
+const DIGIT_RULES: readonly DigitRule[] = [
+  { label: "CREDIT_CARD", lengths: [16, 15, 14], validate: isLuhnValid },
+  { label: "SSN", lengths: [9], validate: isValidSsn },
+];
+
+/**
+ * A contiguous digit run with its inline separators, e.g. `888 12 3456`. Runs
+ * never span a letter or other word boundary, so digits from different fields
+ * ("...3456 and income is 50000") can't merge into one phantom candidate.
+ */
+interface DigitRun {
+  /** Separator-free digits of the run. */
+  readonly digits: string;
+  /** `rawIndex[i]` is the raw offset of `digits[i]`. */
+  readonly rawIndex: number[];
+}
+
+// A run is digits joined only by single inline separators (space, dash, dot).
+const DIGIT_RUN = /\d(?:[ .-]?\d)*/g;
+
+function extractRuns(raw: string): DigitRun[] {
+  const runs: DigitRun[] = [];
+  DIGIT_RUN.lastIndex = 0;
+  for (let m = DIGIT_RUN.exec(raw); m !== null; m = DIGIT_RUN.exec(raw)) {
+    const digits: string[] = [];
+    const rawIndex: number[] = [];
+    for (let i = 0; i < m[0].length; i++) {
+      const code = m[0].charCodeAt(i);
+      if (code >= 0x30 && code <= 0x39) {
+        digits.push(m[0][i]);
+        rawIndex.push(m.index + i);
+      }
+    }
+    runs.push({ digits: digits.join(""), rawIndex });
+  }
+  return runs;
+}
+
+/**
+ * Scan each digit run for fixed-length, validator-backed entities. A candidate
+ * is anchored to the *whole* run (start..len and end-len..end) so we only match
+ * when the run length equals the entity length — preventing a 9-digit SSN from
+ * firing inside a longer account number. The raw span includes interior
+ * separators so redaction covers `888-12-3456` whole.
+ */
+function detectDigitEntities(raw: string): Span[] {
+  const spans: Span[] = [];
+  for (const run of extractRuns(raw)) {
+    for (const rule of DIGIT_RULES) {
+      if (!rule.lengths.includes(run.digits.length)) continue;
+      if (!rule.validate(run.digits)) continue;
+      const start = run.rawIndex[0];
+      const end = run.rawIndex[run.rawIndex.length - 1] + 1;
+      spans.push({ start, end, label: rule.label, score: 1, source: "heuristic", text: raw.slice(start, end) });
+      break; // first matching rule (rules are ordered strict→loose) wins the run
+    }
+  }
+  return spans;
+}
+
+// Text-shaped entities: structure lives in the punctuation, so match raw.
+// EMAIL and URL are structured text-shaped PII: a regex catches them at near-
+// 100% recall, far better than the model (URL recall is ~5% model-alone), so
+// the deterministic layer owns them and premasks them to sentinels before the
+// model — exactly as it does for SSN/CC/IP. The model's vestigial EMAIL/URL
+// head no longer needs to fire.
+const TEXT_RULES: readonly { label: PiiLabel; pattern: RegExp; group?: number }[] = [
+  // Email: local part (with +tags and dots) @ dotted domain. Matches
+  // plus-addressing and sub-domains, e.g. `alex+housing@sub.example.gov`.
+  { label: "EMAIL", pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g },
+  // URL with explicit scheme: everything up to whitespace / closing bracket /
+  // quote. Trailing sentence punctuation is left in the span (harmless — the
+  // sensitive host+path is what matters and the whole run is redacted).
+  { label: "URL", pattern: /\bhttps?:\/\/[^\s<>"'\])}]+/g },
+  // Schemeless web URL: `www.` host followed by the rest of the URL.
+  { label: "URL", pattern: /\bwww\.[A-Za-z0-9.-]+\.[A-Za-z]{2,}(?:\/[^\s<>"'\])}]*)?/g },
+  { label: "IP_ADDRESS", pattern: /\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b/g },
+  // IPv6: full and `::`-compressed forms. Every alternative requires either 8
+  // colon-separated groups or a `::`, so it never fires on times ("12:34") or
+  // MAC addresses (handled below); the lookarounds keep it off larger tokens.
+  {
+    label: "IP_ADDRESS",
+    pattern:
+      /(?<![:.\w])(?:(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,7}:|(?:[0-9A-Fa-f]{1,4}:){1,6}:[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,5}(?::[0-9A-Fa-f]{1,4}){1,2}|(?:[0-9A-Fa-f]{1,4}:){1,4}(?::[0-9A-Fa-f]{1,4}){1,3}|(?:[0-9A-Fa-f]{1,4}:){1,3}(?::[0-9A-Fa-f]{1,4}){1,4}|(?:[0-9A-Fa-f]{1,4}:){1,2}(?::[0-9A-Fa-f]{1,4}){1,5}|[0-9A-Fa-f]{1,4}:(?::[0-9A-Fa-f]{1,4}){1,6}|::(?:[0-9A-Fa-f]{1,4}:){0,6}[0-9A-Fa-f]{1,4})(?![:.\w])/g,
+  },
+  // MAC address: six hex pairs joined by ":" or "-".
+  { label: "IP_ADDRESS", pattern: /\b(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}\b/g },
+];
+
+function detectTextEntities(raw: string): Span[] {
+  const spans: Span[] = [];
+  for (const { label, pattern, group } of TEXT_RULES) {
+    pattern.lastIndex = 0;
+    for (let m = pattern.exec(raw); m !== null; m = pattern.exec(raw)) {
+      const text = group === undefined ? m[0] : m[group];
+      const offset = group === undefined ? 0 : m[0].indexOf(text);
+      spans.push({
+        start: m.index + offset,
+        end: m.index + offset + text.length,
+        label,
+        score: 1,
+        source: "heuristic",
+        text,
+      });
+    }
+  }
+  return spans;
+}
+
+/** Run all heuristic detectors over the raw input. Spans may overlap; the
+ * pipeline's merge step resolves conflicts before redaction. */
+export function detectHeuristics(raw: string): Span[] {
+  return [
+    ...detectDigitEntities(raw),
+    ...detectTextEntities(raw),
+  ];
+}