@nationaldesignstudio/rampart 0.1.1

package/eval/bench/runs/sample-900/summary.json

@@ -0,0 +1,56 @@
+{
+  "rows": 900,
+  "private_total": 3991,
+  "private_recall": 0.9969932347782511,
+  "private_recall_wilson95": [
+    0.994751461461101,
+    0.9982791508164921
+  ],
+  "private_recall_bootstrap95": [
+    0.9952392883988975,
+    0.9984966173891255
+  ],
+  "leaked": 12,
+  "public_total": 2405,
+  "public_retained": 0.8731808731808732,
+  "over_redacted": 305,
+  "span_f1": {
+    "iou_1.00": {
+      "tp": 2386,
+      "fp": 1404,
+      "fn": 1605,
+      "precision": 0.6295514511873351,
+      "recall": 0.59784515159108,
+      "f1": 0.6132887803624213
+    },
+    "iou_0.50": {
+      "tp": 2983,
+      "fp": 807,
+      "fn": 1008,
+      "precision": 0.7870712401055409,
+      "recall": 0.7474317213730894,
+      "f1": 0.7667394936383498
+    },
+    "iou_0.00": {
+      "tp": 3141,
+      "fp": 649,
+      "fn": 850,
+      "precision": 0.8287598944591029,
+      "recall": 0.7870207967927838,
+      "f1": 0.8073512402004883
+    }
+  },
+  "calibration": {
+    "ece": 0.21079228912807385,
+    "n_pairs": 3790
+  },
+  "latency_ms": {
+    "cold": 77.65675199999896,
+    "p50": 14.373870999999781,
+    "p95": 36.97866299999987,
+    "p99": 48.98503100000016,
+    "mean": 17.03590428111117
+  },
+  "model": "./model",
+  "rows_scored": 900
+}

package/eval/bench/score.ts

@@ -0,0 +1,197 @@
+/**
+ * Scoring for the native bench: term-presence recall/retention, span-level F1,
+ * calibration (ECE/Brier) and percentile latency. The predictions are produced
+ * by the shipped TypeScript pipeline, so the metrics describe the artifact that
+ * ships rather than a separate evaluation implementation.
+ */
+
+export interface GoldSpan {
+  readonly label: string;
+  readonly start: number;
+  readonly end: number;
+}
+
+export interface PredSpan extends GoldSpan {
+  /** Detector confidence; deterministic spans are 1. */
+  readonly score: number;
+}
+
+/** Wilson 95% CI for a binomial proportion. Robust at small n and at p=0/1. */
+export function wilsonCi(k: number, n: number, z = 1.96): [number, number] {
+  if (n === 0) return [0, 1];
+  const phat = k / n;
+  const denom = 1 + (z * z) / n;
+  const center = (phat + (z * z) / (2 * n)) / denom;
+  const half = (z * Math.sqrt((phat * (1 - phat)) / n + (z * z) / (4 * n * n))) / denom;
+  return [Math.max(0, center - half), Math.min(1, center + half)];
+}
+
+/** Deterministic PRNG so bootstrap CIs are reproducible run to run. */
+function mulberry32(seed: number): () => number {
+  let a = seed >>> 0;
+  return () => {
+    a = (a + 0x6d2b79f5) | 0;
+    let t = Math.imul(a ^ (a >>> 15), 1 | a);
+    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+/** Percentile-bootstrap CI for the mean of 0/1 outcomes. */
+export function bootstrapCi(successes: readonly boolean[], iters = 1000, alpha = 0.05, seed = 0): [number, number] {
+  if (successes.length === 0) return [0, 1];
+  const rng = mulberry32(seed);
+  const n = successes.length;
+  const means: number[] = [];
+  for (let it = 0; it < iters; it++) {
+    let s = 0;
+    for (let i = 0; i < n; i++) s += successes[Math.floor(rng() * n)] ? 1 : 0;
+    means.push(s / n);
+  }
+  means.sort((a, b) => a - b);
+  return [means[Math.floor((iters * alpha) / 2)], means[Math.floor(iters * (1 - alpha / 2))]];
+}
+
+export function iou(a: GoldSpan, b: GoldSpan): number {
+  const inter = Math.max(0, Math.min(a.end, b.end) - Math.max(a.start, b.start));
+  if (inter === 0) return 0;
+  return inter / (Math.max(a.end, b.end) - Math.min(a.start, b.start));
+}
+
+/** (TP, FP, FN) under one-to-one greedy matching at the given IoU; higher-scored predictions match first. */
+export function matchSpans(gold: readonly GoldSpan[], pred: readonly PredSpan[], threshold = 1): { tp: number; fp: number; fn: number } {
+  const goldUsed = new Array(gold.length).fill(false);
+  const order = [...pred].sort((a, b) => b.score - a.score);
+  let tp = 0;
+  let fp = 0;
+  for (const p of order) {
+    let bestIou = 0;
+    let bestJ = -1;
+    for (let j = 0; j < gold.length; j++) {
+      if (goldUsed[j] || gold[j].label !== p.label) continue;
+      const s = iou(p, gold[j]);
+      if (s >= threshold && s > bestIou) {
+        bestIou = s;
+        bestJ = j;
+      }
+    }
+    if (bestJ >= 0) {
+      goldUsed[bestJ] = true;
+      tp += 1;
+    } else {
+      fp += 1;
+    }
+  }
+  return { tp, fp, fn: goldUsed.filter((u) => !u).length };
+}
+
+export function f1(tp: number, fp: number, fn: number): { precision: number; recall: number; f1: number } {
+  const precision = tp + fp ? tp / (tp + fp) : 0;
+  const recall = tp + fn ? tp / (tp + fn) : 0;
+  const f = precision + recall ? (2 * precision * recall) / (precision + recall) : 0;
+  return { precision, recall, f1: f };
+}
+
+/** Replace each span with a `[LABEL]` placeholder, right to left. */
+export function redactText(raw: string, spans: readonly GoldSpan[]): string {
+  let out = raw;
+  for (const s of [...spans].sort((a, b) => b.start - a.start)) {
+    out = out.slice(0, s.start) + `[${s.label}]` + out.slice(s.end);
+  }
+  return out;
+}
+
+export interface TermResult {
+  readonly leaked: number;
+  readonly protectedCount: number;
+  readonly over: number;
+  readonly retained: number;
+}
+
+/** One row: each private term must vanish from the redacted text; each public term must remain. */
+export function termPresence(redacted: string, goldPrivate: readonly string[], goldPublic: readonly string[]): TermResult {
+  const leaked = goldPrivate.filter((t) => t && redacted.includes(t)).length;
+  const over = goldPublic.filter((t) => t && !redacted.includes(t)).length;
+  return { leaked, protectedCount: goldPrivate.length - leaked, over, retained: goldPublic.length - over };
+}
+
+/** 15-bin reliability ECE over (predicted_score, was_correct) pairs. */
+export function expectedCalibrationError(pairs: readonly [number, boolean][], nBins = 15): number {
+  if (pairs.length === 0) return 0;
+  const bins: [number, boolean][][] = Array.from({ length: nBins }, () => []);
+  for (const [score, correct] of pairs) bins[Math.min(nBins - 1, Math.floor(score * nBins))].push([score, correct]);
+  let ece = 0;
+  for (const b of bins) {
+    if (!b.length) continue;
+    const acc = b.filter(([, c]) => c).length / b.length;
+    const conf = b.reduce((s, [v]) => s + v, 0) / b.length;
+    ece += Math.abs(acc - conf) * (b.length / pairs.length);
+  }
+  return ece;
+}
+
+export function brierScore(pairs: readonly [number, boolean][]): number {
+  if (pairs.length === 0) return 0;
+  return pairs.reduce((s, [score, c]) => s + (score - (c ? 1 : 0)) ** 2, 0) / pairs.length;
+}
+
+export function percentile(sorted: readonly number[], p: number): number {
+  if (sorted.length === 0) return 0;
+  return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))];
+}
+
+/** Accumulates per-row results into one bucket and reports the summary.json shape. */
+export class Stratum {
+  rows = 0;
+  leaked = 0;
+  privateTotal = 0;
+  over = 0;
+  publicTotal = 0;
+  private readonly outcomes: boolean[] = [];
+  private readonly spanTp: Record<string, number> = {};
+  private readonly spanFp: Record<string, number> = {};
+  private readonly spanFn: Record<string, number> = {};
+
+  addTerm(term: TermResult): void {
+    this.rows += 1;
+    this.leaked += term.leaked;
+    this.privateTotal += term.leaked + term.protectedCount;
+    this.over += term.over;
+    this.publicTotal += term.over + term.retained;
+    for (let i = 0; i < term.protectedCount; i++) this.outcomes.push(true);
+    for (let i = 0; i < term.leaked; i++) this.outcomes.push(false);
+  }
+
+  addSpans(gold: readonly GoldSpan[], pred: readonly PredSpan[], thresholds: readonly number[]): void {
+    for (const t of thresholds) {
+      const key = t.toFixed(2);
+      const { tp, fp, fn } = matchSpans(gold, pred, t);
+      this.spanTp[key] = (this.spanTp[key] ?? 0) + tp;
+      this.spanFp[key] = (this.spanFp[key] ?? 0) + fp;
+      this.spanFn[key] = (this.spanFn[key] ?? 0) + fn;
+    }
+  }
+
+  report(): Record<string, unknown> {
+    const recall = this.privateTotal ? (this.privateTotal - this.leaked) / this.privateTotal : 1;
+    const spanF1: Record<string, unknown> = {};
+    for (const key of Object.keys(this.spanTp)) {
+      const tp = this.spanTp[key];
+      const fp = this.spanFp[key];
+      const fn = this.spanFn[key];
+      spanF1[`iou_${key}`] = { tp, fp, fn, ...f1(tp, fp, fn) };
+    }
+    return {
+      rows: this.rows,
+      private_total: this.privateTotal,
+      private_recall: recall,
+      private_recall_wilson95: wilsonCi(this.privateTotal - this.leaked, this.privateTotal),
+      private_recall_bootstrap95: bootstrapCi(this.outcomes),
+      leaked: this.leaked,
+      public_total: this.publicTotal,
+      public_retained: this.publicTotal ? (this.publicTotal - this.over) / this.publicTotal : 1,
+      over_redacted: this.over,
+      span_f1: spanF1,
+    };
+  }
+}

package/eval/bench/webgpu/entry.ts

@@ -0,0 +1,70 @@
+/**
+ * Browser entry for the WebGPU latency harness.
+ *
+ * Runs the exact shipped detection path (heuristics → premask → NER → policy)
+ * the same way `eval/bench/run.ts` does, but inside a real Chromium tab with the
+ * NER model executing on the WebGPU backend. The Node bench measures ORT-CPU
+ * latency; this measures the form factor that actually ships to the browser.
+ *
+ * `bun eval/bench/webgpu.ts` bundles this for the browser, serves it, drives it
+ * with Playwright, and writes the latency summary.
+ */
+
+import { env } from "@huggingface/transformers";
+import { detectHeuristics } from "../../../src/heuristics";
+import { detectNer, loadNerClassifier, type TokenClassifier } from "../../../src/ner/classifier";
+import { applyPolicy } from "../../../src/policy";
+import { premask, projectMaskedSpan } from "../../../src/premask";
+import type { Span } from "../../../src/types";
+
+interface BenchOptions {
+  readonly modelBaseUrl: string;
+  readonly device: "webgpu" | "wasm";
+}
+
+let classifier: TokenClassifier | null = null;
+
+/** Configure transformers.js to fetch the committed local model over HTTP. */
+function configureEnv(baseUrl: string): void {
+  env.allowRemoteModels = false;
+  env.allowLocalModels = true;
+  // transformers.js resolves a model id against localModelPath; "model" → /model.
+  env.localModelPath = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
+}
+
+async function init(opts: BenchOptions): Promise<{ ok: true; adapter: string }> {
+  configureEnv(opts.modelBaseUrl);
+  classifier = await loadNerClassifier({ device: opts.device });
+  const gpu = (navigator as unknown as { gpu?: { requestAdapter: () => Promise<{ info?: { vendor?: string; architecture?: string } } | null> } }).gpu;
+  const adapter = gpu ? await gpu.requestAdapter() : null;
+  const info = adapter?.info;
+  return { ok: true, adapter: info ? `${info.vendor ?? "?"}/${info.architecture ?? "?"}` : "n/a" };
+}
+
+/** One row through the shipped pipeline; returns wall-clock ms for the row. */
+async function scrubOnceTimed(text: string): Promise<number> {
+  if (!classifier) throw new Error("classifier not initialised");
+  const t0 = performance.now();
+  const heuristic = detectHeuristics(text);
+  const map = premask(text, heuristic);
+  const masked = await detectNer(map.masked, classifier);
+  const modelSpans: Span[] = [];
+  for (const s of masked) {
+    const projected = projectMaskedSpan(s, text, map);
+    if (projected !== null) modelSpans.push(projected);
+  }
+  applyPolicy([...heuristic, ...modelSpans]);
+  return performance.now() - t0;
+}
+
+async function run(texts: readonly string[], warmup: number, iters: number): Promise<{ latencies: number[]; cold: number }> {
+  // Cold start: first inference pays model-graph compilation.
+  const cold = await scrubOnceTimed(texts[0]);
+  for (let i = 0; i < warmup; i++) await scrubOnceTimed(texts[i % texts.length]);
+  const latencies: number[] = [];
+  for (let i = 0; i < iters; i++) latencies.push(await scrubOnceTimed(texts[i % texts.length]));
+  return { latencies, cold };
+}
+
+// Exposed to Playwright via window.
+(globalThis as unknown as { rampartBench: unknown }).rampartBench = { init, run };

package/eval/bench/webgpu/index.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <title>Rampart WebGPU bench</title>
+  </head>
+  <body>
+    <h1>Rampart WebGPU latency harness</h1>
+    <pre id="status">loading bundle…</pre>
+    <script type="module" src="/eval/bench/webgpu/bundle.js"></script>
+  </body>
+</html>

package/eval/bench/webgpu.ts

@@ -0,0 +1,209 @@
+/**
+ * WebGPU latency harness.
+ *
+ * Bundles the shipped detection path for the browser, serves the repo over
+ * http://localhost (a WebGPU secure context), drives Playwright's bundled
+ * Chromium, runs warmup + timed inferences with the NER model on the WebGPU
+ * backend, and writes percentile latency to `runs/<out>/latency.json`.
+ *
+ *   bun eval/bench/webgpu.ts                       # WebGPU, q4
+ *   bun eval/bench/webgpu.ts --device wasm         # WASM baseline for contrast
+ *   bun eval/bench/webgpu.ts --iters 400 --headed  # more samples, visible window
+ *
+ * Inputs default to the frozen held-out slice (`eval/bench/data/heldout.jsonl`,
+ * materialised by `bun run bench:fetch`) so browser latency is measured over the
+ * same rows as the Node bench; if that file is absent it falls back to the
+ * committed `eval/public-cases.ts` chat strings. Override with `--data <path>`.
+ *
+ * Uses Playwright's bundled Chromium — no system Chrome required. The launch
+ * strips Playwright's GPU-disabling default args and forces ANGLE/Metal so the
+ * headless browser reaches the real GPU instead of the SwiftShader software
+ * adapter. Point at a system Chrome with --chrome /path/to/chrome if you prefer.
+ */
+
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { createServer } from "node:http";
+import { extname, join } from "node:path";
+import { chromium } from "playwright";
+import { PUBLIC_E2E_CASES } from "../public-cases";
+import { percentile } from "./score";
+
+const ROOT = join(import.meta.dir, "..", "..");
+
+function arg(name: string, fallback: string): string {
+  const eq = Bun.argv.find((a) => a.startsWith(`--${name}=`));
+  if (eq) return eq.slice(`--${name}=`.length);
+  const idx = Bun.argv.indexOf(`--${name}`);
+  return idx >= 0 && Bun.argv[idx + 1] && !Bun.argv[idx + 1].startsWith("--") ? Bun.argv[idx + 1] : fallback;
+}
+
+/**
+ * Load the benchmark input texts. Prefers the frozen held-out slice
+ * (`eval/bench/data/heldout.jsonl`, gitignored, materialised by `bench:fetch`)
+ * so browser latency is measured over the same rows as the Node bench; falls
+ * back to the committed `public-cases.ts` chat strings when it is absent.
+ */
+async function loadTexts(dataPath: string): Promise<{ texts: string[]; source: string }> {
+  try {
+    const raw = await readFile(dataPath, "utf8");
+    const texts = raw
+      .trim()
+      .split("\n")
+      .filter(Boolean)
+      .map((line) => JSON.parse(line).source_text as string)
+      .filter((t) => typeof t === "string" && t.length > 0);
+    if (texts.length) return { texts, source: dataPath };
+  } catch {
+    // fall through to public cases
+  }
+  return { texts: PUBLIC_E2E_CASES.map((c) => c.input), source: "public-cases" };
+}
+
+function resolveChrome(): string | undefined {
+  // Default: Playwright's bundled Chromium (it exposes WebGPU over http://localhost).
+  // Override only if you want to point at a system Chrome install.
+  const override = arg("chrome", "");
+  return override || undefined;
+}
+
+const MIME: Record<string, string> = {
+  ".html": "text/html",
+  ".js": "text/javascript",
+  ".json": "application/json",
+  ".onnx": "application/octet-stream",
+  ".txt": "text/plain",
+  ".wasm": "application/wasm",
+};
+
+async function bundleEntry(): Promise<void> {
+  const result = await Bun.build({
+    entrypoints: [join(import.meta.dir, "webgpu", "entry.ts")],
+    target: "browser",
+    format: "esm",
+    outdir: join(import.meta.dir, "webgpu"),
+    naming: "bundle.js",
+  });
+  if (!result.success) {
+    for (const log of result.logs) console.error(log);
+    throw new Error("bundle failed");
+  }
+}
+
+function serveRoot(): Promise<{ port: number; close: () => void }> {
+  const server = createServer(async (req, res) => {
+    const url = new URL(req.url ?? "/", "http://localhost");
+    const path = url.pathname === "/" ? "/eval/bench/webgpu/index.html" : url.pathname;
+    const file = Bun.file(join(ROOT, decodeURIComponent(path)));
+    if (!(await file.exists())) {
+      res.statusCode = 404;
+      res.end("not found");
+      return;
+    }
+    res.setHeader("Content-Type", MIME[extname(path)] ?? "application/octet-stream");
+    // WebGPU + threaded WASM want cross-origin isolation; harmless for single-thread.
+    res.setHeader("Cross-Origin-Opener-Policy", "same-origin");
+    res.setHeader("Cross-Origin-Embedder-Policy", "require-corp");
+    res.end(Buffer.from(await file.arrayBuffer()));
+  });
+  return new Promise((resolve) => {
+    server.listen(0, () => {
+      const addr = server.address();
+      const port = typeof addr === "object" && addr ? addr.port : 0;
+      resolve({ port, close: () => server.close() });
+    });
+  });
+}
+
+async function main(): Promise<void> {
+  const device = arg("device", "webgpu") as "webgpu" | "wasm";
+  const itersArg = arg("iters", "");
+  const warmup = Number(arg("warmup", "20"));
+  const dataPath = arg("data", join(ROOT, "eval/bench/data/heldout.jsonl"));
+  const headed = Bun.argv.includes("--headed");
+  const outDir = join(import.meta.dir, "runs", arg("out", `${device}-q4`));
+
+  const chrome = resolveChrome();
+
+  await bundleEntry();
+  const { port, close } = await serveRoot();
+  const { texts, source } = await loadTexts(dataPath);
+  // Default to one timed inference per input row (a full pass over the slice);
+  // for the small public-cases set, run a few cycles so percentiles are stable.
+  const iters = itersArg ? Number(itersArg) : Math.max(texts.length, source === "public-cases" ? 300 : texts.length);
+
+  const browser = await chromium.launch({
+    headless: !headed,
+    executablePath: chrome,
+    // Drop Playwright's GPU-disabling defaults so the bundled Chromium can reach
+    // the real GPU. Without removing --use-gl=swiftshader, WebGPU silently falls
+    // back to the SwiftShader software adapter (~800 ms/inference, useless here).
+    ignoreDefaultArgs: ["--disable-gpu", "--use-gl=swiftshader", "--disable-gpu-compositing"],
+    // --use-angle=metal is what gets the bundled headless Chromium onto Metal on
+    // macOS; on Linux the Vulkan feature flag does the equivalent.
+    args: ["--enable-unsafe-webgpu", "--use-angle=metal", "--enable-features=WebGPU,Vulkan,Metal", "--ignore-gpu-blocklist"],
+  });
+  try {
+    const page = await browser.newPage();
+    const errors: string[] = [];
+    page.on("pageerror", (e) => errors.push(e.message));
+    page.on("console", (m) => {
+      if (m.type() === "error") errors.push(m.text());
+    });
+    await page.goto(`http://localhost:${port}/`, { waitUntil: "load" });
+    await page.waitForFunction(() => "rampartBench" in globalThis, { timeout: 30_000 });
+
+    const initResult = await page.evaluate(
+      async ([base, dev]) =>
+        (globalThis as unknown as { rampartBench: { init: (o: unknown) => Promise<{ adapter: string }> } }).rampartBench.init({
+          modelBaseUrl: base,
+          device: dev,
+        }),
+      [`http://localhost:${port}`, device] as const,
+    );
+
+    const { latencies, cold } = await page.evaluate(
+      async ([ts, w, it]) =>
+        (globalThis as unknown as { rampartBench: { run: (t: string[], w: number, it: number) => Promise<{ latencies: number[]; cold: number }> } }).rampartBench.run(
+          ts as string[],
+          w as number,
+          it as number,
+        ),
+      [texts, warmup, iters] as const,
+    );
+
+    if (!latencies.length) throw new Error(`no latencies collected. page errors: ${errors.join(" | ") || "none"}`);
+
+    latencies.sort((a, b) => a - b);
+    const summary = {
+      device,
+      dtype: "q4",
+      adapter: initResult.adapter,
+      browser: browser.version(),
+      inputs: source === "public-cases" ? "public-cases" : "heldout",
+      input_texts: texts.length,
+      rows_scored: latencies.length,
+      warmup,
+      latency_ms: {
+        cold,
+        p50: percentile(latencies, 0.5),
+        p95: percentile(latencies, 0.95),
+        p99: percentile(latencies, 0.99),
+        mean: latencies.reduce((a, b) => a + b, 0) / latencies.length,
+      },
+    };
+
+    await mkdir(outDir, { recursive: true });
+    await writeFile(join(outDir, "latency.json"), JSON.stringify(summary, null, 2) + "\n");
+
+    const l = summary.latency_ms;
+    console.log(`\n${device}/q4 on ${summary.adapter}  (${summary.browser})`);
+    console.log(`inputs: ${summary.inputs} (${summary.input_texts} rows, ${summary.rows_scored} timed)`);
+    console.log(`cold ${l.cold.toFixed(1)} ms · p50 ${l.p50.toFixed(2)} ms · p95 ${l.p95.toFixed(2)} ms · p99 ${l.p99.toFixed(2)} ms · mean ${l.mean.toFixed(2)} ms`);
+    console.log(`wrote ${join(outDir, "latency.json")}`);
+  } finally {
+    await browser.close();
+    close();
+  }
+}
+
+await main();