@mailwoman/corpus 7.2.0 → 7.2.1

package/src/tokenize.ts

@@ -0,0 +1,68 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Tokenizer interface for alignment.
+ *
+ *   Two implementations live in the corpus package:
+ *
+ *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
+ *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
+ *        default for in-container alignment tests and as a fallback when no SentencePiece model is
+ *        available.
+ *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
+ *        trained on the corpus. Same interface, different splits. Locked against the corpus version
+ *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
+ *
+ *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
+ *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
+ *   spans, independent of how the tokenizer chose its splits.
+ */
+
+/** A token with its character span in the source string. */
+export interface TokenSpan {
+	/** The token text, possibly normalized (case unchanged here; tokenizers may differ). */
+	text: string
+
+	/** Inclusive start offset (UTF-16 code-unit index) in the source string. */
+	start: number
+
+	/** Exclusive end offset in the source string. `text === source.slice(start, end)`. */
+	end: number
+}
+
+/** A tokenizer that maps a string to a sequence of `TokenSpan`s. */
+export interface Tokenizer {
+	tokenize(text: string): readonly TokenSpan[]
+}
+
+/**
+ * Whitespace + punctuation tokenizer (pure JS).
+ *
+ * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}` marks, plus `'`, `-`,
+ * `_`). Everything else — whitespace, punctuation, symbols — is treated as a separator and **not** emitted as a token.
+ * The resulting spans cover the original string only on token regions; in-between regions belong to no token.
+ *
+ * This is intentionally lossy at the edges (alignment can still label every meaningful span). A future SentencePiece
+ * tokenizer will preserve all bytes via byte-fallback.
+ */
+export function whitespaceTokenizer(): Tokenizer {
+	// Maximal runs of letters/digits/marks plus the joiners common to addresses
+	// (apostrophe, hyphen, underscore). Comma/space/period etc. are not in the set.
+	const tokenRe = /[\p{L}\p{N}\p{M}'_-]+/gu
+
+	return {
+		tokenize(text: string): readonly TokenSpan[] {
+			const out: TokenSpan[] = []
+			tokenRe.lastIndex = 0
+			let m: RegExpExecArray | null
+
+			while ((m = tokenRe.exec(text))) {
+				out.push({ text: m[0], start: m.index, end: m.index + m[0].length })
+			}
+
+			return out
+		},
+	}
+}

package/src/tools/align-shard.ts

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+import { createWriteStream } from "node:fs"
+
+/**
+ * Re-emit a CANONICAL jsonl ({raw, components, country, source, ...}) as a LABELED jsonl in the CURRENT align format,
+ * by running every row through `alignRow` (corpus/src/align.ts).
+ *
+ * ## Why this exists
+ *
+ * Most synthetic shards are GENERATED on demand by a `build-*-shard` recipe (parametrized by --count), so re-emitting
+ * them in a new label format is just a re-run. A few shards are FIXED corpora with a hand/DeepSeek-authored canonical
+ * source that is never regenerated — notably `deepseek-kryptonite` (the adversarial hard-case set) and the
+ * `deepseek-translit-*` variants. Their committed parquets carry whatever label format was current when they were first
+ * built.
+ *
+ * When the corpus label format changes (the v0.5.0 char-offset triple, #519), those fixed shards must be RE-ALIGNED,
+ * not regenerated — feed the canonical source back through the same `alignRow` the from-source build uses, so the spans
+ * land in the new format with zero drift. That is exactly what this does: canonical jsonl in → labeled jsonl out, one
+ * `alignRow` per row, quarantine on miss.
+ *
+ * It is the uniform counterpart to corpus/src/tools/shard-kryptonite.ts (which couples to a base manifest and writes
+ * parquet directly). Output goes to jsonl so it joins the SAME jsonl-to-parquet path every other overlay shard uses.
+ *
+ * Usage: node scripts/align-canonical-shard.ts\
+ * --input /path/canonical-kryptonite.jsonl\
+ * --output /tmp/kryptonite-labeled.jsonl\
+ * --corpus-version 0.5.0
+ */
+import { alignRow } from "@mailwoman/corpus"
+import { TextSpliterator } from "spliterator"
+
+export interface AlignShardOptions {
+	input: string
+	output: string
+	corpusVersion: string
+}
+
+export async function alignCanonicalShard(args: AlignShardOptions): Promise<void> {
+	// Read phase only — the write path stays on createWriteStream. TextSpliterator + JSON.parse keeps the
+	// original tolerance: the `!line.trim()` guard skips blank lines and a trailing CR is valid JSON whitespace.
+	const outStream = createWriteStream(args.output, { encoding: "utf8" })
+	let labeled = 0
+	let quarantined = 0
+	const quarantineReasons: Record<string, number> = {}
+
+	for await (const line of TextSpliterator.fromAsync(args.input)) {
+		if (!line.trim()) continue
+		const canonical = JSON.parse(line) as Parameters<typeof alignRow>[0]
+		// Stamp the target corpus version so the emitted row's provenance matches the run it joins.
+		canonical.corpus_version = args.corpusVersion
+		const result = alignRow(canonical)
+
+		if (result.kind === "labeled") {
+			outStream.write(JSON.stringify(result.row) + "\n")
+			labeled++
+		} else {
+			quarantined++
+			const r = result.row.reason ?? "unknown"
+			quarantineReasons[r] = (quarantineReasons[r] ?? 0) + 1
+		}
+	}
+	await new Promise<void>((res) => outStream.end(res))
+	console.error(
+		`align-canonical-shard: ${labeled} labeled, ${quarantined} quarantined → ${args.output}\n` +
+			`  quarantine reasons: ${JSON.stringify(quarantineReasons)}`
+	)
+}

package/src/tools/audit.ts

@@ -0,0 +1,352 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman corpus audit` — per-source shard-count vs source_weight diagnostic.
+ *
+ *   Reads a corpus dir's MANIFEST.json (or scans shards directly), counts shards per source,
+ *   optionally loads a training config to pair the counts with the configured source_weights, and
+ *   reports the estimated sampled-row distribution at training time.
+ *
+ *   Would have caught v0.3.0's "NAD = 411/674 train shards × 2.0 weight = ~75% of sampled mix"
+ *   finding before the v0.3.0 retrospective surfaced it.
+ *
+ *   Emits warnings to stderr and the audit table to stdout; never throws on an empty corpus.
+ */
+
+import { existsSync, readFileSync, readdirSync } from "node:fs"
+import { basename, join } from "node:path"
+
+export interface AuditOpts {
+	corpusDir: string
+	configPath?: string
+	/**
+	 * Sample at most N shards per split when counting sources. Default 100 for speed; bump to read the full set on a slow
+	 * run. The first row of each shard determines its source — corpus-v0.2.0+ shards are 100% source-segregated, so a
+	 * one-row read is authoritative.
+	 */
+	sampleShardCount?: number
+}
+
+interface ShardStats {
+	/** Shards per source per split */
+	bySplit: Record<string, Record<string, number>>
+	/** Total shards counted (may be less than file count if sampleShardCount caps reads) */
+	totalShards: number
+	/** Total shards on disk (file count) — equals totalShards unless capped */
+	totalFiles: number
+}
+
+interface ParsedConfig {
+	sourceWeights: Record<string, number>
+}
+
+/**
+ * Try parsing a training YAML's source_weights as a minimal regex-based extract. We don't pull in a YAML lib for this
+ * script — the syntax is so small that a regex over the source_weights block is sufficient + keeps the script
+ * dep-free.
+ */
+function parseConfig(configPath: string): ParsedConfig | null {
+	if (!existsSync(configPath)) return null
+	const text = readFileSync(configPath, "utf8")
+	const lines = text.split("\n")
+	const weights: Record<string, number> = {}
+	let inBlock = false
+	let blockIndent = -1
+
+	for (const raw of lines) {
+		const sourceWeightsMatch = raw.match(/^([\t ]*)source_weights:\s*$/)
+
+		if (sourceWeightsMatch) {
+			inBlock = true
+			blockIndent = sourceWeightsMatch[1]!.length
+			continue
+		}
+
+		if (!inBlock) continue
+
+		// Skip blank lines and comments.
+		if (/^[\t ]*(#|$)/.test(raw)) continue
+		// Lines indented MORE than `source_weights:` are entries; lines with ≤ indent end the block.
+		const indent = raw.match(/^[\t ]*/)![0].length
+
+		if (indent <= blockIndent) {
+			inBlock = false
+			continue
+		}
+		const m = raw.match(/^[\t ]+([\w-]+):\s*([\d.]+)/)
+
+		if (m) {
+			weights[m[1]!] = parseFloat(m[2]!)
+		}
+	}
+
+	return { sourceWeights: weights }
+}
+
+/**
+ * Scan a corpus directory's shards (typically under <corpus_dir>/train, /val, /test) and count shards per source per
+ * split.
+ */
+function scanShards(corpusDir: string, sampleCount: number): ShardStats {
+	const stats: ShardStats = { bySplit: {}, totalShards: 0, totalFiles: 0 }
+
+	for (const split of ["train", "val", "test"]) {
+		const splitDir = join(corpusDir, split)
+
+		if (!existsSync(splitDir)) continue
+		const files = readdirSync(splitDir)
+			.filter((f) => f.endsWith(".parquet"))
+			.sort()
+		stats.totalFiles += files.length
+		const sampleEvery = Math.max(1, Math.floor(files.length / sampleCount))
+		const sampled = files.filter((_, i) => i % sampleEvery === 0).slice(0, sampleCount)
+		const splitMap: Record<string, number> = {}
+
+		// We can't read parquet without a dep, so we infer source from filenames where possible.
+		// The corpus build typically writes deterministically by source — fall back to "<unknown>"
+		// when filename gives no hint. For accurate per-source counts on real corpora, the
+		// MANIFEST.json route below is preferred.
+		for (const f of sampled) {
+			const inferred = inferSourceFromFilename(f)
+			splitMap[inferred] = (splitMap[inferred] ?? 0) + 1
+		}
+		// Scale to estimated full-shard counts.
+		const scale = files.length / Math.max(sampled.length, 1)
+
+		for (const k of Object.keys(splitMap)) {
+			splitMap[k] = Math.round(splitMap[k]! * scale)
+		}
+		stats.bySplit[split] = splitMap
+		stats.totalShards += files.length
+	}
+
+	return stats
+}
+
+function inferSourceFromFilename(filename: string): string {
+	// Many corpus builds write part-<source>-<n>.parquet or part-<n>.parquet. The latter (current
+	// build at corpus-v0.3.0) gives no source signal in the filename — see manifestScan() for the
+	// authoritative path. Return "<unknown>" so the caller flags this case.
+	const m = basename(filename).match(/part-([\w-]+)-\d+\.parquet$/)
+
+	if (m && m[1] !== undefined) return m[1]
+
+	return "<unknown>"
+}
+
+/**
+ * Known source name prefixes. Corpus-v0.3.0 uses these as `source_id` prefixes; matching against the longest prefix
+ * that fits a given `first_source_id` recovers the canonical source name.
+ *
+ * Order matters: longer prefixes must be tried first so `usgov-nad-...` matches `usgov-nad` rather than `usgov`. Sorted
+ * descending by length at use site.
+ */
+const KNOWN_SOURCE_PREFIXES: ReadonlyArray<string> = [
+	"wof-admin",
+	"wof-postalcode",
+	"ban",
+	"tiger",
+	"usgov-nad",
+	"usgov-nppes",
+	"usgov-hrsa-fqhc",
+	"usgov-imls-pls",
+	"state-ia-contractors",
+	"state-tx-notaries",
+	"state-ny-notaries",
+	"openaddresses",
+	// Synthetic adversarial sources (corpus-v0.4.0+, Thread B).
+	"deepseek-kryptonite",
+	"deepseek-translit-cyrl",
+	"deepseek-translit-jpan",
+	"deepseek-translit-hans",
+	"deepseek-translit-hang",
+	"deepseek-translit-armn",
+]
+
+/** Extract the source-name prefix from a `first_source_id` value. */
+function sourceFromID(sourceID: string, knownPrefixes: readonly string[]): string {
+	// Sort longest-first so usgov-nad beats usgov, wof-admin beats wof.
+	const sorted = [...knownPrefixes].sort((a, b) => b.length - a.length)
+
+	for (const prefix of sorted) {
+		if (sourceID.startsWith(prefix + "-") || sourceID === prefix) return prefix
+	}
+
+	return "<unknown>"
+}
+
+/**
+ * Prefer reading MANIFEST.json when present — uses each shard's `first_source_id` + prefix matching to recover the
+ * source name. Falls back to scanShards when MANIFEST is absent.
+ *
+ * NOTE: corpus-v0.3.0 shards can mix sources (see `last_source_id` differing from `first_source_id`). The first-row
+ * source is an approximation; reading the parquet's full source column would be authoritative but requires a parquet
+ * dep. For audit purposes the first-row approximation is accurate within ~5% for the corpus-v0.3.0 shape (most shards
+ * are >95% one source).
+ */
+function manifestScan(corpusDir: string, knownPrefixes: readonly string[]): ShardStats | null {
+	const manifestPath = join(corpusDir, "MANIFEST.json")
+
+	if (!existsSync(manifestPath)) return null
+	const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
+		shards?: Array<{ split: string; source?: string | null; first_source_id?: string | null }>
+	}
+
+	if (!Array.isArray(manifest.shards)) return null
+	const bySplit: Record<string, Record<string, number>> = {}
+
+	for (const shard of manifest.shards) {
+		const split = shard.split
+		const src = shard.source ?? sourceFromID(shard.first_source_id ?? "", knownPrefixes)
+		bySplit[split] ??= {}
+		bySplit[split][src] = (bySplit[split][src] ?? 0) + 1
+	}
+	const total = Object.values(bySplit).reduce((sum, m) => sum + Object.values(m).reduce((a, b) => a + b, 0), 0)
+
+	return { bySplit, totalShards: total, totalFiles: total }
+}
+
+interface AuditRow {
+	source: string
+	shards: number
+	shardPct: number
+	weight: number | "—"
+	effectiveSamplePct: number | "—"
+	overweightFactor?: number
+}
+
+function buildAuditRows(stats: Record<string, number>, weights: Record<string, number>): AuditRow[] {
+	const totalShards = Object.values(stats).reduce((a, b) => a + b, 0)
+	const allSources = new Set([...Object.keys(stats), ...Object.keys(weights)])
+	const rows: AuditRow[] = []
+	// Compute effective sample weight: shard_count × source_weight. Sources with no weight get the
+	// "—" marker (loader skips them).
+	const sampleWeights: Array<[string, number]> = []
+
+	for (const src of allSources) {
+		const shards = stats[src] ?? 0
+		const weight = weights[src]
+		const effective = weight !== undefined ? shards * weight : 0
+		sampleWeights.push([src, effective])
+	}
+	const totalSampleWeight = sampleWeights.reduce((a, [, w]) => a + w, 0)
+
+	for (const src of allSources) {
+		const shards = stats[src] ?? 0
+		const weight = weights[src] ?? "—"
+		const effective = typeof weight === "number" ? (shards * weight) / Math.max(totalSampleWeight, 1) : "—"
+		rows.push({
+			source: src,
+			shards,
+			shardPct: totalShards > 0 ? shards / totalShards : 0,
+			weight,
+			effectiveSamplePct: typeof effective === "number" ? effective : "—",
+		})
+	}
+	// Flag the dominator: empirically calibrated against the v0.3.0 → v0.4.0 retrospective.
+	// v0.3.0 had usgov-nad at 52% effective sample (1.9× ban); the resulting label-space dilution
+	// was responsible for the coarse-F1 regression. So flag a source as "concentration warning"
+	// when it's above 40% effective sample OR more than 1.5× the next-highest.
+	const numeric = rows.filter((r) => typeof r.effectiveSamplePct === "number") as Array<
+		AuditRow & { effectiveSamplePct: number }
+	>
+	numeric.sort((a, b) => b.effectiveSamplePct - a.effectiveSamplePct)
+
+	if (numeric.length >= 1) {
+		const top = numeric[0]!
+		const next = numeric[1]?.effectiveSamplePct ?? 0
+
+		if (top.effectiveSamplePct > 0.4 || (next > 0 && top.effectiveSamplePct / next > 1.5)) {
+			top.overweightFactor = next > 0 ? top.effectiveSamplePct / next : Infinity
+		}
+	}
+	rows.sort((a, b) => b.shards - a.shards)
+
+	return rows
+}
+
+function formatPct(v: number | "—"): string {
+	if (v === "—") return "—"
+
+	return `${(v * 100).toFixed(1)}%`
+}
+
+function printReport(corpusDir: string, configPath: string | undefined, stats: ShardStats, rows: AuditRow[]): void {
+	console.log(`\nCorpus audit — ${corpusDir}`)
+
+	if (configPath) {
+		console.log(`Config:        ${configPath}`)
+	}
+	console.log(
+		`Total shards:  ${stats.totalShards}${stats.totalFiles !== stats.totalShards ? ` (${stats.totalFiles} files on disk)` : ""}`
+	)
+	console.log("")
+	const trainStats = stats.bySplit["train"]
+
+	if (trainStats) {
+		const total = Object.values(trainStats).reduce((a, b) => a + b, 0)
+		console.log(`Train split: ${total} shards`)
+		console.log("")
+		const headers = ["source", "shards", "shard %", "weight", "eff. sample %"]
+		const widths = [22, 8, 10, 8, 14]
+		const fmtRow = (cells: string[]) => cells.map((c, i) => c.padEnd(widths[i]!)).join("  ")
+		console.log(fmtRow(headers))
+		console.log(fmtRow(widths.map((w) => "─".repeat(w))))
+
+		for (const row of rows) {
+			console.log(
+				fmtRow([
+					row.source,
+					String(row.shards),
+					formatPct(row.shardPct),
+					typeof row.weight === "number" ? row.weight.toFixed(2) : "—",
+					formatPct(row.effectiveSamplePct),
+				])
+			)
+		}
+		console.log("")
+		const dominator = rows.find((r) => r.overweightFactor !== undefined)
+
+		if (dominator) {
+			const factor = dominator.overweightFactor
+			const factorStr = factor === Infinity ? "∞" : factor?.toFixed(1)
+			console.error(
+				`⚠ Concentration: ${dominator.source} would sample ${formatPct(dominator.effectiveSamplePct)} ` +
+					`of training rows (${factorStr}× the next-highest). ` +
+					`Past lesson: v0.3.0's NAD at ~52% caused the 21-label coarse regression. ` +
+					`Consider lowering this source's weight or boosting others.`
+			)
+		} else {
+			console.log("✓ No single-source concentration (top source < 40% effective sample AND < 1.5× next).")
+		}
+		const missingWeights = rows.filter((r) => r.weight === "—" && r.shards > 0)
+
+		if (missingWeights.length > 0 && configPath) {
+			console.error(
+				`⚠ Sources present in corpus but absent from config.source_weights ` +
+					`(loader will skip them): ${missingWeights.map((r) => r.source).join(", ")}`
+			)
+		}
+		const orphanWeights = rows.filter((r) => typeof r.weight === "number" && r.shards === 0)
+
+		if (orphanWeights.length > 0) {
+			console.error(
+				`⚠ Sources weighted in config but no shards found in corpus ` +
+					`(no-op weights): ${orphanWeights.map((r) => r.source).join(", ")}`
+			)
+		}
+	}
+}
+
+export function audit(opts: AuditOpts): void {
+	const config = opts.configPath ? parseConfig(opts.configPath) : null
+	// Compose the known-prefix list from both the hardcoded set and any extra names in the config
+	// (forward-compat for future adapters added before this file is updated).
+	const prefixes = [...new Set([...KNOWN_SOURCE_PREFIXES, ...Object.keys(config?.sourceWeights ?? {})])]
+	const stats = manifestScan(opts.corpusDir, prefixes) ?? scanShards(opts.corpusDir, opts.sampleShardCount ?? 100)
+	const trainStats = stats.bySplit["train"] ?? {}
+	const rows = buildAuditRows(trainStats, config?.sourceWeights ?? {})
+	printReport(opts.corpusDir, opts.configPath, stats, rows)
+}

package/src/tools/corpus-stats.ts

@@ -0,0 +1,183 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Pre-compute corpus-wide token + bigram label distributions for the corpus linter.
+ *
+ *   Reads one or more Parquet shards, builds per-(token, label) and per-(bigram, label-bigram)
+ *   histograms, and serializes them as JSON. The output file is consumed by `lint-corpus-shard.ts`
+ *   as the baseline against which a new shard is compared.
+ *
+ *   Stats are cheap to compute (~5–30s per 100K rows) but expensive enough that we cache them between
+ *   linter invocations. Re-run this script whenever the corpus changes substantially (a new
+ *   mainline shard added, a source-pool re-weighted, etc.).
+ *
+ *   Output schema:
+ *
+ *   ```ts
+ *   interface CorpusStats {
+ *     row_count: number
+ *     shard_paths: string[]
+ *     tokens: { [token: string]: { [label: string]: number } }
+ *     bigrams: { [token_bigram: string]: { [label_bigram: string]: number } }
+ *     // token_bigram = "tok1tok2" (US sep), label_bigram = "lab1lab2"
+ *     // For memory: only keep bigrams with count >= MIN_BIGRAM_COUNT (2).
+ *   }
+ * ```
+ *
+ *   Usage: node scripts/build-corpus-stats.ts\
+ *   --shards <glob-pattern-or-dir>\
+ *   --output <stats.json>
+ *
+ *   For a quick local-corpus baseline (limited but useful for linter testing): node
+ *   scripts/build-corpus-stats.ts\
+ *   --shards /mnt/playpen/mailwoman-data/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
+ *   --output /tmp/corpus-stats-local.json
+ */
+
+import { execSync } from "node:child_process"
+import { readdirSync, statSync, writeFileSync } from "node:fs"
+import { join } from "node:path"
+
+const SEP = ""
+const MIN_BIGRAM_COUNT = 2
+
+export interface CorpusStatsOptions {
+	shardsArg: string
+	outputPath: string
+	limitPerShard?: number
+}
+
+function discoverShards(shardsArg: string): string[] {
+	const stat = statSync(shardsArg)
+
+	if (stat.isDirectory()) {
+		return readdirSync(shardsArg)
+			.filter((f) => f.endsWith(".parquet"))
+			.map((f) => join(shardsArg, f))
+	}
+
+	if (stat.isFile() && shardsArg.endsWith(".parquet")) return [shardsArg]
+
+	// Otherwise treat as a literal path list (one per line if it's stdin-friendly).
+	return [shardsArg]
+}
+
+/**
+ * Use a Python subprocess to read parquet (pyarrow is heavier than parquet-wasm but already on the path here, and we
+ * have nothing in the JS ecosystem that reads parquet cleanly at this scale). Emits one JSON object per line: `{tokens:
+ * [...], labels: [...]}`.
+ */
+function streamShardRows(shardPath: string, limit?: number): Array<{ tokens: string[]; labels: string[] }> {
+	// Pipe the python script via stdin instead of `-c` to preserve newlines verbatim
+	// (JSON-encoding the script for -c collapses real newlines into literal `\n`).
+	const py = `
+import pyarrow.parquet as pq
+import json, sys
+t = pq.read_table(${JSON.stringify(shardPath)}, columns=['tokens', 'labels'])
+tokens_col = t['tokens'].to_pylist()
+labels_col = t['labels'].to_pylist()
+n = min(len(tokens_col), ${limit ?? "len(tokens_col)"})
+for i in range(n):
+    sys.stdout.write(json.dumps({"tokens": tokens_col[i], "labels": labels_col[i]}) + "\\n")
+`
+	const buf = execSync(`python3`, { input: py, maxBuffer: 1024 * 1024 * 1024 })
+	const rows: Array<{ tokens: string[]; labels: string[] }> = []
+
+	for (const line of buf.toString("utf8").split("\n")) {
+		if (!line) continue
+		rows.push(JSON.parse(line))
+	}
+
+	return rows
+}
+
+export function buildCorpusStats(args: CorpusStatsOptions): void {
+	const shardPaths = discoverShards(args.shardsArg)
+	console.error(`Discovered ${shardPaths.length} parquet shard(s)`)
+
+	const tokenStats = new Map<string, Map<string, number>>()
+	const bigramStats = new Map<string, Map<string, number>>()
+	let totalRows = 0
+
+	for (const path of shardPaths) {
+		console.error(`Reading ${path}...`)
+		const rows = streamShardRows(path, args.limitPerShard)
+		totalRows += rows.length
+
+		for (const row of rows) {
+			const { tokens, labels } = row
+
+			if (tokens.length !== labels.length) continue
+
+			// skip malformed
+			for (let i = 0; i < tokens.length; i++) {
+				const tk = tokens[i]!
+				const lb = labels[i]!
+				let labelMap = tokenStats.get(tk)
+
+				if (!labelMap) {
+					labelMap = new Map()
+					tokenStats.set(tk, labelMap)
+				}
+				labelMap.set(lb, (labelMap.get(lb) ?? 0) + 1)
+
+				if (i + 1 < tokens.length) {
+					const bigramKey = tk + SEP + tokens[i + 1]!
+					const bigramLabel = lb + SEP + labels[i + 1]!
+					let bMap = bigramStats.get(bigramKey)
+
+					if (!bMap) {
+						bMap = new Map()
+						bigramStats.set(bigramKey, bMap)
+					}
+					bMap.set(bigramLabel, (bMap.get(bigramLabel) ?? 0) + 1)
+				}
+			}
+		}
+		console.error(
+			`  ${rows.length} rows; running totals: ${tokenStats.size} unique tokens, ${bigramStats.size} unique bigrams`
+		)
+	}
+
+	// Prune bigrams below MIN_BIGRAM_COUNT to keep the output file size sane. Token stats
+	// stay complete — they're cheap and we need accuracy at the long tail for label-vacuum
+	// detection.
+	let prunedBigrams = 0
+
+	for (const [k, labelMap] of bigramStats) {
+		let total = 0
+
+		for (const v of labelMap.values()) {
+			total += v
+		}
+
+		if (total < MIN_BIGRAM_COUNT) {
+			bigramStats.delete(k)
+			prunedBigrams++
+		}
+	}
+	console.error(`Pruned ${prunedBigrams} singleton bigrams; ${bigramStats.size} remain`)
+
+	const out = {
+		row_count: totalRows,
+		shard_paths: shardPaths,
+		tokens: {} as Record<string, Record<string, number>>,
+		bigrams: {} as Record<string, Record<string, number>>,
+	}
+
+	for (const [tk, labelMap] of tokenStats) {
+		out.tokens[tk] = Object.fromEntries(labelMap)
+	}
+
+	for (const [k, labelMap] of bigramStats) {
+		out.bigrams[k] = Object.fromEntries(labelMap)
+	}
+
+	writeFileSync(args.outputPath, JSON.stringify(out))
+	const sizeMB = (Buffer.byteLength(JSON.stringify(out)) / 1024 / 1024).toFixed(1)
+	console.error(
+		`Wrote ${args.outputPath} (${sizeMB} MB) — ${totalRows} rows, ${tokenStats.size} tokens, ${bigramStats.size} bigrams`
+	)
+}