@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/tools/audit.ts

@@ -3,56 +3,66 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `mailwoman corpus audit` — per-source slice-count vs source_weight diagnostic.
+ *   `mailwoman corpus audit` — per-source parquet-file count vs source_weight diagnostic.
  *
- *   Reads a corpus dir's MANIFEST.json (or scans slices directly), counts slices per source,
- *   optionally loads a training config to pair the counts with the configured source_weights, and
- *   reports the estimated sampled-row distribution at training time.
+ *   Reads a corpus dir's manifest.json (or scans the parquet files directly), counts files per
+ *   source, optionally loads a training config to pair the counts with the configured
+ *   source_weights. It reports the estimated sampled-row distribution at training time.
  *
- *   Would have caught v0.3.0's "NAD = 411/674 train slices × 2.0 weight = ~75% of sampled mix"
+ *   Would have caught v0.3.0's "NAD = 411/674 train files × 2.0 weight = ~75% of sampled mix"
  *   finding before the v0.3.0 retrospective surfaced it.
  *
- *   Emits warnings to stderr and the audit table to stdout; never throws on an empty corpus.
+ *   Emits warnings to stderr and the audit table to stdout. never throws on an empty corpus.
  */
 
-import { pathExists, readLocalBuffer, readLocalJSONFile } from "@mailwoman/core/fs/readers"
-import { basename, join, type PathBuilderLike } from "path-ts"
-import { TextSpliterator } from "spliterator"
+import { pathExists, readLocalJSONFile, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { basename, type PathBuilderLike, resolvePathBuilder } from "path-ts"
 import { Globerator } from "spliterator/node/fs"
 
+import { readConfigView } from "#source-register/effective-manifest"
+
 /**
- * Share of a slice one source may hold before the mix is flagged as dominated by it.
+ * Share of the sampled mix one source may hold before the mix is flagged as dominated by it.
  */
 const DOMINANT_SOURCE_SHARE = 0.4
 
 /**
- * How far the leading source may outweigh the runner-up before the mix is flagged. Catches the case where no single
- * source clears {@link DOMINANT_SOURCE_SHARE} but the distribution is still lopsided.
+ * How far the leading source may outweigh the runner-up before the mix is flagged.
+ *
+ * Catches the case where no single source clears {@link DOMINANT_SOURCE_SHARE}
+ * but the distribution is still lopsided.
  */
 const MAX_TOP_TO_RUNNER_UP_RATIO = 1.5
 
+/**
+ * Options for {@linkcode audit}.
+ */
 export interface AuditOpts {
 	corpusDir: PathBuilderLike
-	configPath?: string
+	configPath?: PathBuilderLike
 	/**
-	 * Sample at most N slices per split when counting sources. Default 100 for speed; bump to read the full set on a slow
-	 * run. The first row of each slice determines its source — corpus-v0.2.0+ slices are 100% source-segregated, so a
-	 * one-row read is authoritative.
+	 * Sample at most N parquet files per split when counting sources.
+	 *
+	 * Default 100 for speed.
+	 * Bump to read the full set on a slow run.
+	 *
+	 * The first row of each file determines its source — corpus-v0.2.0+ files are 100%
+	 * source-segregated, so a one-row read is authoritative.
 	 */
-	sampleSliceCount?: number
+	sampleFileCount?: number
 }
 
-interface SliceStats {
+interface FileCountStats {
 	/**
-	 * Slices per source per split
+	 * Parquet files per source per split
 	 */
 	bySplit: Record<string, Record<string, number>>
 	/**
-	 * Total slices counted (may be less than file count if sampleSliceCount caps reads)
+	 * Total parquet files counted (may be less than file count if `sampleFileCount` caps reads)
 	 */
-	totalSlices: number
+	totalCounted: number
 	/**
-	 * Total slices on disk (file count) — equals totalSlices unless capped
+	 * Total parquet files on disk — equals totalCounted unless capped
 	 */
 	totalFiles: number
 }
@@ -62,62 +72,32 @@ interface ParsedConfig {
 }
 
 /**
- * Try parsing a training YAML's source_weights as a minimal regex-based extract. We don't pull in a YAML lib for this
- * script — the syntax is so small that a regex over the source_weights block is sufficient + keeps the script
- * dep-free.
+ * Read a training config's `source_weights`, or `null` when the file is absent.
+ *
+ * The scan lives in `readConfigView`, which the effective-manifest reader also calls,
+ * so one reading of the hand-written config subset serves both.
+ * This wrapper keeps the absent-file case a `null` rather than a throw, because
+ * `corpus audit` reports a corpus without a config rather than refusing to audit it.
  */
-async function parseConfig(configPath: string): Promise<ParsedConfig | null> {
+async function parseConfig(configPath: PathBuilderLike): Promise<ParsedConfig | null> {
 	if (!(await pathExists(configPath))) return null
-	const weights: Record<string, number> = {}
-	let inBlock = false
-	let blockIndent = -1
-
-	for (const raw of TextSpliterator.from(await readLocalBuffer(configPath))) {
-		const sourceWeightsMatch = raw.match(/^([\t ]*)source_weights:\s*$/)
-
-		if (sourceWeightsMatch) {
-			inBlock = true
-			blockIndent = sourceWeightsMatch[1]!.length
-
-			continue
-		}
-
-		if (!inBlock) continue
-
-		// Skip blank lines and comments.
-		if (/^[\t ]*(#|$)/.test(raw)) continue
-		// Lines indented MORE than `source_weights:` are entries; lines with ≤ indent end the block.
-		const indent = raw.match(/^[\t ]*/)![0].length
-
-		if (indent <= blockIndent) {
-			inBlock = false
 
-			continue
-		}
-
-		const m = raw.match(/^[\t ]+([\w-]+):\s*([\d.]+)/)
-
-		if (m) {
-			weights[m[1]!] = Number.parseFloat(m[2]!)
-		}
-	}
-
-	return { sourceWeights: weights }
+	return { sourceWeights: readConfigView(await readLocalTextFile(configPath)).sourceWeights }
 }
 
 /**
- * Scan a corpus directory's slices (typically under <corpus_dir>/train, /val, /test) and count slices per source per
- * split.
+ * Scan a corpus directory's parquet files (typically under <corpus_dir>/train, /val, /test)
+ * and count files per source per split.
  */
-async function scanSlices(corpusDir: PathBuilderLike, sampleCount: number): Promise<SliceStats> {
-	const stats: SliceStats = { bySplit: {}, totalSlices: 0, totalFiles: 0 }
+async function scanParquetFiles(corpusDir: PathBuilderLike, sampleCount: number): Promise<FileCountStats> {
+	const stats: FileCountStats = { bySplit: {}, totalCounted: 0, totalFiles: 0 }
 
 	for (const split of ["train", "val", "test"]) {
-		const splitDir = join(corpusDir, split)
+		const splitDir = resolvePathBuilder(corpusDir, split)
 
 		if (!(await pathExists(splitDir))) continue
 
-		const files = await Globerator.files("parquet", { cwd: splitDir, absolute: false }).toSorted()
+		const files = await Globerator.files("parquet", { cwd: splitDir, absolute: false, recursive: false }).toSorted()
 
 		stats.totalFiles += files.length
 		const sampleEvery = Math.max(1, Math.floor(files.length / sampleCount))
@@ -125,15 +105,15 @@ async function scanSlices(corpusDir: PathBuilderLike, sampleCount: number): Prom
 		const splitMap: Record<string, number> = {}
 
 		// We can't read parquet without a dep, so we infer source from filenames where possible.
-		// The corpus build typically writes deterministically by source — fall back to "<unknown>"
-		// when filename gives no hint. For accurate per-source counts on real corpora, the
-		// MANIFEST.json route below is preferred.
+		// The corpus build typically writes deterministically by source —
+		// fall back to "<unknown>" when filename gives no hint.
+		// For accurate per-source counts on real corpora, the manifest.json route below is preferred.
 		for (const f of sampled) {
 			const inferred = inferSourceFromFilename(f)
 			splitMap[inferred] = (splitMap[inferred] ?? 0) + 1
 		}
 
-		// Scale to estimated full-slice counts.
+		// Scale to estimated full-file counts.
 		const scale = files.length / Math.max(sampled.length, 1)
 
 		for (const k of Object.keys(splitMap)) {
@@ -141,16 +121,17 @@ async function scanSlices(corpusDir: PathBuilderLike, sampleCount: number): Prom
 		}
 
 		stats.bySplit[split] = splitMap
-		stats.totalSlices += files.length
+		stats.totalCounted += files.length
 	}
 
 	return stats
 }
 
 function inferSourceFromFilename(filename: string): string {
-	// Many corpus builds write part-<source>-<n>.parquet or part-<n>.parquet. The latter (current
-	// build at corpus-v0.3.0) gives no source signal in the filename — see manifestScan() for the
-	// authoritative path. Return "<unknown>" so the caller flags this case.
+	// Many corpus builds write part-<source>-<n>.parquet or part-<n>.parquet.
+	// The latter (current build at corpus-v0.3.0) gives no source signal in the filename.
+	// See manifestScan() for the authoritative path.
+	// Return "<unknown>" so the caller flags this case.
 	const m = basename(filename).match(/part-([\w-]+)-\d+\.parquet$/)
 
 	if (m && m[1] !== undefined) return m[1]
@@ -159,11 +140,15 @@ function inferSourceFromFilename(filename: string): string {
 }
 
 /**
- * Known source name prefixes. Corpus-v0.3.0 uses these as `source_id` prefixes; matching against the longest prefix
- * that fits a given `first_source_id` recovers the canonical source name.
+ * Known source name prefixes.
  *
- * Order matters: longer prefixes must be tried first so `usgov-nad-...` matches `usgov-nad` rather than `usgov`. Sorted
- * descending by length at use site.
+ * Corpus-v0.3.0 uses these as `source_id` prefixes.
+ * The parser matches the longest prefix that fits a given `first_source_id`
+ * recovers the canonical source name.
+ *
+ * Order matters: longer prefixes must be tried first so `usgov-nad-...` matches
+ * `usgov-nad` rather than `usgov`.
+ * Sorted descending by length at use site.
  */
 const KNOWN_SOURCE_PREFIXES: ReadonlyArray<string> = [
 	"wof-admin",
@@ -202,16 +187,24 @@ function sourceFromID(sourceID: string, knownPrefixes: readonly string[]): strin
 }
 
 /**
- * Prefer reading MANIFEST.json when present — uses each slice's `first_source_id` + prefix matching to recover the
- * source name. Falls back to scanSlices when MANIFEST is absent.
+ * Prefer reading manifest.json when present — uses each file's `first_source_id` +
+ * prefix matching to recover the source name.
+ *
+ * Falls back to scanParquetFiles when manifest is absent.
+ *
+ * Note: corpus-v0.3.0 files can mix sources (see `last_source_id` differing from `first_source_id`).
+ * The first-row source is an approximation.
  *
- * NOTE: corpus-v0.3.0 slices can mix sources (see `last_source_id` differing from `first_source_id`). The first-row
- * source is an approximation; reading the parquet's full source column would be authoritative but requires a parquet
- * dep. For audit purposes the first-row approximation is accurate within ~5% for the corpus-v0.3.0 shape (most slices
- * are >95% one source).
+ * A full read of the parquet source column would be authoritative but requires a parquet dependency.
+ *
+ * For audit purposes the first-row approximation is accurate within ~5% for the
+ * corpus-v0.3.0 shape (most files are >95% one source).
  */
-async function manifestScan(corpusDir: PathBuilderLike, knownPrefixes: readonly string[]): Promise<SliceStats | null> {
-	const manifestPath = join(corpusDir, "MANIFEST.json")
+async function manifestScan(
+	corpusDir: PathBuilderLike,
+	knownPrefixes: readonly string[]
+): Promise<FileCountStats | null> {
+	const manifestPath = resolvePathBuilder(corpusDir, "MANIFEST.json")
 
 	if (!(await pathExists(manifestPath))) return null
 
@@ -222,62 +215,63 @@ async function manifestScan(corpusDir: PathBuilderLike, knownPrefixes: readonly
 	if (!Array.isArray(manifest.slices)) return null
 	const bySplit: Record<string, Record<string, number>> = {}
 
-	for (const slice of manifest.slices) {
-		const split = slice.split
-		const src = slice.source ?? sourceFromID(slice.first_source_id ?? "", knownPrefixes)
+	for (const file of manifest.slices) {
+		const split = file.split
+		const src = file.source ?? sourceFromID(file.first_source_id ?? "", knownPrefixes)
 		bySplit[split] ??= {}
 		bySplit[split][src] = (bySplit[split][src] ?? 0) + 1
 	}
 
 	const total = Object.values(bySplit).reduce((sum, m) => sum + Object.values(m).reduce((a, b) => a + b, 0), 0)
 
-	return { bySplit, totalSlices: total, totalFiles: total }
+	return { bySplit, totalCounted: total, totalFiles: total }
 }
 
 interface AuditRow {
 	source: string
-	slices: number
-	slicePct: number
+	files: number
+	filePct: number
 	weight: number | "—"
 	effectiveSamplePct: number | "—"
 	overweightFactor?: number
 }
 
 function buildAuditRows(stats: Record<string, number>, weights: Record<string, number>): AuditRow[] {
-	const totalSlices = Object.values(stats).reduce((a, b) => a + b, 0)
+	const totalCounted = Object.values(stats).reduce((a, b) => a + b, 0)
 	const allSources = new Set([...Object.keys(stats), ...Object.keys(weights)])
 	const rows: AuditRow[] = []
-	// Compute effective sample weight: slice_count × source_weight. Sources with no weight get the
-	// "—" marker (loader skips them).
+	// Compute effective sample weight: file_count × source_weight.
+	// Sources with no weight get the "—" marker (loader skips them).
 	const sampleWeights: Array<[string, number]> = []
 
 	for (const src of allSources) {
-		const slices = stats[src] ?? 0
+		const files = stats[src] ?? 0
 		const weight = weights[src]
-		const effective = weight !== undefined ? slices * weight : 0
+		const effective = weight !== undefined ? files * weight : 0
 		sampleWeights.push([src, effective])
 	}
 
 	const totalSampleWeight = sampleWeights.reduce((a, [, w]) => a + w, 0)
 
 	for (const src of allSources) {
-		const slices = stats[src] ?? 0
+		const files = stats[src] ?? 0
 		const weight = weights[src] ?? "—"
-		const effective = typeof weight === "number" ? (slices * weight) / Math.max(totalSampleWeight, 1) : "—"
+		const effective = typeof weight === "number" ? (files * weight) / Math.max(totalSampleWeight, 1) : "—"
 
 		rows.push({
 			source: src,
-			slices,
-			slicePct: totalSlices > 0 ? slices / totalSlices : 0,
+			files,
+			filePct: totalCounted > 0 ? files / totalCounted : 0,
 			weight,
 			effectiveSamplePct: typeof effective === "number" ? effective : "—",
 		})
 	}
 
 	// Flag the dominator: empirically calibrated against the v0.3.0 → v0.4.0 retrospective.
-	// v0.3.0 had usgov-nad at 52% effective sample (1.9× ban); the resulting label-space dilution
-	// was responsible for the coarse-F1 regression. So flag a source as "concentration warning"
-	// when it's above 40% effective sample OR more than 1.5× the next-highest.
+	// v0.3.0 had usgov-nad at 52% effective sample (1.9× ban); the resulting label-space
+	// dilution was responsible for the coarse-F1 regression.
+	// So flag a source as "concentration warning" when it's above 40% effective sample
+	// or more than 1.5× the next-highest.
 	const numeric = rows.filter((r) => typeof r.effectiveSamplePct === "number") as Array<
 		AuditRow & { effectiveSamplePct: number }
 	>
@@ -296,7 +290,7 @@ function buildAuditRows(stats: Record<string, number>, weights: Record<string, n
 		}
 	}
 
-	rows.sort((a, b) => b.slices - a.slices)
+	rows.sort((a, b) => b.files - a.files)
 
 	return rows
 }
@@ -309,8 +303,8 @@ function formatPct(v: number | "—"): string {
 
 function printReport(
 	corpusDir: PathBuilderLike,
-	configPath: string | undefined,
-	stats: SliceStats,
+	configPath: PathBuilderLike | undefined,
+	stats: FileCountStats,
 	rows: AuditRow[]
 ): void {
 	console.log(`\nCorpus audit — ${corpusDir}`)
@@ -320,7 +314,7 @@ function printReport(
 	}
 
 	console.log(
-		`Total slices:  ${stats.totalSlices}${stats.totalFiles !== stats.totalSlices ? ` (${stats.totalFiles} files on disk)` : ""}`
+		`Total files:   ${stats.totalCounted}${stats.totalFiles !== stats.totalCounted ? ` (${stats.totalFiles} files on disk)` : ""}`
 	)
 	console.log("")
 
@@ -329,10 +323,10 @@ function printReport(
 	if (trainStats) {
 		const total = Object.values(trainStats).reduce((a, b) => a + b, 0)
 
-		console.log(`Train split: ${total} slices`)
+		console.log(`Train split: ${total} files`)
 		console.log("")
 
-		const headers = ["source", "slices", "slice %", "weight", "eff. sample %"]
+		const headers = ["source", "files", "file %", "weight", "eff. sample %"]
 		const widths = [22, 8, 10, 8, 14]
 		const fmtRow = (cells: string[]) => cells.map((c, i) => c.padEnd(widths[i]!)).join("  ")
 
@@ -343,8 +337,8 @@ function printReport(
 			console.log(
 				fmtRow([
 					row.source,
-					String(row.slices),
-					formatPct(row.slicePct),
+					String(row.files),
+					formatPct(row.filePct),
 					typeof row.weight === "number" ? row.weight.toFixed(2) : "—",
 					formatPct(row.effectiveSamplePct),
 				])
@@ -369,7 +363,7 @@ function printReport(
 			console.log("✓ No single-source concentration (top source < 40% effective sample AND < 1.5× next).")
 		}
 
-		const missingWeights = rows.filter((r) => r.weight === "—" && r.slices > 0)
+		const missingWeights = rows.filter((r) => r.weight === "—" && r.files > 0)
 
 		if (missingWeights.length && configPath) {
 			console.error(
@@ -378,11 +372,11 @@ function printReport(
 			)
 		}
 
-		const orphanWeights = rows.filter((r) => typeof r.weight === "number" && r.slices === 0)
+		const orphanWeights = rows.filter((r) => typeof r.weight === "number" && r.files === 0)
 
 		if (orphanWeights.length) {
 			console.error(
-				`⚠ Sources weighted in config but no slices found in corpus ` +
+				`⚠ Sources weighted in config but no parquet files found in corpus ` +
 					`(no-op weights): ${orphanWeights.map((r) => r.source).join(", ")}`
 			)
 		}
@@ -391,12 +385,13 @@ function printReport(
 
 export async function audit(opts: AuditOpts): Promise<void> {
 	const config = opts.configPath ? await parseConfig(opts.configPath) : null
-	// Compose the known-prefix list from both the hardcoded set and any extra names in the config
-	// (forward-compat for future adapters added before this file is updated).
+	// Compose the known-prefix list from both the hardcoded set and any extra names in the
+	// config (forward-compat for future adapters added before this file is updated).
 	const prefixes = [...new Set([...KNOWN_SOURCE_PREFIXES, ...Object.keys(config?.sourceWeights ?? {})])]
 
 	const stats =
-		(await manifestScan(opts.corpusDir, prefixes)) ?? (await scanSlices(opts.corpusDir, opts.sampleSliceCount ?? 100))
+		(await manifestScan(opts.corpusDir, prefixes)) ??
+		(await scanParquetFiles(opts.corpusDir, opts.sampleFileCount ?? 100))
 
 	const trainStats = stats.bySplit["train"] ?? {}
 	const rows = buildAuditRows(trainStats, config?.sourceWeights ?? {})

package/lib/tools/corpus-stats.ts

@@ -5,15 +5,16 @@
  *
  *   Pre-compute corpus-wide token + bigram label distributions for the corpus linter.
  *
- *   Reads one or more Parquet slices, builds per-(token, label) and per-(bigram, label-bigram)
- *   histograms, and serializes them as JSON. The output file is consumed by `lint-corpus-slice.ts`
- *   as the baseline against which a new slice is compared.
+ *   Reads one or more parquet files, builds per-(token, label) and per-(bigram, label-bigram)
+ *   histograms. The tool serializes them as JSON. `lint/recipe-output/index.ts` consumes the output file
+ *   as the baseline against which a new recipe output is compared.
  *
  *   Stats are cheap to compute (~5–30s per 100K rows) but expensive enough that we cache them between
- *   linter invocations. Re-run this script whenever the corpus changes substantially (a new
- *   mainline slice added, a source-pool re-weighted, etc.).
+ *   linter invocations. Re-run this whenever the corpus changes substantially (a new mainline
+ *   recipe output added, a source-pool re-weighted, etc.).
  *
- *   Output schema:
+ *   Output schema (`slice_paths` is the stats file's own key. a stats file already on disk includes it,
+ *   so the linter reads it under that spelling):
  *
  *   ```ts
  *   interface CorpusStats {
@@ -26,59 +27,72 @@
  *   }
  * ```
  *
- *   Usage: node scripts/build-corpus-stats.ts\
- *   --slices <glob-pattern-or-dir>\
- *   --output <stats.json>
+ *   Usage: mailwoman corpus stats\
+ *   --parquet <parquet-file-or-dir>\
+ *   --out <stats.json>
  *
- *   For a quick local-corpus baseline (limited but useful for linter testing): node
- *   scripts/build-corpus-stats.ts\
- *   --slices $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
- *   --output /tmp/corpus-stats-local.json
+ *   For a quick local-corpus baseline (limited but useful for linter testing): mailwoman corpus stats\
+ *   --parquet $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
+ *   --out /tmp/corpus-stats-local.json
  */
 
 import { ByteFormatter } from "@mailwoman/core/fs/formatters"
 import { statPath } from "@mailwoman/core/fs/readers"
 import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
+import { stringifyJSON } from "@mailwoman/core/json"
+import { PathBuilder } from "path-ts"
 import { Globerator } from "spliterator/node/fs"
 
-import { accumulateCooccurrences, createCooccurrenceStats, streamTokenLabelRows } from "#utils/slice-stats"
+import { accumulateCooccurrences, createCooccurrenceStats, streamTokenLabelRows } from "#utils/cooccurrence-stats"
 
 const MIN_BIGRAM_COUNT = 2
 
+/**
+ * Options for {@linkcode buildCorpusStats}.
+ */
 export interface CorpusStatsOptions {
-	slicesArg: string
+	/**
+	 * A directory of parquet files, one parquet file, or a literal path.
+	 */
+	parquetPath: string
 	outputPath: string
-	limitPerSlice?: number
+	/**
+	 * Read at most this many rows per parquet file.
+	 */
+	limitPerFile?: number
 }
 
-async function discoverSlices(slicesArg: string): Promise<string[]> {
-	const stat = await statPath(slicesArg)
+async function discoverParquetFiles(pathArg: string): Promise<string[]> {
+	const stat = await statPath(pathArg)
 
 	if (stat.isDirectory()) {
-		return await Globerator.files("parquet", { cwd: slicesArg }).toArray()
+		const root = PathBuilder.from(pathArg)
+		const names = await Globerator.files("parquet", { cwd: root, recursive: false }).toSorted()
+
+		return names.map((name) => root(name).toString())
 	}
 
-	if (stat.isFile() && slicesArg.endsWith(".parquet")) return [slicesArg]
+	if (stat.isFile() && pathArg.endsWith(".parquet")) return [pathArg]
 
 	// Otherwise treat as a literal path list (one per line if it's stdin-friendly).
-	return [slicesArg]
+	return [pathArg]
 }
 
 export async function buildCorpusStats(args: CorpusStatsOptions): Promise<void> {
-	const slicePaths = await discoverSlices(args.slicesArg)
+	const parquetPaths = await discoverParquetFiles(args.parquetPath)
 
-	console.error(`Discovered ${slicePaths.length} parquet slice(s)`)
+	console.error(`Discovered ${parquetPaths.length} parquet file(s)`)
 
 	const stats = createCooccurrenceStats()
 	const { tokens: tokenStats, bigrams: bigramStats } = stats
 	let totalRows = 0
 
-	for (const path of slicePaths) {
+	for (const path of parquetPaths) {
 		console.error(`Reading ${path}...`)
 
 		const before = totalRows
 
-		for await (const { tokens, labels } of streamTokenLabelRows(path, args.limitPerSlice)) {
+		for await (const { tokens, labels } of streamTokenLabelRows(path, args.limitPerFile)) {
 			totalRows++
 
 			if (tokens.length !== labels.length) continue // skip malformed
@@ -91,9 +105,9 @@ export async function buildCorpusStats(args: CorpusStatsOptions): Promise<void>
 		)
 	}
 
-	// Prune bigrams below MIN_BIGRAM_COUNT to keep the output file size sane. Token stats
-	// stay complete — they're cheap and we need accuracy at the long tail for label-vacuum
-	// detection.
+	// Prune bigrams below MIN_BIGRAM_COUNT to keep the output file size sane.
+	// Token stats stay complete.
+	// They're cheap and we need accuracy at the long tail for label-vacuum detection.
 	let prunedBigrams = 0
 
 	for (const [k, labelMap] of bigramStats) {
@@ -114,7 +128,7 @@ export async function buildCorpusStats(args: CorpusStatsOptions): Promise<void>
 
 	const out = {
 		row_count: totalRows,
-		slice_paths: slicePaths,
+		slice_paths: parquetPaths,
 		tokens: {} as Record<string, Record<string, number>>,
 		bigrams: {} as Record<string, Record<string, number>>,
 	}
@@ -128,7 +142,7 @@ export async function buildCorpusStats(args: CorpusStatsOptions): Promise<void>
 	}
 
 	await writeLocalJSONFile(out, args.outputPath)
-	const formattedSize = ByteFormatter.formatIEC(Buffer.byteLength(JSON.stringify(out)))
+	const formattedSize = ByteFormatter.formatIEC(Buffer.byteLength(stringifyJSON(out)))
 
 	console.error(
 		`Wrote ${args.outputPath} (${formattedSize}) — ${totalRows} rows, ${tokenStats.size} tokens, ${bigramStats.size} bigrams`