@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/us/tools/fetch/tiger-full.ts

@@ -4,67 +4,36 @@ import { APIClient, pluckResponseData } from "@mailwoman/core/api"
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
+ * Fetch the full TIGER 2024 addrfeat dataset for all US counties from
+ * `https://www2.census.gov/geo/tiger/TIGER2024/addrfeat/`, where files use the pattern
+ * `tl_2024_<statefips><countyfips>_addrfeat.zip`.
  *
- *   TIGER ADDRFEAT 2024 source:
- *
- *   - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
- *   - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
- *
- *   Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
- *   `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
- *   already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
- *   this module is download + provenance only.)
- *
- *   Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
- *   county ZIP to disk (no curl subprocess).
+ * Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
+ * `manifest.json` recording filename, sha256 and bytes so re-runs skip already-verified files.
+ * Extraction and ogr2ogr ingestion happen later, in the `tiger` adapter.
  */
 /* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+   procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
 import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { statPath, pathExists } from "@mailwoman/core/fs/readers"
+import { statPath, pathExists } from "@mailwoman/core/fs/readers/stat"
 import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
 import { sha256File } from "@mailwoman/core/hash"
 import { isoSeconds } from "@mailwoman/core/utils"
 import { sleep } from "@mailwoman/core/utils/sleep"
-import { basename, join } from "path-ts"
+import type { PathBuilder } from "path-ts"
 
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index"
-import { readManifest, streamDownload, writeManifest } from "#tools/fetch/download/index"
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { readManifest, streamDownload, writeManifest } from "#tools/fetch/download"
 
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-/**
- * Lowest 2xx status; anything below is informational.
- */
 const HTTP_OK = 200
-
-/**
- * Lowest 3xx status; at or above it the response is a redirect or an error, not a body.
- */
 const HTTP_REDIRECT = 300
 
 const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT"
 
 export interface FetchTigerFullOptions extends BaseFetchOptions {
-	/**
-	 * Space-separated list of 2-digit state FIPS codes to skip entirely. Default `"50"` — Vermont, already fetched in
-	 * v0.1.1.
-	 */
 	skipStateFips?: string
-	/**
-	 * Seconds to sleep between downloads. Default `0.2`.
-	 */
 	rateSleep?: number
-	/**
-	 * Max concurrent download workers per state. Default `4`.
-	 */
 	maxParallel?: number
-	/**
-	 * Print planned downloads without fetching. Default `false`.
-	 */
 	dryRun?: boolean
 }
 
@@ -74,10 +43,7 @@ interface CountyEntry {
 	bytes: number
 }
 
-/**
- * Read a per-state MANIFEST.json into a filename → entry map.
- */
-async function readCountyManifest(manifestPath: string): Promise<Map<string, CountyEntry>> {
+async function readCountyManifest(manifestPath: PathBuilder): Promise<Map<string, CountyEntry>> {
 	const map = new Map<string, CountyEntry>()
 	const parsed = await readManifest<{ counties?: CountyEntry[] }>(manifestPath)
 
@@ -90,10 +56,7 @@ async function readCountyManifest(manifestPath: string): Promise<Map<string, Cou
 	return map
 }
 
-/**
- * Check whether a file already matches a recorded sha256 and byte count.
- */
-async function fileMatchesSha(path: string, expectedSha: string, expectedBytes: number): Promise<boolean> {
+async function fileMatchesSha(path: PathBuilder, expectedSha: string, expectedBytes: number): Promise<boolean> {
 	if (!(await pathExists(path))) return false
 
 	if ((await statPath(path)).size !== expectedBytes) return false
@@ -105,11 +68,8 @@ type CountyResult =
 	| { ok: true; filename: string; sha256: string; bytes: number }
 	| { ok: false; filename: string; reason: string }
 
-/**
- * Download one county ZIP (size sanity check + sha256).
- */
-async function downloadCounty(url: string, dest: string): Promise<CountyResult> {
-	const filename = basename(dest)
+async function downloadCounty(url: string, dest: PathBuilder): Promise<CountyResult> {
+	const filename = dest.basename()
 	const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 })
 
 	if (status < HTTP_OK || status >= HTTP_REDIRECT) {
@@ -133,20 +93,18 @@ export async function fetchTigerFull(
 	options: FetchTigerFullOptions,
 	report?: (line: string) => void
 ): Promise<FetchSummary> {
-	const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length > 0)
+	const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length)
 	const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000)
 	const maxParallel = options.maxParallel ?? 4
 	const dryRun = options.dryRun ?? false
 
-	const addrfeatDir = join(options.outRoot, "tiger", "addrfeat")
+	const addrfeatDir = options.outRoot("tiger", "addrfeat")
 	await makeDirectories(addrfeatDir)
 
 	// MARK: Step 1 — discover the county file list
 
 	report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`)
 
-	// `responseType: "text"` — an Apache directory index, scraped below. The per-county ARCHIVE downloads
-	// stay on raw `fetch` (they stream to disk; see `downloadOne`).
 	const listingRes = await new APIClient({
 		displayName: "tiger-listing",
 		retry: true,
@@ -160,8 +118,7 @@ export async function fetchTigerFull(
 	const totalCounties = allZips.length
 	report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`)
 
-	// Build a map: state_fips -> list of filenames.
-	// tl_2024_SSCCC_addrfeat.zip — SS = 2-digit state FIPS (chars 8-9), CCC = county FIPS.
+	// `tl_2024_SSCCC_addrfeat.zip`: SS is the 2-digit state FIPS at chars 8-9.
 	const stateFiles = new Map<string, string[]>()
 
 	for (const fname of allZips) {
@@ -182,13 +139,11 @@ export async function fetchTigerFull(
 	let totalBytesFetched = 0
 	const failedCodes: string[] = []
 
-	// Process states in sorted FIPS order for predictable output.
 	const sortedStates = [...stateFiles.keys()].toSorted()
 
 	for (const stateFips of sortedStates) {
 		const countyFiles = stateFiles.get(stateFips) ?? []
 
-		// Respect an explicit request to skip this state before scheduling its counties.
 		if (skipStateFips.includes(stateFips)) {
 			report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`)
 			totalSkippedState += countyFiles.length
@@ -196,24 +151,21 @@ export async function fetchTigerFull(
 			continue
 		}
 
-		const stateDir = join(addrfeatDir, `state-${stateFips}`)
+		const stateDir = addrfeatDir(`state-${stateFips}`)
 		await makeDirectories(stateDir)
-		const manifestPath = join(stateDir, "MANIFEST.json")
+		const manifestPath = stateDir("MANIFEST.json")
 
-		// Load existing manifest for O(1) verified-skip lookup.
 		const manifest = await readCountyManifest(manifestPath)
 
 		report?.(`--- State ${stateFips} — ${countyFiles.length} counties`)
 
-		// Build a list of URLs+dests that need fetching.
-		const pending: Array<{ url: string; dest: string }> = []
+		const pending: Array<{ url: string; dest: PathBuilder }> = []
 
 		for (const fname of countyFiles) {
-			const dest = join(stateDir, fname)
+			const dest = stateDir(fname)
 			const url = `${TIGER_BASE_URL}/${fname}`
 			const known = manifest.get(fname)
 
-			// Skip if already verified via MANIFEST.
 			if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
 				report?.(`  skip (verified) ${fname}`)
 
@@ -237,7 +189,6 @@ export async function fetchTigerFull(
 
 		if (!pending.length) continue
 
-		// Download pending counties with bounded parallelism and rate-limit spacing.
 		const results: CountyResult[] = Array.from({ length: pending.length })
 		let cursor = 0
 
@@ -247,7 +198,6 @@ export async function fetchTigerFull(
 
 				if (i >= pending.length) return
 				const item = pending[i]!
-				// Rate-limit: polite spacing before each fetch.
 				await sleep(rateSleepMs)
 				results[i] = await downloadCounty(item.url, item.dest)
 			}
@@ -255,7 +205,6 @@ export async function fetchTigerFull(
 
 		await Promise.all(workers)
 
-		// Collect results from this state.
 		for (const result of results) {
 			if (result.ok) {
 				report?.(
@@ -274,7 +223,6 @@ export async function fetchTigerFull(
 			}
 		}
 
-		// Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
 		const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename))
 
 		const manifestDoc = {

package/lib/utils/align.ts

@@ -19,47 +19,56 @@
  *        character range. First token in a component span → `B-<tag>`; subsequent tokens →
  *        `I-<tag>`; no overlap → `O`.
  *   5. Emit the located char spans verbatim as `span_starts[]` / `span_ends[]` / `span_tags[]` (the
- *        v0.5.0 char-offset format, #519). The token quantization in step 4 is the part the v0.5.0
- *        rebuild deletes; during the transition both representations ride on every labeled row.
+ *        v0.5.0 char-offset format).
  *
  *   Structural invariants the function preserves (the span ones loudly — a violation throws rather
- *   than quarantines, because it indicates a bug here, not bad source data):
+ *   than quarantines, because it indicates a bug here rather than bad source data):
  *
  *   - `tokens.length === labels.length` always.
  *   - Each component contributes at most one contiguous BIO run (no `B-tag … O … I-tag` gaps). This is
  *       enforced by greedy first-match span assignment + ordered token iteration.
  *   - The span triple is sorted ascending by start and non-overlapping.
- *   - `raw` is NFC-normalized (asserted per row; a non-NFC raw makes char offsets ambiguous downstream
+ *   - `raw` is NFC-normalized (asserted per row. a non-NFC raw makes char offsets ambiguous downstream
  *       — NFD `é` occupies two code units where NFC `é` occupies one — and silently so).
  */
 
 import type { BIOLabel, ComponentTag } from "@mailwoman/codex/component"
+import type { TextSpan } from "@mailwoman/query-shape"
 import { distance as levenshteinDistance } from "fastest-levenshtein"
 
 import type { CanonicalRow, LabeledRow, QuarantinedRow } from "#types"
-import { whitespaceTokenizer, type TokenSpan, type Tokenizer } from "#utils/tokenize"
+import { whitespaceTokenizer, type Tokenizer } from "#utils/tokenize"
 
 /**
  * Options for `alignRow`.
  */
 export interface AlignOptions {
 	/**
-	 * Tokenizer to use. Defaults to `whitespaceTokenizer()`.
+	 * Tokenizer to use.
+	 *
+	 * @defaultValue `whitespaceTokenizer()`
 	 */
 	tokenizer?: Tokenizer
 
 	/**
-	 * Max Levenshtein edit distance to accept when a verbatim substring match fails. Set `0` to require verbatim matches
-	 * only. Default `2`.
+	 * Max Levenshtein edit distance to accept when a verbatim substring match fails.
+	 *
+	 * Set `0` to require verbatim matches only.
 	 *
-	 * Distance is computed against same-length windows in `raw`, so the threshold scales naturally with the component
-	 * value length.
+	 * Distance is computed against same-length windows in `raw`, so the threshold
+	 * scales naturally with the component value length.
+	 *
+	 * @defaultValue `2`
 	 */
 	maxEditDistance?: number
 
 	/**
-	 * Case-insensitive comparison for substring search. Default `true`. The retained span in `raw` is the original case;
-	 * only matching is case-insensitive.
+	 * Case-insensitive comparison for substring search.
+	 *
+	 * The retained span in `raw` is the original case.
+	 * Only matching is case-insensitive.
+	 *
+	 * @defaultValue `true`
 	 */
 	caseInsensitive?: boolean
 }
@@ -70,8 +79,7 @@ export interface AlignOptions {
 export type AlignmentResult = { kind: "labeled"; row: LabeledRow } | { kind: "quarantined"; row: QuarantinedRow }
 
 /**
- * One located char-offset label span over a row's `raw` ([start, end) in UTF-16 code units). The element type behind
- * the parallel `span_starts[]`/`span_ends[]`/`span_tags[]` triple on `LabeledRow` (#519).
+ * One located char-offset label span over a row's `raw` ([start, end) in UTF-16 code units). The element type behind the parallel `span_starts[]`/`span_ends[]`/`span_tags[]` triple on `LabeledRow`.
  */
 export interface ComponentSpan {
 	tag: ComponentTag
@@ -91,10 +99,6 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 		return { kind: "quarantined", row: { row, reason: "raw-empty" } }
 	}
 
-	// #519 NFC handling — relaxed from a hard throw to normalization (2026-06-12, DeepSeek-validated):
-	// one non-NFC row (e.g. a non-Latin name variant like "দক্ষিণ কোরিয়া") must not crash a multi-hour
-	// build. Normalize `raw` AND every component value to NFC, compute spans over the NFC raw, and
-	// store the NFC raw — preserving the #519 single-normalization-form principle while keeping the row.
 	const raw = row.raw.normalize("NFC")
 	const components = { ...row.components }
 
@@ -111,10 +115,9 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 
 	const haystack = caseInsensitive ? raw.toLowerCase() : raw
 
-	// Longest value first: a short component must not claim a word that a longer, more specific
-	// component owns ("Alaska Regional Dr, Alaska" — region "Alaska" stealing the street's first
-	// word quarantined the street; pilot2's residual class). Emit order is unaffected — spans are
-	// re-sorted by start below.
+	// Longest value first: a short component must not claim a word that a longer,
+	// more specific component owns.
+	// Emit order is unaffected, since spans are re-sorted by start below.
 	const entries = (Object.entries(components) as Array<[ComponentTag, string | undefined]>).toSorted(
 		(a, b) => (b[1]?.length ?? 0) - (a[1]?.length ?? 0)
 	)
@@ -132,12 +135,8 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 			}
 		}
 
-		// Defensive bounds quarantine (2026-06-12): locateSpan's fuzzy/boundary logic can over-run the
-		// raw by a code unit on some non-Latin / combining-mark strings (e.g. Bengali name variants),
-		// yielding a span past the end. An out-of-bounds offset can never be a valid char span —
-		// quarantine the row rather than crash assertSpanInvariants and take down a multi-hour build.
-		// (If this class proves large in the quarantine report, locateSpan's boundary logic needs a
-		// combining-mark fix to KEEP these non-Latin rows.)
+		// An out-of-bounds offset can never be a valid char span, so quarantine the row
+		// rather than crash assertSpanInvariants and take down a multi-hour build.
 		if (span.start < 0 || span.end > raw.length || span.start >= span.end) {
 			return {
 				kind: "quarantined",
@@ -160,8 +159,6 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 		components,
 		tokens: tokens.map((t) => t.text),
 		labels,
-		// The v0.5.0 char-offset triple (#519): the located spans, emitted verbatim. The token
-		// quantization above is what the rebuild deletes; both ride during the transition.
 		span_starts: componentSpans.map((s) => s.start),
 		span_ends: componentSpans.map((s) => s.end),
 		span_tags: componentSpans.map((s) => s.tag),
@@ -171,13 +168,13 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 }
 
 /**
- * Enforce the #519 span-triple invariants — in-bounds, sorted ascending by start, non-overlapping — loudly.
+ * Enforce the span-triple invariants: in-bounds, sorted ascending by start, non-overlapping.
+ *
+ * For `alignRow`: `claimed`-span bookkeeping in `locateSpan` already makes overlap impossible
+ * and the caller sorts, so a violation here is a bug in this file rather than bad source data.
+ * Throw rather than quarantine, so the corruption cannot ride into a corpus.
  *
- * For `alignRow`: `claimed`-span bookkeeping in `locateSpan` already makes overlap impossible and the caller sorts, so
- * a violation here is a bug in this file, not bad source data: throw (naming the row) rather than quarantine, so the
- * corruption can't ride into a corpus. Exported for every OTHER span producer (`composeAdversarialRow`'s offset
- * arithmetic, future synthesis paths) — any code that emits the triple without going through `alignRow` must pass its
- * output through this.
+ * Any code that emits the triple without going through `alignRow` must pass its output through this.
  */
 export function assertSpanInvariants(
 	spans: readonly ComponentSpan[],
@@ -213,11 +210,13 @@ export function assertSpanInvariants(
 }
 
 /**
- * Locate `needle` in `haystack` (both already normalized for case if requested), preferring verbatim substring match.
- * Falls back to a fuzzy window scan when verbatim fails and `maxEditDistance > 0`. Already-claimed spans are skipped so
- * two components don't grab overlapping ranges.
+ * Locate `needle` in `haystack` (both already normalized for case if requested),
+ * preferring verbatim substring match.
  *
- * Returns the span in the original `raw` (not the lower-cased `haystack`).
+ * Falls back to a fuzzy window scan when verbatim fails and `maxEditDistance > 0`.
+ * Already-claimed spans are skipped so two components don't grab overlapping ranges.
+ *
+ * @returns The span in the original `raw` (not the lower-cased `haystack`).
  */
 function locateSpan(args: {
 	haystack: string
@@ -225,18 +224,17 @@ function locateSpan(args: {
 	raw: string
 	claimed: Array<[number, number]>
 	maxEditDistance: number
-}): { start: number; end: number } | undefined {
+}): { start: number; end: number } | null {
 	const { haystack, needle, claimed, maxEditDistance } = args
 
-	if (!needle.length) return undefined
+	if (!needle) return null
 
-	// Pass 1: verbatim substring. Word-boundary-aligned matches are PREFERRED over intra-word ones
-	// — leftmost-substring alone let a short value claim the inside of an earlier word (region "AK"
-	// matched inside "Umak"/"Lake", scrambling every later span; caught by the v0.5.0 pilot build).
+	// Pass 1: verbatim substring.
+	// Word-boundary-aligned matches are preferred over intra-word ones, so a short
+	// value cannot claim the inside of an earlier word.
 	// Intra-word matches stay allowed as the fallback because they are essential for affix
-	// supervision (street_suffix "straße" inside "Hauptstraße" has no boundary-aligned occurrence —
-	// sub-word spans are the point of the char-offset format).
-	let intraWord: { start: number; end: number } | undefined
+	// supervision (street_suffix "straße" inside "Hauptstraße" has no boundary-aligned occurrence).
+	let intraWord: { start: number; end: number } | null = null
 	let from = 0
 
 	while (true) {
@@ -255,23 +253,22 @@ function locateSpan(args: {
 
 	if (intraWord) return intraWord
 
-	if (maxEditDistance <= 0) return undefined
+	if (maxEditDistance <= 0) return null
 
-	// Pass 2: fuzzy sliding-window. Walk over candidate windows of length `needle.length`
-	// across haystack, compute Levenshtein, pick the leftmost window under the threshold.
+	// Pass 2: fuzzy sliding-window.
 	const len = needle.length
 
 	for (let i = 0; i + len <= haystack.length; i++) {
 		if (overlapsClaimed(i, i + len, claimed)) continue
 		const window = haystack.slice(i, i + len)
 
-		if (window === needle) return { start: i, end: i + len } // covered by pass 1, but cheap
+		if (window === needle) return { start: i, end: i + len }
 		const d = levenshteinDistance(window, needle)
 
 		if (d <= maxEditDistance) return { start: i, end: i + len }
 	}
 
-	return undefined
+	return null
 }
 
 const WORD_CHAR = /[\p{L}\p{N}]/u
@@ -295,16 +292,12 @@ function overlapsClaimed(start: number, end: number, claimed: Array<[number, num
 }
 
 /**
- * Assign BIO labels to tokens given the component spans. Components MUST be sorted by start offset. For each token,
- * find the first component span that contains the token's start offset; if the token is the first one inside that span
- * emit `B-<tag>`, else `I-<tag>`.
+ * Assign BIO labels to tokens from component spans sorted by start offset.
  */
-function labelTokens(tokens: readonly TokenSpan[], spans: readonly ComponentSpan[]): readonly BIOLabel[] {
+function labelTokens(tokens: readonly TextSpan[], spans: readonly ComponentSpan[]): readonly BIOLabel[] {
 	const out: BIOLabel[] = []
 	const seenSpan = new Set<number>()
 
-	// index into `spans`
-
 	for (const token of tokens) {
 		let assigned: BIOLabel = "O"
 

package/lib/utils/{slice-stats.ts → cooccurrence-stats.ts}

@@ -4,21 +4,25 @@
  * @author Teffen Ellis, et al.
  *
  *   Token/label co-occurrence accounting shared by the corpus-stats builder (`tools/corpus-stats.ts`)
- *   and the slice linter (`tools/lint-slice.ts`). The two sides MUST agree on the bigram key format:
- *   the linter looks its slice's bigrams up in the stats file's `bigrams` table, and a key built with
+ *   and the recipe-output linter (`tools/lint/recipe-output/index.ts`). The two sides must agree on the bigram key
+ *   format: the linter looks its recipe output's bigrams up in the stats file's `bigrams` table. A key built with
  *   a different separator never matches — which silently blanks the bigram-collision check.
  */
 
-import { streamParquetRows } from "#utils/parquet"
+import { openParquetRowStream } from "#parquet/streams"
 
 /**
- * Separator inside a bigram key (`tok1␟tok2`) and a label-bigram value (`lab1␟lab2`): U+001F UNIT SEPARATOR, a
- * character no address token contains. Render a key for humans with `key.split(SLICE_STATS_SEP).join(" ")`.
+ * Separator inside a bigram key (`tok1␟tok2`) and a label-bigram value (`lab1␟lab2`):
+ * U+001F unit separator, a character no address token contains.
+ *
+ * Render a key for humans with `key.split(COOCCURRENCE_KEY_SEP).join(" ")`.
  */
-export const SLICE_STATS_SEP = ""
+export const COOCCURRENCE_KEY_SEP = ""
 
 /**
- * Nested tally: outer key → (inner key → count). Token → label counts, or bigram key → label-bigram counts.
+ * Nested tally: outer key → (inner key → count).
+ *
+ * Token → label counts, or bigram key → label-bigram counts.
  */
 type CooccurrenceTable = Map<string, Map<string, number>>
 
@@ -49,8 +53,9 @@ function bump(table: CooccurrenceTable, key: string, sub: string): void {
 }
 
 /**
- * Fold one row's parallel `tokens`/`labels` into the tables. The caller has already checked the arrays are the same
- * length.
+ * Fold one row's parallel `tokens`/`labels` into the tables.
+ *
+ * The caller has already checked the arrays are the same length.
  */
 export function accumulateCooccurrences(
 	stats: CooccurrenceStats,
@@ -64,13 +69,13 @@ export function accumulateCooccurrences(
 		bump(stats.tokens, tk, lb)
 
 		if (i + 1 < tokens.length) {
-			bump(stats.bigrams, tk + SLICE_STATS_SEP + tokens[i + 1]!, lb + SLICE_STATS_SEP + labels[i + 1]!)
+			bump(stats.bigrams, tk + COOCCURRENCE_KEY_SEP + tokens[i + 1]!, lb + COOCCURRENCE_KEY_SEP + labels[i + 1]!)
 		}
 	}
 }
 
 /**
- * A slice row projected to the two columns the co-occurrence pass reads.
+ * A parquet row projected to the two columns the co-occurrence pass reads.
  */
 export interface TokenLabelRow {
 	tokens: string[]
@@ -79,15 +84,16 @@ export interface TokenLabelRow {
 }
 
 /**
- * Stream a slice's `tokens`/`labels` columns.
+ * Stream a parquet file's `tokens`/`labels` columns.
  *
- * Projected rather than read whole: parquet is columnar, so the unused columns are never touched. `limit` stops the
- * iteration rather than filtering afterwards, so a capped run reads only the row groups it needs.
+ * Projected rather than read whole: parquet is columnar, so the unused columns are never touched.
+ * `limit` stops the iteration rather than filtering afterwards, so a capped
+ * run reads only the row groups it needs.
  */
-export async function* streamTokenLabelRows(slicePath: string, limit?: number): AsyncIterable<TokenLabelRow> {
+export async function* streamTokenLabelRows(parquetPath: string, limit?: number): AsyncIterable<TokenLabelRow> {
 	let emitted = 0
 
-	for await (const row of streamParquetRows<TokenLabelRow>(slicePath, ["tokens", "labels"], { limit })) {
+	for await (const row of openParquetRowStream<TokenLabelRow>(parquetPath, { columns: ["tokens", "labels"], limit })) {
 		yield row
 
 		emitted++