@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/tools/golden/expand.ts

@@ -8,19 +8,20 @@
  *
  *   ## Why this approach
  *
- *   The Phase 2 golden set has 74 entries; session-notes.md called for ≥500/locale. Manual curation
+ *   The Phase 2 golden set has 74 entries. session-notes.md called for ≥500/locale. Manual curation
  *   doesn't scale. Pure-LLM generation (invent raw + labels from scratch) is too noisy — labels
  *   would be unverified.
  *
  *   This module takes the middle path:
  *
- *   1. **Seeds come from corpus-v0.2.0 test slice** — already through the alignment pipeline, so labels
+ *   1. **Seeds come from the corpus-v0.2.0 test split's parquet files** — already through the alignment pipeline, so
+ *        labels
  *        are pipeline-verified.
  *   2. **LLM only varies the surface form** — case, abbreviations, reordering, dropped components. The
- *        component VALUES (locality string, postcode digits, etc.) are preserved verbatim.
- *   3. **Programmatic validator drops hallucinations** — every component value must appear as a
+ *        component values (locality string, postcode digits, etc.) are preserved verbatim.
+ *   3. **Programmatic validator drops hallucinations**. Every component value must appear as a
  *        substring (case-insensitive, whitespace-normalized) of the variant's raw. Failures dropped
- *        silently; cost is wasted tokens, never bad-labeled golden entries.
+ *        silently. cost is wasted tokens, never bad-labeled golden entries.
  *
  *   ## Usage
  *
@@ -37,7 +38,7 @@
  *   - `DEEPSEEK_API_KEY` — required for provider `deepseek`
  *   - `ANTHROPIC_API_KEY` — required for provider `anthropic`
  *
- *   ## What this module does NOT do
+ *   ## What this module does not do
  *
  *   - Does not commit anything or modify the versioned golden dir. Candidates land in
  *       `data/eval/golden/candidates/` for operator review (skim, prune, then run
@@ -49,14 +50,15 @@
 import { APIClient, pluckResponseData } from "@mailwoman/core/api"
 import { dataRootPath } from "@mailwoman/core/data-root"
 import { makeDirectories } from "@mailwoman/core/fs/writers"
-import { tryParsingJSON } from "@mailwoman/core/json"
-import { isPresent } from "@mailwoman/core/objects"
+import { tryParsingJSON, stringifyJSON, prettyJSON } from "@mailwoman/core/json"
+import { SeededRandom } from "@mailwoman/core/random"
+import { extractDelimited } from "@mailwoman/core/scripting/arguments"
 import { foldCaseWhitespace } from "@mailwoman/normalize/fold"
 import { dirname } from "path-ts"
-import { createNewlineWriter } from "spliterator"
+import { createNewlineWriter, TextSpliterator } from "spliterator"
 
 import { $private } from "#env"
-import { streamParquetRows } from "#utils/parquet"
+import { openParquetRowStream } from "#parquet/streams"
 
 // ── Types ─────────────────────────────────────────────────────────────────
 
@@ -65,6 +67,14 @@ import { streamParquetRows } from "#utils/parquet"
  */
 const MAX_CANDIDATE_LENGTH = 500
 
+/**
+ * The stream position the per-source subsample draws from.
+ *
+ * Fixed rather than supplied, because the point is that two runs over one corpus choose the same seeds.
+ * A caller-supplied seed would make that the caller's problem to remember.
+ */
+const SUBSAMPLE_SEED = 20_260_913
+
 interface CorpusRow {
 	raw: string
 	tokens: string[]
@@ -101,31 +111,45 @@ interface GoldenCandidate {
 
 export interface ExpandGoldenOptions {
 	/**
-	 * Corpus test slice path(s), comma-separated. Default: the v0.2.0 test slice under the data root.
+	 * Corpus test-split parquet path(s), comma-separated.
+	 *
+	 * Default: the v0.2.0 test parquet file under the data root.
 	 */
 	corpus?: string
 	/**
-	 * Total seeds to process. Default `100` (pilot).
+	 * Total seeds to process.
+	 *
+	 * Default `100` (pilot).
 	 */
 	count?: number
 	/**
-	 * Variants requested per seed. Default `5`.
+	 * Variants requested per seed.
+	 *
+	 * Default `5`.
 	 */
 	variants?: number
 	/**
-	 * JSONL output path. Default `data/eval/golden/candidates/expand-<ts>.jsonl`.
+	 * Jsonl output path.
+	 *
+	 * Default `data/eval/golden/candidates/expand-<ts>.jsonl`.
 	 */
 	output?: string
 	/**
-	 * LLM provider. Default `deepseek`.
+	 * LLM provider.
+	 *
+	 * Default `deepseek`.
 	 */
 	provider?: "deepseek" | "anthropic"
 	/**
-	 * Model id. Default depends on provider.
+	 * Model id.
+	 *
+	 * Default depends on provider.
 	 */
 	model?: string
 	/**
-	 * Parallel LLM calls. Default `4`.
+	 * Parallel LLM calls.
+	 *
+	 * Default `4`.
 	 */
 	concurrency?: number
 	/**
@@ -145,9 +169,11 @@ export interface ExpandGoldenSummary {
 // ── Seed loading ──────────────────────────────────────────────────────────
 
 /**
- * Decode BIO labels + tokens into a verified components map. Mirrors the Python `decode_components` in
- * mailwoman_train/eval.py — first-occurrence-wins per tag, contiguous B-X/I-X runs concatenated with a single space
- * (the canonical separator used by corpus alignment).
+ * Decode BIO labels + tokens into a verified components map.
+ *
+ * Mirrors the Python `decode_components` in mailwoman_train/eval.py —
+ * first-occurrence-wins per tag, contiguous B-X/I-X runs concatenated with a single
+ * space (the canonical separator used by corpus alignment).
  */
 function decodeComponents(tokens: string[], labels: string[]): Record<string, string> {
 	const out: Record<string, string> = {}
@@ -195,27 +221,25 @@ async function loadSeeds(
 	includeSources: Set<string> | null,
 	report?: (line: string) => void
 ): Promise<Seed[]> {
-	const paths = corpusPath
-		.split(",")
-		.map((p) => p.trim())
-		.filter(isPresent)
+	const paths = extractDelimited(corpusPath)
 
-	report?.(`reading seeds from ${paths.length} slice(s) (target: ${count}, stratified)`)
+	report?.(`reading seeds from ${paths.length} parquet file(s) (target: ${count}, stratified)`)
 
 	if (includeSources) {
 		report?.(`  include-sources filter: ${Array.from(includeSources).join(", ")}`)
 	}
 
-	// Stratified sampling: read all rows from all slices, group by source. Bounded by per-source
-	// reservoir: keep at most max(2*count, 5000) rows per source so we don't blow memory on train
-	// slices (1M rows × many slices). Sampling later is uniform within each pool.
+	// Stratified sampling: read all rows from all parquet files, group by source.
+	// Bounded by per-source reservoir: keep at most max(2*count, 5000) rows per source
+	// so we don't blow memory on train parquet files (1M rows × many files).
+	// Later samples are uniform within each pool.
 	const bySource = new Map<string, Seed[]>()
 	const PER_SOURCE_CAP = Math.max(2 * count, 5000)
 	let scanned = 0
 	let skippedThinComponents = 0
 
 	for (const path of paths) {
-		for await (const row of streamParquetRows<CorpusRow>(path)) {
+		for await (const row of openParquetRowStream<CorpusRow>(path)) {
 			scanned++
 
 			// Source allow-list (--include-sources) — applied early to skip parsing rows we won't use
@@ -251,7 +275,7 @@ async function loadSeeds(
 	}
 
 	report?.(
-		`  scanned ${scanned} rows across ${paths.length} slice(s); thin-components dropped: ${skippedThinComponents}`
+		`  scanned ${scanned} rows across ${paths.length} parquet file(s); thin-components dropped: ${skippedThinComponents}`
 	)
 
 	report?.(`  per-source pool sizes:`)
@@ -260,23 +284,26 @@ async function loadSeeds(
 		report?.(`    ${src}: ${pool.length}`)
 	}
 
-	// Round-robin sample. Each source gives floor(count / nSources) seeds; rounding goes
-	// to sources in alphabetical order. If a pool is smaller than its target, take all of it.
+	// Round-robin sample.
+	// Each source gives floor(count / nSources) seeds.
+	// Sources in alphabetical order receive the rounding remainder.
+	// If a pool is smaller than its target, take all of it.
 	const sources = Array.from(bySource.keys()).toSorted()
 	const perSource = Math.floor(count / sources.length)
 	const remainder = count - perSource * sources.length
 	const picked: Seed[] = []
+	// One stream across every source, so the same inputs choose the same seeds on every run
+	// and two sources do not draw the same positions.
+	// The previous shuffle called `Math.random()` under a comment promising determinism.
+	const random = new SeededRandom(SUBSAMPLE_SEED)
 
 	for (let i = 0; i < sources.length; i++) {
 		const src = sources[i]!
 		const pool = bySource.get(src)!
 		const target = perSource + (i < remainder ? 1 : 0)
 
-		// Random subsample without replacement — deterministic via shuffle then slice
-		for (let j = pool.length - 1; j > 0; j--) {
-			const k = Math.floor(Math.random() * (j + 1))
-			;[pool[j], pool[k]] = [pool[k]!, pool[j]!]
-		}
+		// Subsample without replacement — shuffle, then take the head.
+		random.shuffle(pool)
 
 		const take = Math.min(target, pool.length)
 		picked.push(...pool.slice(0, take))
@@ -325,7 +352,7 @@ OUTPUT a JSON array of N objects, each shaped {"raw": "...", "dropped": ["..."]}
 
 function buildUserPrompt(seed: Seed, n: number): string {
 	return `INPUT:
-${JSON.stringify({ raw: seed.raw, components: seed.components, country: seed.country }, null, 2)}
+${prettyJSON({ raw: seed.raw, components: seed.components, country: seed.country })}
 
 N: ${n}`
 }
@@ -431,7 +458,7 @@ function parseCandidates(text: string): Candidate[] {
 // ── Validator ─────────────────────────────────────────────────────────────
 
 /**
- * Components that are NEVER allowed to be dropped — keeps degenerate single-token candidates out.
+ * Components that are never allowed to be dropped — keeps degenerate single-token candidates out.
  */
 const REQUIRED_COMPONENT_TAGS = new Set(["locality", "region", "street", "house_number", "venue"])
 
@@ -462,7 +489,7 @@ function validate(seed: Seed, candidate: Candidate): boolean {
 		keptCount++
 	}
 
-	// Reject degenerate single-component candidates ("VT", "Paris" alone).
+	// Reject degenerate single-component candidates ("VT", "Paris" by itself).
 	if (keptCount < 2) return false
 
 	return true
@@ -483,7 +510,11 @@ export async function expandGolden(
 	const providerName = options.provider ?? "deepseek"
 	const model = options.model ?? (providerName === "anthropic" ? "claude-haiku-4-5-20251001" : "deepseek-chat")
 	const concurrencyLimit = options.concurrency ?? 4
-	const includeSources = options.includeSources ? new Set(options.includeSources.split(",").map((s) => s.trim())) : null
+
+	const includeSources = options.includeSources
+		? new Set(TextSpliterator.from(options.includeSources, { delimiter: "," }))
+		: null
+
 	const ts = new Date().toISOString().replaceAll(/[:.]/g, "-").slice(0, 19)
 	const outputPath = options.output ?? `data/eval/golden/candidates/expand-${ts}.jsonl`
 
@@ -530,7 +561,7 @@ export async function expandGolden(
 
 						// Remove dropped components from the components map
 						for (const tag of goldenCandidate.dropped_components) {
-							// oxlint-disable-next-line typescript/no-dynamic-delete -- removing one key from a plain record; the object is not on a hot path
+							// oxlint-disable-next-line typescript/no-dynamic-delete -- removing one key from a plain record. the object is not on a hot path
 							delete goldenCandidate.components[tag]
 						}
 
@@ -558,7 +589,7 @@ export async function expandGolden(
 		await using out = createNewlineWriter(outputPath)
 
 		for (const row of outRows) {
-			await out.write(JSON.stringify(row))
+			await out.write(stringifyJSON(row))
 		}
 	}
 

package/lib/tools/golden/promote.ts

@@ -8,18 +8,18 @@
  *
  *   ## What it does
  *
- *   1. Reads a candidates JSONL from `data/eval/golden/candidates/`
+ *   1. Reads a candidates jsonl from `data/eval/golden/candidates/`
  *   2. Reads the previous-version golden dir for forward-copy + dedup base
  *   3. Applies filters that drop candidates unlikely to be human-typed:
  *
  *        - Components-glued-without-commas (5+ components but <2 separators → freeform jumble)
- *        - Postcode-at-start with many other components (US/UK conventions put postcode last; FR puts it
+ *        - Postcode-at-start with many other components (US/UK conventions put postcode last. FR puts it
  *                 before locality only)
  *        - Suspicious-token signals (unmatched brackets, control chars, etc.)
  *   4. Dedupes by normalized raw (case-insensitive, whitespace-collapsed)
  *   5. Splits by country (US/FR/other) and writes `data/eval/golden/v<X.Y.Z>/{us,fr,other}.jsonl`
- *   6. Forward-copies the prior version's adversarial.jsonl + README as-is
- *   7. Writes MANIFEST.json with sha256 of each output file
+ *   6. Forward-copies the prior version's adversarial.jsonl + readme as-is
+ *   7. Writes manifest.json with sha256 of each output file
  *
  *   ## Usage
  *
@@ -34,8 +34,9 @@
 import { pathExists } from "@mailwoman/core/fs/readers"
 import { copyFileTo, makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
 import { sha256File } from "@mailwoman/core/hash"
+import { stringifyJSON } from "@mailwoman/core/json"
 import { foldCaseWhitespace } from "@mailwoman/normalize/fold"
-import { join } from "path-ts"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
 import { createNewlineWriter, JSONSpliterator } from "spliterator"
 import { Globerator } from "spliterator/node/fs"
 
@@ -49,7 +50,7 @@ import type { GoldenCandidateEntry as GoldenEntry } from "#utils/golden"
 const MIN_GOLDEN_COMPONENTS = 5
 
 /**
- * Components an existing entry must still carry to survive promotion.
+ * Components an existing entry must still include to survive promotion.
  */
 const MIN_PROMOTABLE_COMPONENTS = 4
 
@@ -67,7 +68,7 @@ export interface PromoteStats {
 
 export interface PromoteGoldenOptions {
 	/**
-	 * Candidates JSONL (required).
+	 * Candidates jsonl (required).
 	 */
 	input: string
 	/**
@@ -75,13 +76,17 @@ export interface PromoteGoldenOptions {
 	 */
 	bumpTo: string
 	/**
-	 * Previous version to forward-copy + dedup against. Default `v0.1.0`.
+	 * Previous version to forward-copy + dedup against.
+	 *
+	 * Default `v0.1.0`.
 	 */
 	prior?: string
 	/**
-	 * Golden dir root. Default `data/eval/golden`.
+	 * Golden dir root.
+	 *
+	 * Default `data/eval/golden`.
 	 */
-	goldenRoot?: string
+	goldenRoot?: PathBuilderLike
 	/**
 	 * Skip the human-typed-likelihood filters (keep everything that passed expand-golden's validator).
 	 */
@@ -95,8 +100,8 @@ export interface PromoteGoldenOptions {
 // ── Filters ───────────────────────────────────────────────────────────────
 
 /**
- * Heuristic: an address with 5+ components but fewer than 2 separators (commas/newlines/dashes) is most likely glued
- * together rather than human-typed.
+ * Heuristic: an address with 5+ components but fewer than 2 separators (commas/newlines/dashes)
+ * is most likely glued together rather than human-typed.
  */
 function isComponentsGlued(entry: GoldenEntry): boolean {
 	const componentCount = Object.keys(entry.components).length
@@ -108,9 +113,11 @@ function isComponentsGlued(entry: GoldenEntry): boolean {
 }
 
 /**
- * Heuristic: in US/UK conventions, postcode goes at the END of the address. If postcode appears in the first third of a
- * multi-component raw AND there are 4+ components, the LLM probably over-aggressively reordered. FR is exempt (postcode
- * often precedes locality there).
+ * Heuristic: in US/UK conventions, postcode goes at the END of the address.
+ *
+ * If postcode appears in the first third of a multi-component raw and there are 4+
+ * components, the LLM probably over-aggressively reordered.
+ * FR is exempt (postcode often precedes locality there).
  */
 function isPostcodeBadlyLeading(entry: GoldenEntry): boolean {
 	if (Object.keys(entry.components).length < MIN_PROMOTABLE_COMPONENTS) return false
@@ -127,8 +134,8 @@ function isPostcodeBadlyLeading(entry: GoldenEntry): boolean {
 }
 
 /**
- * Heuristic: catch-all for visually-bad outputs — unmatched brackets, control chars, suspicious punctuation that
- * suggests the LLM emitted markup instead of an address.
+ * Heuristic: catch-all for visually-bad outputs — unmatched brackets, control chars,
+ * suspicious punctuation that suggests the LLM emitted markup instead of an address.
  */
 function isSuspicious(entry: GoldenEntry): boolean {
 	const raw = entry.raw
@@ -153,29 +160,30 @@ export async function promoteGolden(
 	report?: (line: string) => void
 ): Promise<PromoteStats> {
 	const prior = options.prior ?? "v0.1.0"
-	const goldenRoot = options.goldenRoot ?? "data/eval/golden"
+	const goldenRoot = PathBuilder.from(options.goldenRoot ?? "data/eval/golden")
 	const applyFilters = !options.noFilters
 	const dryRun = options.dryRun ?? false
 
 	report?.(`reading candidates: ${options.input}`)
 
-	// The candidates file is the REQUIRED input, so its absence must not read as "zero candidates" —
-	// that promotes the prior version unchanged and reports success. `JSONSpliterator.fromAsync`
-	// reports a missing path as a bare `TypeError` naming its own internals, so name the path here.
+	// The candidates file is the required input, so its absence must not read as "zero candidates".
+	// That promotes the prior version unchanged and reports success.
+	// `JSONSpliterator.fromAsync` reports a missing path as a bare `TypeError`
+	// naming its own internals, so name the path here.
 	if (!(await pathExists(options.input))) {
 		throw new Error(`Candidates file not found: ${options.input}`)
 	}
 
-	// Forward-copy base: existing entries from the prior golden version go forward verbatim,
+	// Forward-copy base: existing entries from the prior golden version go forward verbatim.
 	// and we dedupe new candidates against them so v_new = v_old ∪ accepted_candidates.
-	const priorDir = join(goldenRoot, prior)
+	const priorDir = goldenRoot(prior)
 	const priorEntries: { country: string; entries: GoldenEntry[] }[] = []
 	const seenNormalized = new Set<string>()
 
 	if (await pathExists(priorDir)) {
-		for await (const f of Globerator.files("jsonl", { cwd: priorDir, absolute: false })) {
+		for await (const f of Globerator.files("jsonl", { cwd: priorDir, absolute: false, recursive: false })) {
 			const country = f.replace(".jsonl", "").toUpperCase()
-			const entries = await JSONSpliterator.fromAsync<GoldenEntry>(join(priorDir, f)).toArray()
+			const entries = await JSONSpliterator.fromAsync<GoldenEntry>(priorDir(f)).toArray()
 
 			priorEntries.push({ country, entries })
 
@@ -252,7 +260,8 @@ export async function promoteGolden(
 	const buckets = new Map<string, GoldenEntry[]>()
 
 	for (const { country, entries } of priorEntries) {
-		// Existing files keyed by filename uppercase (us.jsonl → US, adversarial.jsonl → ADVERSARIAL)
+		// The filename supplies the key for existing files after uppercasing
+		// (us.jsonl → US, adversarial.jsonl → adversarial)
 		buckets.set(country, [...entries])
 	}
 
@@ -267,7 +276,7 @@ export async function promoteGolden(
 	}
 
 	// Output
-	const outDir = join(goldenRoot, options.bumpTo)
+	const outDir = goldenRoot(options.bumpTo)
 	report?.(`=== plan ===`)
 	report?.(`output dir: ${outDir}${dryRun ? " (dry-run)" : ""}`)
 
@@ -311,29 +320,29 @@ export async function promoteGolden(
 
 	for (const [key, entries] of buckets) {
 		const filename = `${key.toLowerCase()}.jsonl`
-		const path = join(outDir, filename)
+		const path = outDir(filename)
 
 		{
 			await using out = createNewlineWriter(path)
 
 			for (const entry of entries) {
-				await out.write(JSON.stringify(entry))
+				await out.write(stringifyJSON(entry))
 			}
 		}
 
 		manifest.files[filename] = { entries: entries.length, sha256: await sha256File(path) }
 	}
 
-	// Forward-copy non-.jsonl files (README.md, etc.) from prior
+	// Forward-copy non-.jsonl files (readme.md, etc.) from prior
 	if (await pathExists(priorDir)) {
 		for await (const f of Globerator.from("*", { cwd: priorDir, absolute: false })) {
 			if (f.endsWith(".jsonl")) continue
-			await copyFileTo(join(priorDir, f), join(outDir, f))
+			await copyFileTo(priorDir(f), outDir(f))
 			report?.(`  forward-copied: ${f}`)
 		}
 	}
 
-	await writeLocalJSONFile(manifest, join(outDir, "MANIFEST.json"))
+	await writeLocalJSONFile(manifest, outDir("MANIFEST.json"))
 	report?.(`✓ promoted to ${outDir}`)
 
 	return stats