mailwoman 10.0.0 → 10.2.0

package/README.md

@@ -15,8 +15,8 @@ npx mailwoman parse "1600 Amphitheatre Parkway, Mountain View, CA 94043"
 ```ts
 // Library — parse programmatically
 import { decodeAsJSON } from "@mailwoman/core/decoder"
-import { createRuntimePipeline } from "mailwoman"
 import { NeuralAddressClassifier } from "@mailwoman/neural"
+import { createRuntimePipeline } from "mailwoman"
 
 const classifier = await NeuralAddressClassifier.loadFromWeights({ locale: "en-US" })
 const parse = createRuntimePipeline({ classifier })
@@ -29,7 +29,7 @@ console.log(decodeAsJSON(tree))
 
 ## What it does
 
-Mailwoman is **not** an LLM and nothing about it is generative. It is a small
+Mailwoman is **not** an LLM and none of it is generative. It is a small
 transformer encoder (~30M params) doing BIO token classification over a 33-label
 address schema — boring NER, which is a feature for short, structured strings.
 
@@ -91,8 +91,8 @@ classification with production-ready defaults.
 
 ```ts
 import { decodeAsJSON, decodeAsTuples } from "@mailwoman/core/decoder"
-import { createRuntimePipeline } from "mailwoman"
 import { NeuralAddressClassifier } from "@mailwoman/neural"
+import { createRuntimePipeline } from "mailwoman"
 
 const classifier = await NeuralAddressClassifier.loadFromWeights({ locale: "en-US" })
 const parse = createRuntimePipeline({ classifier })
@@ -132,7 +132,7 @@ for (const root of result.tree.roots) {
 
 Confidence calibration ships with the weights: `loadFromWeights` applies the bundled
 per-locale calibrator, so node confidences are calibrated probabilities (a `0.88` is
-right about 88% of the time), not raw scores.
+right about 88% of the time) rather than raw scores.
 
 ### Options
 
@@ -143,22 +143,22 @@ ones you'll reach for most:
 const parse = createRuntimePipeline({
 	classifier,
 	resolver, // optional — see "Geocoding" below
-	normalizeCase: true, // title-case detected all-caps input before the model (default: off)
+	caseNormalization: "title-case", // title-case detected all-caps input before the model (the default)
 })
 
 await parse("350 5TH AVE, NEW YORK, NY 10118", {
 	locale: "en-US", // assert a locale instead of detecting it
-	hardPlaceCountry: true, // confine resolution to a confidently-detected country (default: on)
+	placerCountryUse: "filter", // confine resolution to a confidently-detected country (the default)
 })
 ```
 
-## Geocoding
+## Geocode addresses
 
 Resolution turns parsed components into a Who's On First place ID and coordinate. It needs a
 gazetteer SQLite database — build one with `mailwoman gazetteer build admin` +
 `mailwoman gazetteer build fts`, or point at a prebuilt database. The resolver is
-administrative/postcode-level, not rooftop: it returns place centroids (locality, region,
-postcode), not delivery-point coordinates.
+administrative/postcode-level rather than rooftop: it returns place centroids (locality, region,
+postcode) rather than delivery-point coordinates.
 
 ```bash
 # CLI — resolve while parsing, or geocode directly
@@ -175,7 +175,7 @@ for the worked example.
 
 ### Which place the coordinate comes from
 
-When nothing finer than an administrative centroid resolved, result assembly walks a ladder and takes the first rung that has a coordinate. That ladder is **not fixed** — it leads with the postcode when the postcode is the finer thing, and with the locality otherwise:
+When no place finer than an administrative centroid resolved, result assembly walks a ladder and takes the first rung that has a coordinate. That ladder is **not fixed** — it leads with the postcode when the postcode is the finer thing, and with the locality otherwise:
 
 ```
 GB unit / NL PC6 / CA urban LDU   →   postcode, locality, dependent_locality, region, country
@@ -215,7 +215,7 @@ package is the umbrella that wires them together as a single `npm install`.
 | `@mailwoman/kind-classifier`      | Stage 2.5: query kind classification               |
 | `@mailwoman/phrase-grouper`       | Stage 2.7: phrase boundary discovery               |
 | `@mailwoman/codex`                | Postal reference data                              |
-| `@mailwoman/corpus`               | Training corpus pipeline                           |
+| `@mailwoman/corpus`               | Corpus training pipeline                           |
 | `@mailwoman/spatial`              | Spatial utilities                                  |
 | `@mailwoman/formatter`            | Address formatting + match key                     |
 | `@mailwoman/record`               | Record schema + normalizers                        |
@@ -231,7 +231,7 @@ package is the umbrella that wires them together as a single `npm install`.
 
 ## License
 
-Dual-licensed: **[AGPL-3.0-only](https://www.gnu.org/licenses/agpl-3.0.html)** for
+Dual-licensed: **[AGPL-3.0-only](https://www.gnu.org/licenses/AGPL-3.0.html)** for
 open-source use, or a **commercial license** for closed-source use without the AGPL's
 source-sharing obligation (contact `teffen@sister.software`). Portions derived from
 [Pelias Parser](https://github.com/pelias/parser) remain under MIT, and `@mailwoman/core`

package/cli/commands/clients/generate.tsx

@@ -0,0 +1,79 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman clients generate` — emit all four surfaces' OpenAPI documents, generate a Python package and
+ *   a Rust crate, then verify both build. Output lands under the gitignored `clients-build/`, and no
+ *   generated file is committed.
+ */
+
+import { Box, Text } from "ink"
+
+import {
+	CheckList,
+	type CommandSpec,
+	CommandTaskResult,
+	type CommandComponent,
+	phaseReporter,
+	useCommandTask,
+} from "#cli-kit"
+
+export const description = "Generate + verify the Python and Rust API clients from the emitted OpenAPI specs"
+
+/**
+ * Native command-line interface consumed by the filesystem command router.
+ */
+export const spec = {
+	name: "generate",
+	description,
+	options: {
+		"out-dir": { type: "string", description: "Output root. Default <repo>/clients-build (gitignored)" },
+		"skip-verify": {
+			type: "boolean",
+			default: false,
+			description: "Skip client build verification (development only)",
+		},
+	},
+} as const satisfies CommandSpec
+
+const ClientsGenerate: CommandComponent<typeof spec> = ({ options }) => {
+	const state = useCommandTask(
+		async () => {
+			const { generateClients } = await import("#tools/generate-clients")
+
+			return generateClients({
+				outDir: options.outDir,
+				skipVerify: options.skipVerify,
+				onPhase: phaseReporter(),
+			})
+		},
+		(result) => (result.ok ? 0 : 1)
+	)
+
+	// Progress streams to stderr until the summary lands.
+	// `CommandTaskResult` renders that summary.
+	if (state.status !== "done") return <CommandTaskResult state={state} />
+
+	const { ok, checks, receipt } = state.result
+
+	return (
+		<Box flexDirection="column">
+			<CheckList checks={checks} verdict={ok} />
+			{ok && (
+				<Box flexDirection="column" marginTop={1}>
+					<Text>version: {receipt.version}</Text>
+					<Text>specs: {receipt.specsDir}</Text>
+					<Text>
+						python: {receipt.pythonDir}
+						{receipt.pythonWheel ? ` (${receipt.pythonWheel})` : ""}
+					</Text>
+					<Text>rust: {receipt.rustDir}</Text>
+					<Text>elapsed: {receipt.elapsedSeconds.toFixed(1)}s</Text>
+				</Box>
+			)}
+		</Box>
+	)
+}
+
+export default ClientsGenerate

package/cli/commands/corpus/align-slice.tsx

@@ -0,0 +1,47 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Align one canonical recipe output with the current tokenizer.
+ */
+
+import { isCorpusVersion, type CommandSpec, CommandTaskResult, type CommandComponent, useCommandTask } from "#cli-kit"
+
+/**
+ * Native command-line interface consumed by the filesystem command router.
+ */
+export const spec = {
+	name: "align-slice",
+	description: "Align a canonical recipe output with the current tokenizer.",
+	options: {
+		input: { type: "string", required: true, description: "Canonical jsonl input" },
+		out: { type: "string", required: true, description: "Labeled jsonl output", deprecatedName: "output" },
+		"corpus-version": {
+			type: "string",
+			required: true,
+			description: "Corpus version stamp for the emitted rows",
+			validate: isCorpusVersion,
+			validationMessage:
+				"--corpus-version is the version alone, without the `corpus-v` prefix, as `0.7.0` or `0.7.0-de-holdout`.",
+		},
+	},
+} as const satisfies CommandSpec
+
+const Cmd: CommandComponent<typeof spec> = ({ options }) => {
+	const state = useCommandTask(async () => {
+		const { alignCanonicalRows } = await import("@mailwoman/corpus/tools/align-canonical")
+
+		await alignCanonicalRows({
+			input: options.input,
+			output: options.out,
+			corpusVersion: options.corpusVersion,
+		})
+
+		return "done"
+	})
+
+	return <CommandTaskResult state={state} />
+}
+
+export default Cmd

package/cli/commands/corpus/audit.tsx

@@ -0,0 +1,47 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman corpus audit <corpus-dir>` — per-source parquet-file count vs `source_weight` diagnostic.
+ *   Pair with `--config` to weight the counts by a training YAML's source_weights block.
+ */
+
+import { type CommandSpec, CommandTaskResult, type CommandComponent, useCommandTask } from "#cli-kit"
+
+/**
+ * Native command-line interface consumed by the filesystem command router.
+ */
+export const spec = {
+	name: "audit",
+	description: "Audit corpus parquet-file counts against source weights.",
+	positionals: [
+		{
+			name: "corpus-dir",
+			required: true,
+			description: "Corpus directory (MANIFEST.json or train/val/test parquet files)",
+		},
+	],
+	options: {
+		config: { type: "string", description: "Training YAML whose source_weights pair with the file counts" },
+		sample: {
+			type: "number",
+			default: 100,
+			description: "Max parquet files sampled per split when scanning without a MANIFEST",
+		},
+	},
+} as const satisfies CommandSpec
+
+const CorpusAudit: CommandComponent<typeof spec, [string]> = ({ options, args }) => {
+	const state = useCommandTask(async () => {
+		const { audit } = await import("@mailwoman/corpus/tools/audit")
+
+		audit({ corpusDir: args[0], configPath: options.config, sampleFileCount: options.sample })
+	})
+
+	if (state.status !== "done") return <CommandTaskResult state={state} />
+
+	return null
+}
+
+export default CorpusAudit

package/cli/commands/corpus/build.tsx

@@ -0,0 +1,263 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   End-to-end corpus build: drives every registered adapter (or the filtered subset) from `--inputs`.
+ *   It runs synthesis and alignment. It computes the locality-holdout split and writes the final jsonl
+ *   and parquet files plus per-stage manifests under `<out>/corpus-v<version>/`.
+ *
+ *   Adapters whose id is missing from `--inputs` are skipped and noted in the manifest. This lets the
+ *   CLI handles partial builds during development.
+ *
+ *   A build whose `--inputs` names `wof-admin` or `wof-postalcode` needs
+ *   `NODE_OPTIONS="--max-old-space-size=20480"`. Both adapters hold every source record in memory.
+ *   Node's default heap makes the process abort with SIGABRT during the adapter phase. This command
+ *   refuses such a launch through `assertHeapForAdapters`. `buildCorpus` prints the heap limit it has.
+ */
+
+import { isAlpha2CodeShape } from "@mailwoman/codex/country"
+import { CommandError } from "@mailwoman/core/scripting/command"
+import { heapLimitBytes } from "@mailwoman/core/utils/system"
+import type { BuildStage } from "@mailwoman/corpus"
+import { BuildProfile } from "@mailwoman/corpus/build/types"
+import type { AdapterOptions } from "@mailwoman/corpus/types"
+import { LicensePolicy } from "@mailwoman/corpus/utils/license"
+import { Box, Text } from "ink"
+import { useState } from "react"
+
+import { isCorpusVersion, type CommandSpec, CommandTaskResult, type CommandComponent, useCommandTask } from "#cli-kit"
+
+/**
+ * The accepted `--license-policy` values, for the flag's validation message.
+ *
+ * Derived from {@linkcode LicensePolicy} so a value added there reaches the CLI.
+ */
+const LICENSE_POLICIES: readonly LicensePolicy[] = Object.values(LicensePolicy)
+
+/**
+ * The accepted `--profile` values, for the flag's validation message.
+ *
+ * Derived from {@linkcode BuildProfile} so a value added there reaches the CLI.
+ */
+const BUILD_PROFILES: readonly BuildProfile[] = Object.values(BuildProfile)
+
+/**
+ * `--inputs` accepts a bare path string when an adapter needs no extra options.
+ *
+ * An `AdapterOptions` object supplies options such as an OpenAddresses country filter or a fixture `limit`.
+ */
+export const spec = {
+	name: "build",
+	description: "Build a versioned corpus.",
+	options: {
+		"corpus-version": {
+			type: "string",
+			default: "0.1.0-dev",
+			description: "Corpus version",
+			validate: isCorpusVersion,
+			validationMessage:
+				"--corpus-version is the version alone, without the `corpus-v` prefix, as `0.7.0` or `0.7.0-de-holdout`.",
+		},
+		out: { type: "string", required: true, description: "Output root", deprecatedName: "output" },
+		inputs: {
+			type: "string",
+			required: true,
+			description:
+				"Adapter inputs: a path to a committed record under `packages/corpus/data/builds/<version>/inputs.json`, " +
+				"or a JSON map from adapter id to an input path or options object",
+		},
+		synthesize: { type: "boolean", default: true, description: "Enable augmentation" },
+		"rows-per-file": {
+			type: "number",
+			default: 1_000_000,
+			validate: (value) => Number.isInteger(value) && value > 0,
+			validationMessage: "--rows-per-file must be a positive integer.",
+			description: "Max rows per parquet file",
+			deprecatedName: "rows-per-slice",
+		},
+		"shuffle-window": {
+			type: "number",
+			validate: (value) => Number.isInteger(value) && value >= 0,
+			validationMessage: "--shuffle-window must be a non-negative integer.",
+			description: "Rows held in memory while shuffling each split. 0 writes arrival order",
+		},
+		"shuffle-seed": {
+			type: "number",
+			validate: (value) => Number.isInteger(value) && value >= 0,
+			validationMessage: "--shuffle-seed must be a non-negative integer.",
+			description: "Seed for --shuffle-window",
+		},
+		profile: {
+			type: "string",
+			default: BuildProfile.Exploratory,
+			validate: (value): boolean => BUILD_PROFILES.includes(value as BuildProfile),
+			validationMessage: `--profile must be one of ${BUILD_PROFILES.join(", ")}.`,
+			description:
+				"Whether a row's source must be eligible for ingest before the row enters: " +
+				"exploratory admits every row an adapter yields, and release-eligible admits a row only " +
+				"when the address-source register marks its source eligible",
+		},
+		"license-policy": {
+			type: "string",
+			default: LicensePolicy.All,
+			validate: (value): boolean => LICENSE_POLICIES.includes(value as LicensePolicy),
+			validationMessage: `--license-policy must be one of ${LICENSE_POLICIES.join(", ")}.`,
+			description:
+				"Which rows to admit on the evidence of their license obligations: " +
+				"all, share-alike-free (refuse a license carrying or mentioning share-alike), " +
+				"or resolved-only (also refuse a license resolving to no SPDX expression)",
+		},
+		"exclude-licenses": {
+			type: "string",
+			description: "Comma-separated license prefixes to refuse, e.g. ODbL,CC-BY-SA",
+		},
+	},
+} as const satisfies CommandSpec
+
+type AdapterInput = string | AdapterOptions
+
+function isAdapterInputMap(input: unknown): input is Record<string, AdapterInput> {
+	if (typeof input !== "object" || !input || Array.isArray(input)) return false
+
+	return Object.values(input).every((value) => {
+		if (typeof value === "string") return true
+
+		if (typeof value !== "object" || value === null || Array.isArray(value)) return false
+
+		if (!("inputPath" in value) || typeof value.inputPath !== "string") return false
+
+		if ("outputDir" in value && value.outputDir !== undefined && typeof value.outputDir !== "string") return false
+
+		// Check the shape because a lower-case `nl` is still a string.
+		// The four adapters that filter per row compare it with the row's upper-case code.
+		// A lower-case value selects zero rows and reports no matches.
+		if ("country" in value && value.country !== undefined && !isAlpha2CodeShape(value.country)) return false
+
+		return (
+			!("limit" in value) ||
+			value.limit === undefined ||
+			(typeof value.limit === "number" && Number.isInteger(value.limit) && value.limit > 0)
+		)
+	})
+}
+
+/**
+ * Read the inline `--inputs` JSON map.
+ *
+ * @throws {CommandError} When the text is not JSON, or an entry is not an input path or options object.
+ */
+function readInlineAdapterInputs(
+	text: string,
+	parseJSONStrict: (text: string) => unknown
+): Record<string, AdapterOptions> {
+	let parsed: unknown
+
+	try {
+		parsed = parseJSONStrict(text)
+
+		if (!isAdapterInputMap(parsed)) throw new TypeError("expected an adapter-id to input map")
+	} catch (error) {
+		throw new CommandError(`invalid --inputs JSON: ${(error as Error).message}`)
+	}
+
+	return Object.fromEntries(
+		Object.entries(parsed).map(([id, value]) => [id, typeof value === "string" ? { inputPath: value } : value])
+	)
+}
+
+const CorpusBuild: CommandComponent<typeof spec> = ({ options }) => {
+	const [stage, setStage] = useState<{ name: BuildStage; message: string }>()
+
+	const state = useCommandTask(async () => {
+		const { parseJSONStrict } = await import("@mailwoman/core/json")
+		const { assertHeapForAdapters, buildCorpus, defaultAdapterRegistry } = await import("@mailwoman/corpus")
+		const { readBuildInputs } = await import("@mailwoman/corpus/build/inputs")
+		const { compileLicenseExcludes } = await import("@mailwoman/corpus/utils/license")
+
+		// A record's `inputPath` is data-root-relative and a JSON map's is absolute,
+		// so the two forms are read by different functions rather than normalized into one.
+		const adapterInputs = options.inputs.trimStart().startsWith("{")
+			? readInlineAdapterInputs(options.inputs, parseJSONStrict)
+			: await readBuildInputs(options.inputs).catch((error: Error) => {
+					throw new CommandError(`--inputs ${options.inputs} could not be read as a build record: ${error.message}`)
+				})
+
+		const adapters = defaultAdapterRegistry.list()
+
+		// The refusal lives at the launch rather than in `buildCorpus`, because the library's
+		// own tests run the WOF adapter against a five-kilobyte fixture under the default heap.
+		assertHeapForAdapters(Object.keys(adapterInputs), heapLimitBytes())
+
+		const m = await buildCorpus({
+			outputDir: options.out,
+			corpusVersion: options.corpusVersion,
+			adapters,
+			adapterInputs,
+			synthesize: options.synthesize,
+			rowsPerFile: options.rowsPerFile,
+			shuffleWindow: options.shuffleWindow,
+			shuffleSeed: options.shuffleSeed,
+			profile: options.profile as BuildProfile,
+			licensePolicy: options.licensePolicy as LicensePolicy,
+			excludeLicenses: options.excludeLicenses ? compileLicenseExcludes(options.excludeLicenses) : undefined,
+			onProgress: (name, message) => setStage({ name, message }),
+		})
+
+		return {
+			total: m.slices.total_rows,
+			aligned: m.total_aligned_rows,
+			quarantined: m.quarantine_count,
+			adapters: m.adapters.length,
+			refusedByLicense: m.excluded_by_license,
+			refusedKinds: m.refused_by_license_kind,
+			unresolvedLicenseRows: m.admitted_unresolved_license_rows,
+			refusedByEligibility: m.excluded_by_eligibility,
+			ineligibleSources: Object.keys(m.ineligible_sources),
+		}
+	})
+
+	if (state.status === "error") return <CommandTaskResult state={state} />
+
+	if (state.status === "done") {
+		const done = state.result
+
+		return (
+			<Box flexDirection="column">
+				<Text>
+					corpus-v{options.corpusVersion}: <Text color="green">{done.total}</Text> rows ({done.adapters} adapters,{" "}
+					<Text dimColor>{done.quarantined} quarantined</Text>)
+				</Text>
+				<Text dimColor>
+					license policy {options.licensePolicy}: {done.refusedByLicense} rows refused
+					{Object.keys(done.refusedKinds).length
+						? ` (${Object.entries(done.refusedKinds)
+								.map(([kind, rows]) => `${kind}=${rows}`)
+								.join(", ")})`
+						: ""}
+					, {done.unresolvedLicenseRows} admitted whose license resolves to no expression
+				</Text>
+				{done.ineligibleSources.length ? (
+					<Text color="yellow">
+						profile {options.profile}: {done.refusedByEligibility} rows refused from {done.ineligibleSources.join(", ")}
+						. MANIFEST.json records each reason under `ineligible_sources`.
+					</Text>
+				) : null}
+				<Text dimColor>{options.out}</Text>
+			</Box>
+		)
+	}
+
+	return (
+		<Box flexDirection="column">
+			<Text>building corpus-v{options.corpusVersion}…</Text>
+			{stage ? (
+				<Text dimColor>
+					[{stage.name}] {stage.message}
+				</Text>
+			) : null}
+		</Box>
+	)
+}
+
+export default CorpusBuild

package/cli/commands/corpus/download.tsx

@@ -0,0 +1,130 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman corpus download` — pull corpus + tokenizer from Cloudflare R2 via rclone.
+ *
+ *   Intended for GPU provider instances. Pulls the versioned corpus, tokenizer and training code
+ *   from R2 at datacenter speed (~1-10 Gbps depending on provider locality). It also works locally for
+ *   syncing a fresh checkout.
+ *
+ *   Requires RCLONE_S3_* env vars (Cloudflare R2 credentials).
+ */
+
+import { dataRootPath } from "@mailwoman/core/data-root"
+import { Box, Text } from "ink"
+import { useState } from "react"
+
+import { type CommandSpec, type CommandComponent, useCommandTask } from "#cli-kit"
+
+const DEFAULT_BUCKET = "mailwoman-assets"
+
+/**
+ * Native command-line interface consumed by the filesystem command router.
+ */
+export const spec = {
+	name: "download",
+	description: "Download corpus artifacts from R2.",
+	options: {
+		bucket: { type: "string", default: DEFAULT_BUCKET, description: "R2 bucket" },
+		"out-dir": { type: "string", default: dataRootPath().toString(), description: "Local output root" },
+		"dry-run": { type: "boolean", default: false, description: "Show without downloading" },
+	},
+} as const satisfies CommandSpec
+
+interface Step {
+	label: string
+	status: "pending" | "running" | "done" | "error"
+	detail?: string
+}
+
+const CorpusDownload: CommandComponent<typeof spec> = ({ options }) => {
+	const [steps, setSteps] = useState<Step[]>([
+		{ label: "Download corpus v0.3.0", status: "pending" },
+		{ label: "Download corpus v0.4.0", status: "pending" },
+		{ label: "Download tokenizer", status: "pending" },
+		{ label: "Download training code", status: "pending" },
+	])
+
+	const updateStep = (idx: number, update: Partial<Step>) => {
+		setSteps((prev) => prev.map((s, i) => (i === idx ? { ...s, ...update } : s)))
+	}
+
+	useCommandTask(async () => {
+		const { $ } = await import("zx")
+
+		const rcloneBase = `:s3:${options.bucket}`
+		const dryFlag = options.dryRun ? "--dry-run" : ""
+		const out = options.outDir
+
+		// Step 0: Download v0.3.0 corpus
+		updateStep(0, { status: "running" })
+
+		try {
+			await $`rclone sync ${rcloneBase}/corpus/v0.3.0/ ${out}/corpus/versioned/v0.3.0/corpus-v0.3.0/ --progress --transfers 8 --checkers 16 ${dryFlag}`.quiet()
+			updateStep(0, { status: "done" })
+		} catch (error: unknown) {
+			const e = error as Record<string, unknown>
+			updateStep(0, { status: "error", detail: String(e.stderr ?? e.message ?? error).slice(0, 100) })
+
+			return
+		}
+
+		// Step 1: Download the v0.4.0 adapter parquet files
+		updateStep(1, { status: "running" })
+
+		try {
+			await $`rclone sync ${rcloneBase}/corpus/v0.4.0/ ${out}/corpus/versioned/v0.4.0/corpus-v0.4.0/ --progress --transfers 4 ${dryFlag}`.quiet()
+			updateStep(1, { status: "done" })
+		} catch (error: unknown) {
+			const e = error as Record<string, unknown>
+			updateStep(1, { status: "error", detail: String(e.stderr ?? e.message ?? error).slice(0, 100) })
+
+			return
+		}
+
+		// Step 2: Download tokenizer
+		updateStep(2, { status: "running" })
+
+		try {
+			await $`rclone sync ${rcloneBase}/models/tokenizer/ ${out}/models/tokenizer/ --progress ${dryFlag}`.quiet()
+			updateStep(2, { status: "done" })
+		} catch (error: unknown) {
+			const e = error as Record<string, unknown>
+			updateStep(2, { status: "error", detail: String(e.stderr ?? e.message ?? error).slice(0, 100) })
+
+			return
+		}
+
+		// Step 3: Download training code
+		updateStep(3, { status: "running" })
+
+		try {
+			await $`rclone sync ${rcloneBase}/corpus-python/ ./corpus-python/ --progress ${dryFlag}`.quiet()
+			updateStep(3, { status: "done" })
+		} catch (error: unknown) {
+			const e = error as Record<string, unknown>
+			updateStep(3, { status: "error", detail: String(e.stderr ?? e.message ?? error).slice(0, 100) })
+		}
+	})
+
+	return (
+		<Box flexDirection="column">
+			<Text bold>Corpus Download ← R2 ({options.bucket})</Text>
+			{Boolean(options.dryRun) && <Text color="yellow">DRY RUN — no files will be transferred</Text>}
+			<Text> </Text>
+			{steps.map((step, i) => (
+				<Box key={i}>
+					<Text>
+						{step.status === "done" ? "✓" : step.status === "running" ? "◼" : step.status === "error" ? "✗" : "○"}{" "}
+						{step.label}
+						{step.detail ? ` — ${step.detail}` : ""}
+					</Text>
+				</Box>
+			))}
+		</Box>
+	)
+}
+
+export default CorpusDownload