mailwoman 8.6.0 → 9.0.0

package/README.md

@@ -170,7 +170,7 @@ Programmatically, build a `WofSqlitePlaceLookup` backend (from
 `@mailwoman/resolver-wof-sqlite`), pass it to `createWOFResolver` (from `@mailwoman/resolver`),
 and hand the resolver to `createRuntimePipeline({ classifier, resolver })`. The resolved
 `result.tree` roots then carry a `wof:id` and coordinate. See
-[Getting started → Adding resolution](https://mailwoman.sister.software/articles/getting-started/)
+[Getting started → Adding resolution](https://mailwoman.sister.software/docs/developers/get-started/install-and-first-parse)
 for the worked example.
 
 ## Architecture
@@ -210,7 +210,7 @@ package is the umbrella that wires them together as a single `npm install`.
 ## Related
 
 - [Documentation & blog](https://mailwoman.sister.software)
-- [What Mailwoman Is](https://mailwoman.sister.software/articles/concepts/what-mailwoman-is/)
+- [What Mailwoman Is](https://mailwoman.sister.software/docs/developers/get-started/what-mailwoman-is)
 - [GitHub](https://github.com/sister-software/mailwoman)
 
 ## License

package/api-engine.ts

@@ -8,7 +8,7 @@
  *   `/api/batch`, `/api/resolve-tree`, `/api/reload` handlers, `AddressRouter`'s retired `/parse`
  *   handler, and `HealthRouter`'s `/health` data block — onto the engine-agnostic `@mailwoman/api`
  *   contract's live routes (`/v1/parse`, `/v1/geocode`, `/v1/batch`, `/v1/resolve`, `/v1/reload`).
- *   `mailwoman/server/` is deleted (Task 2) — this file is its sole successor, a fresh port rather
+ *   `mailwoman/server/` is deleted — this file is its sole successor, a fresh port rather
  *   than a thin wrapper.
  *
  *   `createServeEngine` builds the shared stack ONCE, at boot, instead of express's lazy
@@ -26,14 +26,14 @@
  */
 
 import { existsSync, readdirSync, readFileSync } from "node:fs"
-import { createRequire } from "node:module"
-import { join } from "node:path"
+import { fileURLToPath } from "node:url"
 
 import type { BatchRow, GeocodeOutcome, HealthData, MailwomanAPIEngine, ResolveTreeOutcome } from "@mailwoman/api"
 import { recordTimed } from "@mailwoman/api-kit"
 import { decodeAsTuples, decodeAsXML } from "@mailwoman/core"
 import type { AddressTree } from "@mailwoman/core/decoder"
 import { $public } from "@mailwoman/core/env"
+import { tryParsingJSON } from "@mailwoman/core/objects"
 import { deriveInputMode } from "@mailwoman/core/pipeline"
 import { classifyKindSync } from "@mailwoman/kind-classifier"
 import { computeQueryShape } from "@mailwoman/query-shape"
@@ -48,7 +48,13 @@ import {
 	ShardProvider,
 } from "./geocode-core.ts"
 import { INTERP_RADIUS_CALIBRATION, interpCalibrationForRegion } from "./interp-calibration.ts"
-import { createResolverBackend, mailwomanDataRoot, resolveCandidateDBPath, wofShardPaths } from "./resolver-backend.ts"
+import {
+	buildNoGazetteerMessage,
+	createResolverBackend,
+	mailwomanDataRoot,
+	resolveCandidateDBPath,
+	wofShardPaths,
+} from "./resolver-backend.ts"
 
 /**
  * Default per-state shard root + interp calibration — mirrors the express server's defaults (`GeocodeRouter.ts`).
@@ -82,30 +88,13 @@ function wofPaths(): string[] {
 
 /**
  * #1009-style boot preflight message. Same shape as the drop-ins' (`photon/cli.ts`, `nominatim/cli.ts`) — a stranger's
- * first `mailwoman serve` must say exactly what data is missing and the one command that fixes it — adapted to this
- * package's own override (`MAILWOMAN_WOF_DB`, a comma-separated FTS shard list) alongside the shared
- * candidate-gazetteer fast path (`MAILWOMAN_CANDIDATE_DB`).
+ * first `mailwoman serve` must say exactly what data is missing and the one command that fixes it.
  */
 function buildPreflightMessage(): string {
-	const conventionCandidate = join(DATA_ROOT, "wof", "candidate.db")
-
-	return [
-		"✗ no gazetteer data found — `mailwoman serve` needs a WOF SQLite distribution to geocode/resolve.",
-		"",
-		"  Fastest path (worldwide resolution, ~1.4 GB, byte-range friendly):",
-		`    mkdir -p ${join(DATA_ROOT, "wof")}`,
-		`    curl -fSL https://public.sister.software/mailwoman/gazetteer/2026-07-07a/candidate.db \\`,
-		`      -o ${conventionCandidate}`,
-		"",
-		"  Then point `$MAILWOMAN_CANDIDATE_DB` at that file and re-run `serve`:",
-		`    export MAILWOMAN_CANDIDATE_DB=${conventionCandidate}`,
-		"",
-		"  Or point at your own gazetteer:",
-		"    $MAILWOMAN_WOF_DB=<path1,path2,...>    (admin WOF SQLite shard(s) — the FTS backend)",
-		"    $MAILWOMAN_CANDIDATE_DB=<path>         (candidate-table gazetteer — population-first, demo-parity)",
-		"",
-		"  Docs: https://mailwoman.sister.software/docs",
-	].join("\n")
+	return buildNoGazetteerMessage({
+		dataRoot: DATA_ROOT,
+		docsPath: "/docs/developers/get-started/ten-minute-trial",
+	})
 }
 
 /**
@@ -120,7 +109,13 @@ function readModelCard(): Record<string, unknown> | null {
 	}
 
 	try {
-		candidates.push(createRequire(import.meta.url).resolve("@mailwoman/neural-weights-en-us/model-card.json"))
+		// Native ESM resolution of the weights package's card. `@mailwoman/neural-weights-*` packages carry no `exports`
+		// map, so the subpath resolves as a plain file inside the package, and (unlike `node:module`'s
+		// `findPackageJSON`) `import.meta.resolve` realpaths through the workspace symlink — the same string the CJS
+		// `require.resolve` this replaced returned. It does NOT throw for a missing FILE inside a resolvable package,
+		// only for an unresolvable package; the `existsSync` below already gates every candidate, so that is a no-op
+		// here.
+		candidates.push(fileURLToPath(import.meta.resolve("@mailwoman/neural-weights-en-us/model-card.json")))
 	} catch {
 		/* package not resolvable from here — fall through */
 	}
@@ -129,9 +124,13 @@ function readModelCard(): Record<string, unknown> | null {
 
 	for (const p of candidates) {
 		try {
-			if (existsSync(p)) return JSON.parse(readFileSync(p, "utf8")) as Record<string, unknown>
+			if (existsSync(p)) {
+				const card = tryParsingJSON<Record<string, unknown>>(readFileSync(p, "utf8"))
+
+				if (card) return card
+			}
 		} catch {
-			/* unreadable / malformed — try the next candidate */
+			/* unreadable — try the next candidate */
 		}
 	}
 
@@ -325,7 +324,7 @@ export async function createServeEngine(): Promise<ServeEngine> {
 	// `onnxruntime-node`'s `session.run()` blocks the JS thread rather than releasing to the libuv pool,
 	// and `node:sqlite` reads are synchronous. The pool bought nothing but the appearance of tuning, so
 	// it's a plain loop now. To actually parallelize, cross a thread boundary — see
-	// `mailwoman/geocode-stream.ts`. Receipts: `docs/articles/plan/reference/performance.mdx`.
+	// `mailwoman/geocode-stream.ts`. Receipts: `docs/engineering/reference/performance.mdx`.
 	const batch: MailwomanAPIEngine["batch"] = async (addresses, opts) => {
 		// Decision A endpoint default: batch rows are the record register.
 		const inputMode = opts?.inputMode ?? "formatted"

package/cli-kit/dropin.ts

@@ -0,0 +1,195 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The shared skeleton behind the drop-in CLIs — `mailwoman-libpostal`, `mailwoman-nominatim`,
+ *   `mailwoman-photon`. All three are the same program with a different engine bolted in: dispatch
+ *   `serve` / `openapi` off the first positional, boot the neural classifier with a friendly
+ *   failure, (for the two geocoding drop-ins) locate a gazetteer with a friendly failure, then print
+ *   a banner. Those pieces were copied between the three files, and the copies were being kept in
+ *   sync BY HAND — nominatim/cli.ts said so in a comment ("same message shape as
+ *   @mailwoman/photon's pre-flight (kept in lockstep)"). This module is the lockstep.
+ *
+ *   WHY IT LIVES HERE and not in `@mailwoman/api-kit`, which already owns `printOpenAPIDocument` and
+ *   `serveNode` and would otherwise be the obvious home: the preflights need `@mailwoman/neural` and
+ *   this package's `resolver-backend`, and api-kit is engine-agnostic by charter. Splitting the
+ *   skeleton across two packages to preserve that charter costs more than the duplication did. All
+ *   three drop-ins already declare `mailwoman` as a dependency, so landing the whole thing here adds
+ *   no dependency to anyone.
+ *
+ *   WHY IT IS NOT IN `cli-kit/index.ts`: that barrel is the Pastel/Ink toolkit for
+ *   `mailwoman/commands/*` and imports `ink` + `react`. The drop-ins are plain `parseArgs` scripts
+ *   that render no UI, and `npx @mailwoman/libpostal serve` should not pay for a TUI runtime to
+ *   start an HTTP server. Hence a standalone module with its own `mailwoman/cli-kit/dropin`
+ *   subpath, deliberately NOT re-exported through the barrel.
+ */
+
+import { existsSync } from "node:fs"
+import { parseArgs } from "node:util"
+
+import { NeuralAddressClassifier } from "@mailwoman/neural"
+
+import {
+	buildNoGazetteerMessage,
+	mailwomanDataRoot,
+	resolveCandidateDBPath,
+	wofShardPaths,
+} from "../resolver-backend.ts"
+
+/**
+ * The docs page every drop-in's missing-gazetteer message points a stranger at (#1009). One constant so the three
+ * messages cannot drift onto different pages when the docs move.
+ */
+const GAZETTEER_DOCS_PATH = "/docs/developers/get-started/ten-minute-trial"
+
+/**
+ * Print an error and exit non-zero. Typed `never` so callers get definite-assignment narrowing after the call.
+ */
+function fail(message: string): never {
+	console.error(message)
+
+	process.exit(1)
+}
+
+/**
+ * Parse the `openapi` subcommand's flags, validating `--flavor`. Exits 1 with the binary's usage line on a bad flavor.
+ *
+ * Returns the shape `printOpenAPIDocument` takes, so a drop-in's `openapi` command is this call plus building its app
+ * around a stub engine — which is what keeps the command pure route-table introspection that never boots a classifier
+ * or opens a gazetteer.
+ */
+export function parseOpenAPIFlags(binaryName: string): { flavor?: string; out?: string } {
+	const { values } = parseArgs({
+		options: {
+			flavor: { type: "string", default: "3.1" },
+			out: { type: "string" },
+		},
+		allowPositionals: true,
+	})
+
+	if (values.flavor !== "3.1" && values.flavor !== "3.0") {
+		console.error(`✗ --flavor must be "3.1" or "3.0" (got "${values.flavor}")`)
+
+		fail(`Usage: ${binaryName} openapi [--flavor 3.1|3.0] [--out <path>]`)
+	}
+
+	return values
+}
+
+/**
+ * Load the en-US neural classifier, failing FRIENDLY (#1009).
+ *
+ * `resolveWeights` (`neural/weights.ts`) already names the exact fix command; this guard only keeps that message from
+ * being buried under an unhandled-rejection stack trace. Eager, so a missing-weights boot fails at startup rather than
+ * on the first request.
+ */
+export async function loadClassifierOrExit(): Promise<NeuralAddressClassifier> {
+	try {
+		return await NeuralAddressClassifier.loadFromWeights({ locale: "en-US" })
+	} catch (error) {
+		return fail(`✗ ${error instanceof Error ? error.message : String(error)}`)
+	}
+}
+
+/**
+ * Where a geocoding drop-in reads its places from.
+ */
+export interface GazetteerPaths {
+	/**
+	 * The candidate gazetteer (worldwide resolution), when one was found.
+	 */
+	candidateDb: string | undefined
+	/**
+	 * The WOF admin FTS shards that exist on disk.
+	 */
+	wofPaths: string[]
+	/**
+	 * The first shard — the admin DB the reverse geocoder opens. `undefined` when there are no shards.
+	 */
+	adminDBPath: string | undefined
+}
+
+/**
+ * Locate the gazetteer for a geocoding drop-in, with both of the #1009 friendly failures:
+ *
+ * - An EXPLICIT `--candidate-db` that does not exist errors loudly. It must never silently fall back to whatever ambient
+ *   data-root file happens to be present — a typo'd path would otherwise serve the wrong gazetteer without a word.
+ * - No candidate DB AND no shards prints the named-artifact message with the one command that fixes it, instead of
+ *   letting the resolver throw its internal "resolveShards: at least one shard is required".
+ */
+export function resolveGazetteerOrExit(candidateDBFlag: string | undefined): GazetteerPaths {
+	if (candidateDBFlag && !existsSync(candidateDBFlag)) {
+		fail(`✗ --candidate-db not found: ${candidateDBFlag}`)
+	}
+
+	const wofPaths = wofShardPaths().filter(existsSync)
+	// Candidate gazetteer = worldwide resolution (population-first ranking + global coverage + the FTS5-trigram typo
+	// fallback). --candidate-db, else $MAILWOMAN_CANDIDATE_DB, else the `<data-root>/wof/candidate.db` convention
+	// path. Absent → the admin FTS shards.
+	const candidateDb = resolveCandidateDBPath(candidateDBFlag)
+
+	if (!candidateDb && !wofPaths.length) {
+		fail(buildNoGazetteerMessage({ dataRoot: mailwomanDataRoot(), docsPath: GAZETTEER_DOCS_PATH }))
+	}
+
+	return { adminDBPath: wofPaths[0], candidateDb, wofPaths }
+}
+
+/**
+ * The `cors:` line of a drop-in's startup banner.
+ */
+export function corsBannerLine(cors: boolean): string {
+	return `  cors: ${cors ? "enabled (Access-Control-Allow-Origin: *)" : "disabled (--no-cors)"}`
+}
+
+/**
+ * The `wof:` + `resolver:` lines of a geocoding drop-in's startup banner — which gazetteer the process actually opened,
+ * and (when it fell back to admin-only) the flag that widens it.
+ */
+export function gazetteerBannerLines({ adminDBPath, candidateDb }: GazetteerPaths): string[] {
+	return [
+		`  wof: ${adminDBPath ?? "(none found — set MAILWOMAN_WOF_DB)"}`,
+		candidateDb
+			? `  resolver: candidate gazetteer (worldwide) — ${candidateDb}`
+			: `  resolver: admin-only (US-optimized) — point --candidate-db / $MAILWOMAN_CANDIDATE_DB at a candidate gazetteer for worldwide`,
+	]
+}
+
+/**
+ * A drop-in CLI's two subcommands.
+ */
+export interface DropInCLI {
+	/**
+	 * The binary name, for the usage message (e.g. `mailwoman-photon`).
+	 */
+	binaryName: string
+	/**
+	 * Usage lines for the subcommands, without the leading `Usage:` header.
+	 */
+	usage: string[]
+	serve: () => Promise<void>
+	openapi: () => void
+}
+
+/**
+ * Dispatch a drop-in CLI's subcommand off the first positional.
+ *
+ * `strict: false` because the per-command parsers own their own flags — this pass only reads the positional. An unknown
+ * command exits 1; a bare invocation prints usage and exits 0.
+ */
+export async function runDropInCLI({ binaryName, openapi, serve, usage }: DropInCLI): Promise<void> {
+	const command = parseArgs({ strict: false, allowPositionals: true }).positionals[0]
+
+	switch (command) {
+		case "serve":
+			await serve()
+			break
+		case "openapi":
+			openapi()
+			break
+		default:
+			console.error([`Usage: ${binaryName} <command>`, ...usage].join("\n"))
+			process.exit(command ? 1 : 0)
+	}
+}

package/cli-kit/index.ts

@@ -10,6 +10,7 @@
  *   importable under node's type stripping (the dev `node →` exports condition).
  */
 
+import { type PlacetypeRole, PlacetypeRoles } from "@mailwoman/core"
 import { Box, Text } from "ink"
 import { createElement as h, useEffect, useState } from "react"
 import type * as React from "react"
@@ -129,3 +130,29 @@ export function CheckList({ checks, verdict }: { checks: readonly Check[]; verdi
 
 	return h(Box, { flexDirection: "column" }, ...lines, summary)
 }
+
+/**
+ * Parse a `--roles a,b,c` flag into validated {@link PlacetypeRole}s, or `undefined` when the flag is absent (which
+ * every caller reads as "all roles").
+ *
+ * Rejects an unknown role with {@link commandError} rather than silently filtering it — a typo in a role name would
+ * otherwise produce an empty, entirely plausible-looking result.
+ */
+export function parseRoles(raw: string | undefined): PlacetypeRole[] | undefined {
+	if (!raw) return undefined
+
+	const valid = new Set<string>(PlacetypeRoles)
+
+	const parsed = raw
+		.split(",")
+		.map((s) => s.trim())
+		.filter(Boolean)
+
+	for (const role of parsed) {
+		if (!valid.has(role)) {
+			throw commandError(`Unknown placetype role '${role}'. Valid roles: ${PlacetypeRoles.join(", ")}.`)
+		}
+	}
+
+	return parsed as PlacetypeRole[]
+}

package/cli.ts

@@ -8,14 +8,15 @@
 
 import { readFileSync } from "node:fs"
 
+import { parseJSONStrict } from "@mailwoman/core/objects"
 import Pastel from "pastel"
 
 // Read the real version from this package's package.json rather than hardcoding it (it drifted to
 // 0.1.0 while the package shipped 4.x). `import.meta.url` points at the compiled out/cli.js, so
 // `../package.json` resolves to the installed package root in every layout (dev, tarball, global).
-const { version } = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as {
-	version: string
-}
+const { version } = parseJSONStrict<{ version: string }>(
+	readFileSync(new URL("../package.json", import.meta.url), "utf8")
+)
 
 const app = new Pastel({
 	importMeta: import.meta,

package/commands/autocomplete.tsx

@@ -19,13 +19,16 @@ import { readFileSync } from "node:fs"
 
 import { $public } from "@mailwoman/core/env"
 import { Text } from "ink"
+import { argument } from "pastel"
 import zod from "zod"
 
 import { type CommandComponent, commandError, useCommandTask } from "../cli-kit/index.ts"
 
 export { ArgumentsSchema as args, AutocompleteConfigSchema as options }
 
-const ArgumentsSchema = zod.array(zod.string().describe("Prefix string to complete"))
+const ArgumentsSchema = zod
+	.array(zod.string())
+	.describe(argument({ name: "prefix", description: "Prefix string to complete" }))
 
 /**
  * Config accepted by `mailwoman autocomplete`, validated before the server starts so a bad flag fails fast.

package/commands/clients/generate.tsx

@@ -9,8 +9,8 @@
  *   `feat/api-clients` branch) and a Rust crate (progenitor `generate_api!`, the salvaged Cargo.toml
  *   / lib.rs pattern), then VERIFY both actually build (`uv build` + a wheel import-check, `cargo
  *   check --examples`). Output lands under the gitignored `clients-build/` — nothing generated here
- *   is committed. This is the local, receipt-verified proof the gated CI job (Phase 5 Task 4)
- *   replays on dispatch. See `docs/articles/api.mdx` "Client libraries" for install/usage snippets
+ *   is committed. This is the local, receipt-verified proof the gated CI job replays on dispatch.
+ *   See `docs/articles/api.mdx` "Client libraries" for install/usage snippets
  *   and the not-yet-published status.
  */
 

package/commands/corpus/build.tsx

@@ -14,6 +14,7 @@
  *   the CLI handles partial builds during development.
  */
 
+import { parseJSONStrict } from "@mailwoman/core/objects"
 import { buildCorpus, defaultAdapterRegistry, type BuildStage } from "@mailwoman/corpus"
 import type { AdapterOptions } from "@mailwoman/corpus/types"
 import { Box, Text } from "ink"
@@ -72,7 +73,7 @@ const CorpusBuild: CommandComponent<typeof BuildConfigSchema> = ({ options }) =>
 		let inputsParsed: Record<string, zod.infer<typeof AdapterInputSchema>>
 
 		try {
-			inputsParsed = InputsSchema.parse(JSON.parse(options.inputs))
+			inputsParsed = InputsSchema.parse(parseJSONStrict(options.inputs))
 		} catch (error) {
 			throw commandError(`invalid --inputs JSON: ${(error as Error).message}`)
 		}

package/commands/corpus/fetch.tsx

@@ -17,9 +17,11 @@ import {
 	fetchNAD,
 	fetchNPPES,
 	fetchOpenAddresses,
+	fetchOurAirports,
 	fetchStateHISchools,
 	fetchStateSources,
 	fetchTigerFull,
+	fetchWikidataSubVenue,
 } from "@mailwoman/corpus/tools"
 import { Text } from "ink"
 import { argument } from "pastel"
@@ -36,14 +38,17 @@ const ArgsSchema = zod.tuple([
 			"imls-pls",
 			"nppes",
 			"openaddresses",
+			"ourairports",
 			"state-sources",
 			"state-hi-schools",
 			"tiger-full",
+			"wikidata-subvenue",
 		])
 		.describe(
 			argument({
 				name: "source",
-				description: "Fetch source id (ban, nad, hrsa, imls-pls, nppes, openaddresses, state-sources, …)",
+				description:
+					"Fetch source id (ban, nad, hrsa, imls-pls, nppes, openaddresses, ourairports, state-sources, wikidata-subvenue, …)",
 			})
 		),
 ])
@@ -107,6 +112,10 @@ function runSource(source: FetchSourceID, options: Options): Promise<FetchSummar
 			return fetchNPPES(base, report)
 		case "openaddresses":
 			return fetchOpenAddresses({ ...base, country: options.country }, report)
+		case "ourairports":
+			return fetchOurAirports(base, report)
+		case "wikidata-subvenue":
+			return fetchWikidataSubVenue(base, report)
 		case "state-sources":
 			return fetchStateSources(base, report)
 		case "state-hi-schools":

package/commands/corpus/golden-relabel.tsx

@@ -0,0 +1,53 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman corpus golden-relabel` — move a golden answer-key version onto the corpus's US
+ *   street-suffix convention (folded `street` → `street` + `street_suffix`). Writes a NEW version
+ *   dir plus a review deck; the parent version is read-only. See
+ *   `corpus/src/tools/golden-relabel-street.ts` for what it changes and, more importantly, what it
+ *   refuses to.
+ */
+
+import { Text } from "ink"
+import zod from "zod"
+
+import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
+
+const OptionsSchema = zod.object({
+	input: zod.string().describe("Parent golden version dir (read-only), e.g. data/eval/golden/v0.1.2"),
+	output: zod.string().describe("Output golden version dir, e.g. data/eval/golden/v0.1.3"),
+	deck: zod.string().optional().describe("Review-deck JSONL path. Default <output>/REVIEW-DECK.jsonl"),
+	commit: zod.string().optional().describe("Commit SHA recorded as tool provenance in the manifest"),
+	splitPrefix: zod
+		.boolean()
+		.default(true)
+		.describe("Also lift a folded leading directional into street_prefix. --no-split-prefix measures the fold's cost"),
+})
+
+export { OptionsSchema as options }
+
+const Cmd: CommandComponent<typeof OptionsSchema> = ({ options }) => {
+	const state = useCommandTask(async () => {
+		const { relabelGoldenDirectory } = await import("@mailwoman/corpus/tools")
+
+		const report = await relabelGoldenDirectory({
+			input: options.input,
+			output: options.output,
+			...(options.deck ? { deck: options.deck } : {}),
+			...(options.commit ? { commit: options.commit } : {}),
+			splitPrefix: options.splitPrefix,
+		})
+
+		return `${report.totalChanged} rows split, ${report.totalFlagged} flagged → ${report.outputDir}`
+	})
+
+	if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
+
+	if (state.status === "done") return <Text color="green">✓ {state.result}</Text>
+
+	return null
+}
+
+export default Cmd

package/commands/corpus/overlay-manifest.tsx

@@ -16,7 +16,12 @@ const OptionsSchema = zod.object({
 	base: zod.string().describe("Base corpus manifest path"),
 	newDir: zod.string().describe("New overlay corpus dir"),
 	modalRoot: zod.string().describe("Modal volume root the manifest paths are relative to"),
-	version: zod.string().describe("New corpus version"),
+	// NOT `version`: Pastel registers `-v, --version` on the ROOT program, and commander resolves it
+	// before the subcommand's own options, so `--version 9.9.9` printed the CLI version (8.7.0) and
+	// exited 0 without ever running the tool (#1491). Pastel decamelizes the key, so this derives
+	// `--corpus-version`. `mailwoman/test/command-option-collisions.test.ts` keeps the whole command
+	// tree off the reserved names.
+	corpusVersion: zod.string().describe("New corpus version"),
 	shardParquet: zod.string().describe("The ONE shard parquet to add"),
 	source: zod.string().describe("Shard source label"),
 	note: zod.string().describe("Manifest note"),
@@ -32,7 +37,7 @@ const Cmd: CommandComponent<typeof OptionsSchema> = ({ options }) => {
 			base: options.base,
 			newDir: options.newDir,
 			modalRoot: options.modalRoot,
-			version: options.version,
+			version: options.corpusVersion,
 			shardParquet: options.shardParquet,
 			source: options.source,
 			note: options.note,

package/commands/corpus/shard/index.tsx

@@ -64,6 +64,14 @@ const OptionsSchema = zod.object({
 		.optional()
 		.describe("fr-fragment: REQUIRED — reserved street-surface list to exclude (the fragment board's eval set)"),
 	multilocaleCount: zod.string().optional().describe("street-affix: multilocale row count"),
+	lexicon: zod.string().optional().describe("sub-venue: lexicon JSON (default: the committed corpus/data one)"),
+	extractsDir: zod.string().optional().describe("sub-venue: OSM sub-venue extract JSONL directory"),
+	poiDb: zod.string().optional().describe("sub-venue: poi.db path (en-US / fr-FR venue + confound pools)"),
+	subVenueTuples: zod.string().optional().describe("sub-venue: GB/US/FR address-context tuples JSONL"),
+	negativeFraction: zod
+		.string()
+		.optional()
+		.describe("sub-venue: share of rows that are confound negatives (default 0.3)"),
 })
 
 export { ArgumentsSchema as args, OptionsSchema as options }
@@ -119,6 +127,11 @@ const CorpusShard: CommandComponent<typeof OptionsSchema, typeof ArgumentsSchema
 			banDir: options.banDir,
 			excludeSurfaces: options.excludeSurfaces,
 			multilocaleCount: num(options.multilocaleCount),
+			lexicon: options.lexicon,
+			extractsDir: options.extractsDir,
+			poiDb: options.poiDb,
+			subVenueTuples: options.subVenueTuples,
+			negativeFraction: num(options.negativeFraction),
 		}
 
 		console.error(`▸ shard recipe "${name}" [${recipe.mode}] seed=${seed} → ${options.output}`)
@@ -138,7 +151,7 @@ const CorpusShard: CommandComponent<typeof OptionsSchema, typeof ArgumentsSchema
 
 		return [
 			`recipe: ${name}`,
-			`${stats.emitted.toLocaleString()} rows emitted, ${stats.skipped.toLocaleString()} skipped${stats.read != null ? `, ${stats.read.toLocaleString()} read` : ""} → ${options.output}`,
+			`${stats.emitted.toLocaleString()} rows emitted, ${stats.skipped.toLocaleString()} skipped${stats.read != null ? `, ${stats.read.toLocaleString()} read` : ""}${stats.contaminated ? `, ${stats.contaminated.toLocaleString()} board-reserved` : ""} → ${options.output}`,
 		]
 	})
 

package/commands/corpus/shard/kryptonite.tsx

@@ -5,7 +5,7 @@
  *
  *   `mailwoman corpus shard kryptonite` — build a parquet shard from the DeepSeek-generated
  *   kryptonite JSONL and emit the combined corpus MANIFEST (base shards + the new shard). See
- *   docs/articles/plan/reference/CORPUS_V0_4_0_GENERATION.md for the reproducibility contract.
+ *   docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for the reproducibility contract.
  */
 
 import { buildKryptoniteShard } from "@mailwoman/corpus/tools"

package/commands/corpus/shard/translit.tsx

@@ -6,7 +6,7 @@
  *   `mailwoman corpus shard translit` — build per-script parquet shards from the DeepSeek-generated
  *   transliteration JSONL (one shard per `deepseek-translit-<slug>` source) and emit the combined
  *   corpus MANIFEST. Sibling of `corpus shard kryptonite`; also canonicalizes legacy base-shard
- *   paths (`/mnt/playpen/mailwoman-data/…` → `/data/…`).
+ *   paths (`$MAILWOMAN_DATA_ROOT/…` → `/data/…`).
  */
 
 import { buildTranslitShard } from "@mailwoman/corpus/tools"
@@ -23,8 +23,9 @@ const OptionsSchema = zod.object({
 	canonicalPathPrefix: zod.string().default("/data/").describe("Prefix replacing legacy base-shard paths"),
 	legacyPathPrefix: zod
 		.string()
-		.default("/mnt/playpen/mailwoman-data/")
-		.describe("Legacy base-shard path prefix to rewrite"),
+
+		.optional()
+		.describe("Legacy base-shard path prefix to rewrite (default: $MAILWOMAN_DATA_ROOT)"),
 })
 
 export { OptionsSchema as options }