@mailwoman/corpus 9.0.0 → 9.2.0

package/src/shard-recipes/trailing-region.ts

@@ -0,0 +1,183 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `trailing-region` — the `«locality», «region», «country»` admin tail, the Portopetro class
+ *   (board row `es-op3-southeast-portopetro`): `…, 07691 Portopetro, Illes Balears, Spain` parsed
+ *   with the REGION mislabeled `locality` and the true locality dropped entirely. Spanish sources
+ *   are postcode-complete and rarely write the region, so the trailing-region segment is
+ *   under-attested and the model reads the last populous name as the locality.
+ *
+ *   Tuples are real `(locality, region)` ancestor pairs from the WOF admin DB (the fr-hood-pairs
+ *   extraction shape). Two surface forms per pair — with and without the trailing country — so the
+ *   region is attested both as the middle and the final segment. A bare `«region»` form is
+ *   deliberately absent: that surface is the bare-toponym class with its own rules, and teaching it
+ *   here as `region` would fight the locality/region ambiguity the dominance race arbitrates.
+ *
+ *   The recipe is country-agnostic; the country tail surface comes from the tuple's `country`
+ *   field ("Spain", "United Kingdom") so one recipe serves every extraction.
+ *
+ *   POSTCODE-PREFIXED FORMS (2026-08-20, #1748). The two bare forms above were the whole shard, and the
+ *   board row this recipe was written for is NOT bare — it reads `…, 07691 Portopetro, Illes Balears,
+ *   Spain`. Measured over both built shards: 88,904 rows, zero containing a postcode. So the model
+ *   learned the bare tail correctly and had never once seen the shape it was failing on, which is why no
+ *   decode lever moved it.
+ *
+ *   The collapse is two-staged and neither trigger is a street, which is what makes this cheap to teach:
+ *
+ *       Portopetro, Illes Balears, Spain             locality ✓  region ✓
+ *       07691 Portopetro, Illes Balears, Spain       locality ✓  region DISCARDED
+ *       15, 07691 Portopetro, Illes Balears, Spain   locality DISPLACED by the region
+ *
+ *   A postcode alone drops the region (every locale measured); a house number then displaces the
+ *   locality. So the added surfaces are postcode-prefixed and house-number-plus-postcode-prefixed, and
+ *   the shard still needs no street names.
+ *
+ *   POSTCODE PLACEMENT. The three surfaces above are the LEADING form, and for a long time they were the
+ *   only one, so this shard taught only the countries that write the postcode first. That is a real gap
+ *   and not a stylistic one, because the same digits change TAG with position. Measured on the shipped
+ *   model: `Barcelona 6001, Anzoátegui, Venezuela` tags `6001` as `house_number` and loses the locality
+ *   into the street, while `6001 Barcelona, Anzoátegui, Venezuela` tags it `postcode` and recovers
+ *   `locality: Barcelona`. `Sandton 2196` vs `2196 Sandton` behaves the same way, and no decode-time
+ *   lever moves it — `postcodeShapeCoherence: true` leaves all eight VE board rows byte-identical.
+ *
+ *   So the tuple's `postcodePlacement` selects the surface, and it keeps apart two trailing conventions
+ *   that are NOT the same shape: VE writes `Barcelona 6001, Anzoátegui, Venezuela` (the code on the
+ *   LOCALITY segment) and IN writes `…, Bengaluru, Karnataka 560038, India` (on the REGION segment).
+ *   Each of the three placements matches a board row verbatim. A tuple with no placement means `leading`,
+ *   so a tuples file written before the field existed produces the rows it always did.
+ *
+ *   LEFT CONTEXT (v25). A tuple may carry a `dependentLocality`, and when it does the surface becomes
+ *   `«dep_locality», «locality»…`. This is not decoration: without it EVERY row in the shard begins with
+ *   the locality, and at a 9.4% share that taught the model the first named segment is the locality.
+ *   Measured on the v4.8.0 candidate — `Ye Three Lords, 27 Minories, London EC3N 1DE` came back
+ *   `locality: "Ye Three Lords"` with venue and street both gone, `Le Colimaçon, 44 Rue Vieille du
+ *   Temple, 75004 Paris` came back `locality: "Le Colimaçon"`, and 11 of 25 regressions were venue-led
+ *   rows across seven countries. The house-number prefix does NOT supply this: a number before the
+ *   locality does not teach that a NAME can precede one. `no-fragment.ts`'s header records the same
+ *   trap from the other direction.
+ *
+ *   The (postcode, locality, region) triples are REAL — `postalcode-intl.db` parents joined to admin
+ *   localities and their region ancestors — with one filter that had to be measured rather than assumed.
+ *   A handful of localities act as catch-all parents: `Schwedt/Oder` claims 9,222 postcodes, `Korb`
+ *   4,846, against a p50 of 1 and a p99 of 53. Eight such hubs held 47% of the join. Capping at the p99
+ *   drops them and keeps 17,908 triples. The house number is synthetic because a house number asserts no
+ *   fact about a place; the postcode is not, because it does.
+ */
+
+import {
+	alignAndWrite,
+	makeMulberry32,
+	type PostcodePlacement,
+	readTuples,
+	type ShardRecipe,
+	shardSourceID,
+} from "./scaffold.ts"
+
+/**
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const trailingRegionRecipe: ShardRecipe = {
+	name: "trailing-region",
+	description: "Admin tails (Portopetro class): '«locality», «region»[, «country»]' from WOF ancestor pairs",
+	mode: "tuples",
+	async run(opts, write) {
+		makeMulberry32(opts.seed)
+		let read = 0
+		let emitted = 0
+		let skipped = 0
+
+		for await (const t of readTuples(opts.input!)) {
+			read++
+			const locality = String(t.locality ?? "").trim()
+			const region = String(t.region ?? "").trim()
+			const country = String(t.country ?? "").trim()
+			const dependentLocality = String(t.dependentLocality ?? "").trim()
+
+			// A pair whose region EQUALS its locality (Santa Cruz de Tenerife inside Santa Cruz de
+			// Tenerife) teaches nothing about the boundary this shard exists for.
+			if (!locality || !region || locality === region) {
+				skipped++
+
+				continue
+			}
+
+			const withCountry = read % 2 === 0 && country.length > 0
+			const postcode = String(t.postcode ?? "").trim()
+
+			// A tuple carrying a postcode emits the STRUCTURED tail — the shape the bare-only shard never
+			// contained. Every fourth such row also carries a house number, which is the second trigger:
+			// the postcode discards the region, the house number then displaces the locality.
+			const withHouseNumber = postcode.length > 0 && read % 4 === 1
+			const houseNumber = withHouseNumber ? String((read % 97) + 1) : ""
+
+			const components: Record<string, string> = { locality, region }
+
+			// LEFT CONTEXT. Without it every row begins with the locality, and the shard teaches that the first named
+			// segment IS the locality — measured on the v4.8.0 candidate: `Ye Three Lords, 27 Minories, London EC3N 1DE`
+			// came back `locality: "Ye Three Lords"` with the venue and street gone, and 11 of its 25 regressions were
+			// venue-led rows across seven countries. The house-number prefix does not supply it, because a NUMBER before
+			// the locality does not teach that a NAME can precede one.
+			if (dependentLocality && dependentLocality !== locality) {
+				components["dependent_locality"] = dependentLocality
+			}
+
+			if (withCountry) {
+				components["country"] = country
+			}
+
+			if (postcode) {
+				components["postcode"] = postcode
+			}
+
+			if (withHouseNumber) {
+				components["house_number"] = houseNumber
+			}
+
+			const placement = (t.postcodePlacement as PostcodePlacement | undefined) ?? "leading"
+			// Both trailing placements put the code inside the ADMIN tail; they differ in which segment carries it.
+			const bareLocality = postcode && placement === "after_locality" ? `${locality} ${postcode}` : locality
+
+			const localitySegment =
+				components["dependent_locality"] === undefined ? bareLocality : `${dependentLocality}, ${bareLocality}`
+
+			const regionSegment = postcode && placement === "after_region" ? `${region} ${postcode}` : region
+
+			const tail = withCountry
+				? `${localitySegment}, ${regionSegment}, ${country}`
+				: `${localitySegment}, ${regionSegment}`
+
+			// A leading postcode joins the head, ahead of the locality; the other two are already in the tail, so the
+			// head carries at most the house number.
+			const leadingPostcode = postcode && placement === "leading" ? `${postcode} ` : ""
+			const head = withHouseNumber ? `${houseNumber}, ${leadingPostcode}` : leadingPostcode
+			const raw = `${head}${tail}`
+			// A DISTINCT source for the structured rows. The sampler buckets by `source` and weights each bucket,
+			// so emitting these under `synth-trailing-region` would pool them with the 88,904 bare rows and make
+			// the new surface unweightable — the dose would silently be whatever the bare shard's weight bought.
+			const sourceLabel = postcode ? "synth-trailing-region-structured" : "synth-trailing-region"
+			const source_id = shardSourceID(sourceLabel, { ...components, v: String(read) })
+
+			const canonical = {
+				raw,
+				components,
+				country: String(t.cc ?? "").trim() || "und",
+				locale: String(t.locale ?? "und"),
+				source: sourceLabel,
+				source_id,
+				corpus_version: "0.11.0",
+				license: "Synthetic — trailing-region; (locality, region) ancestor pairs from WOF (CC0/ODC-By per source)",
+			}
+
+			if (alignAndWrite(write, canonical, "trailing-region")) {
+				emitted++
+			} else {
+				skipped++
+			}
+		}
+
+		return { read, emitted, skipped }
+	},
+}

package/src/shard-recipes/unit.ts

@@ -21,16 +21,15 @@
  *   framework LCG) is consumed in the exact call order the legacy script used.
  */
 
-import { spawnSync } from "node:child_process"
-
 import { US_UNIT_DESIGNATOR_PREFERRED_ABBR, type USUnitDesignator } from "@mailwoman/codex/us"
 import type { ComponentTag } from "@mailwoman/core/types"
 import { dataRootPath } from "@mailwoman/core/utils"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow } from "../types.ts"
-import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
+import { stableSourceID } from "#adapters/utils"
+import type { CanonicalRow } from "#types"
+import { alignRow } from "#utils"
+
+import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 /**
  * A cached OpenAddresses extract: the zip, the CSV member, and the implied (file-level) region.
@@ -122,19 +121,11 @@ interface UnitTuple {
 /**
  * Stream real US tuples (number/street/city/postcode + the bare OA unit id) out of a cached OA zip.
  */
-function readTuples(source: UnitSource): UnitTuple[] {
-	const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
-
-	if (r.status !== 0) {
-		console.error(`  WARN: unzip failed for ${source.zip} (status ${r.status})`)
-
-		return []
-	}
-
+async function readTuples(source: UnitSource): Promise<UnitTuple[]> {
 	const tuples: UnitTuple[] = []
 	const seen = new Set<string>()
 
-	for (const row of readCSVRecords(r.stdout)) {
+	for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
 		const street = row.street ?? ""
 		const locality = row.city ?? ""
 		const house_number = row.number ?? ""
@@ -263,7 +254,7 @@ export const unitRecipe: ShardRecipe = {
 		const pool: UnitTuple[] = []
 
 		for (const s of sources) {
-			const t = readTuples(s)
+			const t = await readTuples(s)
 
 			console.error(`  ${s.csv}: ${t.length} unique tuples`)
 

package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts}

@@ -46,10 +46,10 @@
  *   tokenizer splits `/`) and is the worst within-token class — but it inherently requires non-base
  *   AU/NZ/UK locales, which contradict the US/FR/DE base (the lint catch). It belongs in a
  *   separately-scoped AU/NZ/UK boundary-coverage shard that ALSO adds AU base coverage, not in this
- *   base-locale shard. `synthesize-boundary-stress.test.ts` proves the alignments.
+ *   base-locale shard. `synthesizers/boundary-stress.test.ts` proves the alignments.
  */
 
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "#types"
 
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
    cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the

package/src/{synthesize-german.ts → synthesizers/german.ts}

@@ -22,8 +22,9 @@
  *   `raw` so alignment lands.
  */
 
-import { formatAddress } from "./format.ts"
-import type { CanonicalRow } from "./types.ts"
+import { formatAddress } from "@mailwoman/formatter"
+
+import type { CanonicalRow } from "#types"
 
 /**
  * A real address tuple (e.g. one OpenAddresses row): street + locality required, rest optional.

package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts}

@@ -9,7 +9,7 @@
  *   house_number recall by ~4-5pp. DeepSeek's turn-8 root-cause:
  *
  *   1. Direct: `5th Avenue Theatre`-style adversarial venues teach the model that tokens like "5th"
- *        belong to venues, not house_numbers. (Fixed in `synthesize-no-street.ts` by removing
+ *        belong to venues, not house_numbers. (Fixed in `no-street.ts` by removing
  *        digit+ordinal venue patterns.)
  *   2. Distributional dilution: synth-no-street adds 122K rows where house_number is absent. The model's
  *        training distribution shifts toward "house_number is rare," and it under-emits the tag at
@@ -23,12 +23,12 @@
  *   Real-world shape: business cards, mailing labels, store directories — `"123 Main St, Sunrise
  *   Bakery, Springfield, IL 62701"` is a perfectly ordinary address form.
  *
- *   Venue pool: PLAIN_VENUES from `synthesize-no-street.ts` (re-exported here). Adversarial venues
+ *   Venue pool: PLAIN_VENUES from `no-street.ts` (re-exported here). Adversarial venues
  *   are deliberately NOT used here — the point is to teach co-occurrence, not to re-introduce
  *   decompose-mode pressure.
  */
 
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "#types"
 
 export interface HouseVenueBaseTuple {
 	locality: string
@@ -61,7 +61,7 @@ export interface SynthesizedHouseVenueRow {
 
 /**
  * PLAIN venue names, carrying no street-typing tokens. This shard teaches house_number + venue coexistence, NOT
- * decompose-mode pressure — adversarial venue names live in `synthesize-no-street.ts`.
+ * decompose-mode pressure — adversarial venue names live in `no-street.ts`.
  */
 const PLAIN_VENUES: ReadonlyArray<string> = [
 	"Bob's Pizza",
@@ -245,9 +245,21 @@ export function synthesizeHouseVenueRow(
 	// 75005 Paris" — the v4.0.0 gauntlet's venue-led failure family, the run-2 contingency's exact
 	// target shape). GB (#1366) renders locality-then-postcode with NO region and NO comma between
 	// them ("Ye Three Lords, 27 Minories, London EC3N 1DE" — the third tail the shard must teach).
-	// Every other country keeps the original US-order tail.
+	// VE (#1821) renders locality-then-postcode and KEEPS the region after it. Every other country
+	// keeps the original US-order tail, which is itself the `after_region` surface — `Springfield, IL
+	// 02101` and `Bengaluru, Karnataka 560038` are the same shape.
 	const frOrder = base.country === "FR"
 	const gbOrder = base.country === "GB"
+	// VE writes the code on the LOCALITY segment with the region still after it — `…, Barcelona 6001, Anzoátegui,
+	// Venezuela`, the four `ve_city_postcode_trailing_state` board rows. That is neither GB's form (which drops the
+	// region) nor the default (which puts the region before the code), so it needs its own branch.
+	//
+	// It belongs HERE rather than in a standalone admin shard, and that is measured: three trailing-region shards
+	// carrying only admin segments all graded DO-NOT-SHIP, and the way they failed was by damaging the classes they did
+	// not contain — v4.8.0 turned `Ye Three Lords, 27 Minories, London EC3N 1DE` into `locality: "Ye Three Lords"`,
+	// losing the venue and the street. Every row this synthesizer emits carries a venue, a street AND a house number,
+	// so the surface is taught with the alternatives present rather than against them.
+	const veOrder = base.country === "VE"
 
 	// GB rows draw from the GB pool 70% of the time (institutional/archaic/brand-dash-place forms,
 	// incl. directional-led names — the #1366 target class) and the shared pool otherwise; real GB
@@ -280,7 +292,9 @@ export function synthesizeHouseVenueRow(
 		? `${base.postcode} ${base.locality}`
 		: gbOrder
 			? `${base.locality} ${base.postcode}`
-			: `${base.locality}, ${base.region} ${base.postcode}`
+			: veOrder
+				? `${base.locality} ${base.postcode}, ${base.region}`
+				: `${base.locality}, ${base.region} ${base.postcode}`
 
 	// Trailing country surface (Addendum 3): appended AFTER the tail in every order, tagged.
 	const countrySurfaces = COUNTRY_SURFACES[base.country]

package/src/{synthesize-intersection.ts → synthesizers/intersection.ts}

@@ -19,7 +19,7 @@
  *   US-idiomatic only (the harness intersection cases are US: "X & Y, City, ST ZIP").
  */
 
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "#types"
 
 /**
  * Street name cores (no suffix) — proper-noun streets that often appear bare.

package/src/{synthesize-no-street.ts → synthesizers/no-street.ts}

@@ -34,10 +34,10 @@
  *   counter-example signal the model is missing.
  *
  *   This complements (does not replace) the existing US-base-tuple source used by
- *   `synthesize-po-box.ts`; the same `NoStreetBaseTuple` shape is consumed.
+ *   `po-box.ts`; the same `NoStreetBaseTuple` shape is consumed.
  */
 
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "#types"
 
 //#region Types
 

package/src/{synthesize-po-box.ts → synthesizers/po-box.ts}

@@ -24,7 +24,7 @@
  *   - USPS DMM 508 §4.1.4 / §4.5.4 — PO Box and street-addressed PO Box
  */
 
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "#types"
 
 /**
  * Digits a box number needs before a thousands comma is plausible (`1,234`).

package/src/{synthesize-street.ts → synthesizers/street.ts}

@@ -19,8 +19,10 @@
  *   adapter.
  */
 
-import { decomposeStreet } from "./adapters/tiger/street-decompose.ts"
-import type { CanonicalRow } from "./types.ts"
+import { isPresent } from "@mailwoman/core/objects"
+
+import { decomposeStreet } from "#adapters/tiger/street-decompose"
+import type { CanonicalRow } from "#types"
 
 // Hand-curated US street name pool. Real frequency-weighted street names — sampled
 // from US Census TIGER 2024 top-1000 by occurrence count. Keep ~50 entries so the
@@ -223,7 +225,7 @@ export function synthesizeStreetRow(
 	const trailing = pick(TRAILING_DIRECTIONALS, random)
 
 	// Build the "full" street string the adapter would receive from TIGER FULLNAME.
-	const parts = [prefix, name, suffix, trailing].filter(Boolean)
+	const parts = [prefix, name, suffix, trailing].filter(isPresent)
 	const fullStreet = parts.join(" ")
 
 	// Pass through the same decomposeStreet TIGER uses — match the training distribution.

package/src/{synthesize.ts → synthesizers/utils.ts}

@@ -33,12 +33,14 @@ import {
 	matchLeadingDesignator,
 	matchTrailingSuffix,
 } from "@mailwoman/codex/us"
+import { isPresent } from "@mailwoman/core/objects"
 import type { BIOLabel, ComponentTag } from "@mailwoman/core/types"
 import { mulberry32 } from "@mailwoman/core/utils"
 
-import { alignRow, assertSpanInvariants, type ComponentSpan } from "./align.ts"
-import { whitespaceTokenizer, type Tokenizer } from "./tokenize.ts"
-import type { CanonicalRow, LabeledRow, QuarantinedRow } from "./types.ts"
+import type { CanonicalRow, LabeledRow, QuarantinedRow } from "#types"
+import { alignRow, assertSpanInvariants, type ComponentSpan } from "#utils/align"
+
+import { whitespaceTokenizer, type Tokenizer } from "../utils/tokenize.ts"
 
 /**
  * An augmentation transforms a single row. Return `null` if the augmentation doesn't apply (e.g. accent-strip on a row
@@ -238,7 +240,7 @@ export const typoInject: Augmentation = (row) => {
 		return n
 	}
 
-	const values = Object.values(row.components).filter(Boolean) as string[]
+	const values = Object.values(row.components).filter(isPresent)
 
 	const eligible = (Object.entries(row.components) as Array<[ComponentTag, string]>).filter(
 		([, v]) => v && ALPHA_NAME.test(v) && occurs(v) === 1 && !values.some((o) => o !== v && o.includes(v))

package/src/tools/align-shard.ts

@@ -1,6 +1,8 @@
 #!/usr/bin/env node
 import { createWriteStream } from "node:fs"
 
+import { JSONSpliterator } from "spliterator"
+
 /**
  * Re-emit a CANONICAL jsonl ({raw, components, country, source, ...}) as a LABELED jsonl in the CURRENT align format,
  * by running every row through `alignRow` (corpus/src/align.ts).
@@ -26,8 +28,7 @@ import { createWriteStream } from "node:fs"
  * --output /tmp/kryptonite-labeled.jsonl\
  * --corpus-version 0.5.0
  */
-import { alignRow } from "@mailwoman/corpus"
-import { JSONSpliterator } from "spliterator"
+import { alignRow } from "#utils"
 
 export interface AlignShardOptions {
 	input: string

package/src/tools/fetch/ban.ts

@@ -24,6 +24,7 @@ import { join } from "node:path"
 import { setTimeout as sleep } from "node:timers/promises"
 import { gunzipSync } from "node:zlib"
 
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/utils"
 import { sha256File } from "@mailwoman/core/utils"
 
 import type { BaseFetchOptions, FetchSummary } from "./download.ts"
@@ -33,7 +34,6 @@ import { downloadToFile, loadManifestEntries, writeManifest } from "./download.t
  * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
  * than data.
  */
-const BYTES_PER_KIB = 1024
 
 const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv"
 
@@ -156,26 +156,6 @@ export interface BanManifestEntry {
 
 export type FetchBanOptions = BaseFetchOptions
 
-/**
- * Mimic `numfmt --to=iec` for a friendly byte-size log line.
- */
-function iec(bytes: number): string {
-	if (bytes < BYTES_PER_KIB) return String(bytes)
-	const units = ["K", "M", "G", "T", "P"]
-	let value = bytes / 1024
-	let i = 0
-
-	while (value >= BYTES_PER_KIB && i < units.length - 1) {
-		value /= 1024
-
-		i++
-	}
-
-	const rounded = value < 10 ? value.toFixed(1) : Math.round(value).toString()
-
-	return `${rounded}${units[i] ?? ""}`
-}
-
 export async function fetchBan(options: FetchBanOptions, report?: (line: string) => void): Promise<FetchSummary> {
 	const banDir = join(options.outRoot, "ban")
 	const manifestPath = join(banDir, "MANIFEST.json")
@@ -281,7 +261,7 @@ export async function fetchBan(options: FetchBanOptions, report?: (line: string)
 			bytes,
 		})
 
-		report?.(`  ✓ ${iec(bytes)}  sha256=${sha}`)
+		report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)}  sha256=${sha}`)
 
 		fetched++
 

package/src/tools/fetch/download.ts

@@ -3,9 +3,7 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Download + manifest plumbing for the `mailwoman corpus fetch <source>` family — one download-with-retry and
- *   one MANIFEST.json idiom instead of the per-script clones the 2026-07-09 dedupe survey counted
- *   (6× `downloadToFile`, 2× `isTransientStatus`, 9× manifest writes).
+ *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
  */
 
 import { existsSync } from "node:fs"
@@ -52,6 +50,23 @@ export interface FetchSummary {
 /**
  * A status worth retrying: rate limiting or a server-side failure.
  */
+/**
+ * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
+ * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
+ * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is
+ * 500. Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns
+ * a CI run and vanishes locally.
+ */
+export class HTTPStatusError extends Error {
+	readonly status: number
+
+	constructor(status: number, message: string) {
+		super(message)
+		this.name = "HTTPStatusError"
+		this.status = status
+	}
+}
+
 export function isTransientStatus(status: number): boolean {
 	return status === HTTP_TOO_MANY_REQUESTS || (status >= HTTP_SERVER_ERROR_MIN && status <= HTTP_SERVER_ERROR_MAX)
 }
@@ -92,6 +107,9 @@ export async function downloadToFile(options: DownloadOptions): Promise<{ bytes:
 		let res: Response
 
 		try {
+			// Raw `fetch`, deliberately: this is the shared FILE downloader and the body is piped to disk below.
+			// `APIClient` is the repo default for API requests — small bodies, repeated calls — and buffers a
+			// non-stream response in memory, which is the one thing a multi-gigabyte transfer must not do.
 			res = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) })
 		} catch (error) {
 			// AbortSignal timeouts and network-level failures are retryable.
@@ -101,7 +119,7 @@ export async function downloadToFile(options: DownloadOptions): Promise<{ bytes:
 		}
 
 		if (!res.ok) {
-			const error = new Error(`HTTP ${res.status} ${res.statusText} — ${url}`)
+			const error = new HTTPStatusError(res.status, `HTTP ${res.status} ${res.statusText} — ${url}`)
 
 			if (!isTransientStatus(res.status)) throw error
 			lastError = error