@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/south-asia/recipes/register.ts

@@ -0,0 +1,214 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `pk-register` and `bd-register` slice recipes — the typed forms of Pakistan and Bangladesh the country templates do
+ *   not render, over the OpenStreetMap corpus JSONL the `osm` adapter reads (#1744, the F8 family). The adapter's own
+ *   rows carry the template shape (`4 38th Street, DHA Phase 6, Karachi`; `24 Road 104, Dhaka - 1207`); these two carry
+ *   the register:
+ *
+ *   - the HOUSE line, `House 4, Street 25, F-7/2, Islamabad` and `House 34, Road 4, Sector 9, Uttara, Dhaka 1230` —
+ *     `House` in front of the number (untagged, the same rule as Singapore's `Blk`), the numbered street, the scheme or
+ *     sector as the dependent locality, the city;
+ *   - the PLAIN line with the trailing postcode and no dash, `58 Kalabagan 1st Ln, Dhaka 1205` — what a person types,
+ *     where the template writes `Dhaka - 1205`.
+ *
+ *   Islamabad's sector codes (`F-7/2`, `G-10/3`) are the one synthesized value: the register carries them on a handful
+ *   of rows, mostly inside `addr:city` (`F7/2 Islamabad`), so a row whose city is Islamabad and whose scheme is unknown
+ *   draws one from the capital's grid. Every other value is the row's own. The components are the `osm` adapter's
+ *   mapping (`componentsForOSMRow`), so the two surfaces cannot disagree about what a field means.
+ *
+ *   ⚠ ODbL: the rows inherit OpenStreetMap's share-alike license, and `--exclude-share-alike` drops them.
+ *
+ *   Run: mailwoman corpus slice pk-register --input <osm-pk.corpus.jsonl> --count N --seed S
+ */
+
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
+
+import { componentsForOSMRow, OSM_LICENSE, sameName } from "#adapters/osm/adapter"
+import { alignAndWrite, type CorpusRecipe, readTuples, type SliceRecipeOpts, sliceSourceID } from "#recipes/scaffold"
+import type { CanonicalRow } from "#types"
+
+/**
+ * Islamabad's residential sectors: the lettered rows E to I, the numbered columns the Capital Development Authority has
+ * developed, and the four sub-sectors each splits into.
+ */
+const ISLAMABAD_SECTOR_ROWS = ["E", "F", "G", "I"] as const
+const ISLAMABAD_SECTOR_COLUMNS = [6, 7, 8, 9, 10, 11] as const
+const ISLAMABAD_SUB_SECTORS = [1, 2, 3, 4] as const
+
+/**
+ * Share of rows rendered in the house register; the rest take the plain line.
+ */
+const P_HOUSE = 0.55
+
+/**
+ * Share of house lines that spell the number `House No. 4` / `House # 4` rather than `House 4`.
+ */
+const P_HOUSE_NO = 0.25
+
+export type SouthAsiaRegister = "house" | "plain"
+
+export interface SouthAsiaRendering {
+	raw: string
+	components: Record<string, string>
+	register: SouthAsiaRegister
+}
+
+/**
+ * Draw one Islamabad sector code, `F-7/2`.
+ */
+export function drawIslamabadSector(random: () => number): string {
+	const row = ISLAMABAD_SECTOR_ROWS[Math.floor(random() * ISLAMABAD_SECTOR_ROWS.length)]!
+	const column = ISLAMABAD_SECTOR_COLUMNS[Math.floor(random() * ISLAMABAD_SECTOR_COLUMNS.length)]!
+	const sub = ISLAMABAD_SUB_SECTORS[Math.floor(random() * ISLAMABAD_SUB_SECTORS.length)]!
+
+	return `${row}-${column}/${sub}`
+}
+
+/**
+ * Render one register line from the adapter's components. Returns null when the row has no house number or no locality:
+ * both registers need the two.
+ */
+export function renderSouthAsiaRegister(
+	components: CanonicalRow["components"],
+	random: () => number,
+	register?: SouthAsiaRegister
+): SouthAsiaRendering | null {
+	const { house_number: number, street, locality, postcode } = components
+
+	if (!number || !street || !locality) return null
+
+	let dependent = components.dependent_locality
+
+	if (!dependent && sameName(locality, "Islamabad")) {
+		dependent = drawIslamabadSector(random)
+	}
+
+	const chosen = register ?? (random() < P_HOUSE ? "house" : "plain")
+	const out: Record<string, string> = { house_number: number, street, locality }
+	const tail = postcode ? `${locality} ${postcode}` : locality
+
+	if (postcode) {
+		out.postcode = postcode
+	}
+
+	if (dependent) {
+		out.dependent_locality = dependent
+	}
+
+	const middle = dependent ? `${street}, ${dependent}` : street
+
+	if (chosen === "house") {
+		const spelling = random() < P_HOUSE_NO ? (random() < 0.5 ? "House No. " : "House # ") : "House "
+
+		return { raw: `${spelling}${number}, ${middle}, ${tail}`, components: out, register: "house" }
+	}
+
+	return { raw: `${number} ${middle}, ${tail}`, components: out, register: "plain" }
+}
+
+function makeRecipe(name: string, country: "PK" | "BD", locale: string, description: string): CorpusRecipe {
+	const source = `synth-${name}`
+
+	return {
+		name,
+		description,
+		mode: "tuples",
+		options: [
+			{ flag: "--count", description: "Rows to emit (default: every input row once)" },
+			{
+				flag: "--input",
+				description: `The OpenStreetMap ${country} corpus JSONL (osm-${country.toLowerCase()}.corpus.jsonl)`,
+			},
+		],
+		async run(opts: SliceRecipeOpts, write) {
+			if (!opts.input) {
+				throw new Error(`${name}: --input <osm-${country.toLowerCase()}.corpus.jsonl> is required`)
+			}
+
+			const random = makeMulberry32(opts.seed)
+			const sourceName = opts.sourceName ?? source
+			const registers: Record<string, number> = {}
+			let read = 0
+			let emitted = 0
+			let skipped = 0
+			let quarantined = 0
+
+			for await (const tuple of readTuples(opts.input)) {
+				read++
+
+				if (opts.count !== undefined && emitted >= opts.count) break
+
+				const components = componentsForOSMRow(tuple as Parameters<typeof componentsForOSMRow>[0])
+				const rendering = components ? renderSouthAsiaRegister(components, random) : null
+
+				if (!rendering) {
+					skipped++
+
+					continue
+				}
+
+				if (opts.golden) {
+					write(JSON.stringify({ raw: rendering.raw, components: rendering.components, country, locale }) + "\n")
+
+					emitted++
+					registers[rendering.register] = (registers[rendering.register] ?? 0) + 1
+
+					continue
+				}
+
+				const ok = alignAndWrite(
+					write,
+					{
+						raw: rendering.raw,
+						components: rendering.components,
+						country,
+						locale,
+						source: sourceName,
+						source_id: sliceSourceID(sourceName, { raw: rendering.raw }),
+						corpus_version: "0.1.0",
+						license: OSM_LICENSE,
+					},
+					`${name}:${rendering.register}`
+				)
+
+				if (ok) {
+					emitted++
+					registers[rendering.register] = (registers[rendering.register] ?? 0) + 1
+				} else {
+					quarantined++
+				}
+			}
+
+			console.error(
+				`  ${name}: ${emitted} emitted (${Object.entries(registers)
+					.map(([k, v]) => `${k} ${v}`)
+					.join(", ")}), ${skipped} skipped, ${quarantined} quarantined of ${read} read`
+			)
+
+			return { read, emitted, skipped }
+		},
+	}
+}
+
+/**
+ * Pakistan: `House 4, Street 25, F-7/2, Islamabad` and `B230 Street 5, Block 3, Karachi`.
+ */
+export const pkRegisterRecipe: CorpusRecipe = makeRecipe(
+	"pk-register",
+	"PK",
+	"en-PK",
+	"Pakistan typed registers over the OpenStreetMap rows: the House line with the sector or scheme, the plain line with the trailing postcode"
+)
+
+/**
+ * Bangladesh: `House 34, Road 4, Sector 9, Uttara, Dhaka 1230` and `58 Kalabagan 1st Ln, Dhaka 1205`.
+ */
+export const bdRegisterRecipe: CorpusRecipe = makeRecipe(
+	"bd-register",
+	"BD",
+	"en-BD",
+	"Bangladesh typed registers over the OpenStreetMap rows: the House line, the plain line with the trailing postcode and no dash"
+)

package/lib/synthesizers/anchor-absorption.ts

@@ -30,7 +30,7 @@
  *   ZIPs (anchor-fp) are 5-digit strings deliberately absent from the lookup.
  */
 
-import type { ComponentTag } from "@mailwoman/core/types"
+import type { ComponentTag } from "@mailwoman/codex/component"
 
 import { pick } from "#synthesizers/utils"
 

package/lib/synthesizers/locale.ts

@@ -0,0 +1,261 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Locale address synthesizer — first introduced for German coverage (night-shift 2026-06-02,
+ *   DE-1), now shared by country-oriented recipes.
+ *
+ *   The neural model is out-of-distribution on German: it truncates `Straußstraße`→`Strau` (exits at
+ *   the ß-piece boundary), absorbs the house number into the street (`Hauptstraße 5` → one span),
+ *   and mis-tags the native-order house number as a postcode (`Prenzlauer Allee 36, 10405 Berlin` →
+ *   postcode `36`). The cause is ORDER: the model was trained US+FR (house-number-FIRST,
+ *   postcode-AFTER-city), and never saw the German convention (house-number-AFTER-street,
+ *   postcode-BEFORE-city). DE-0 confirmed the tokenizer round-trips German orthography cleanly, so
+ *   this is a coverage gap, not a tokenizer ceiling.
+ *
+ *   The original German generator produced the missing signal as a small targeted supplement slice
+ *   (synthesis-as-supplement discipline: weight < 0.25, one-and-done). It does NOT synthesize
+ *   German street names (German morphology is hard to fake) — it takes REAL German component tuples
+ *   (from OpenAddresses Berlin/Saxony) and renders them in idiomatic German order via the OpenCage
+ *   `DE` template (`formatAddress(..., "DE")` → `"Straußstraße 27, 12623 Berlin"`). The corpus
+ *   aligner turns the row into BIO labels; every emitted component surface form occurs verbatim in
+ *   `raw` so alignment lands. Its locale-neutral API now also serves the international recipe.
+ */
+
+import { formatAddress } from "@mailwoman/codex/address-format"
+
+import type { CanonicalRow } from "#types"
+
+/**
+ * A real address tuple (e.g. one OpenAddresses row): street + locality required, rest optional.
+ */
+/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
+   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
+   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
+   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   extracted as named constants above. */
+
+export interface LocaleBaseTuple {
+	house_number?: string
+	street: string
+	locality: string
+	/**
+	 * A sub-locality that sits BELOW the locality (a suburb / district). NZ is the case that needs it: the OA DISTRICT
+	 * column holds the city (`Auckland`) and CITY holds the suburb (`Birkenhead`), so the real envelope carries both (`31
+	 * Rawene Road, Birkenhead, Auckland`). Rendered between street and locality in both orders when present.
+	 */
+	dependent_locality?: string
+	region?: string
+	postcode?: string
+}
+
+/**
+ * @deprecated Alias — use LocaleBaseTuple.
+ */
+export type GermanBaseTuple = LocaleBaseTuple
+
+export interface SynthesizedLocaleRow {
+	raw: string
+	components: CanonicalRow["components"]
+	locale: string
+}
+
+/**
+ * @deprecated Alias — use SynthesizedLocaleRow.
+ */
+export type SynthesizedGermanRow = SynthesizedLocaleRow
+
+export interface LocaleSynthesisOpts {
+	random?: () => number
+	/**
+	 * Rendering order for the SAME components. `"native"` (default) uses the country's own template (DE →
+	 * house-AFTER-street, postcode-BEFORE-city). `"international"` renders house-FIRST, postcode-AFTER-city — the US/GB
+	 * layout that international feeds, US-centric systems, and our own OpenAddresses de-sample impose on non-US
+	 * addresses. Training both teaches the model that a German address can arrive either way, so the eval's US-order
+	 * rendering stops reading as a collapse. See `docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md` (the
+	 * order-artifact correction).
+	 */
+	order?: "native" | "international"
+	/**
+	 * Postcode surface shape. `"conventional"` (default) canonicalizes to the country's rendered form (NL: OA's glued
+	 * `1011AB` → the spaced `1011 AB`); `"as-source"` keeps the source's own surface — the form OA (and the OA-derived
+	 * evals) feed, which for NL is 100% glued. Only NL differs today; every other country passes through identically
+	 * either way. Mixing both teaches the two-letter-suffix `1012 LM` shape AND the glued feed shape (#241 — the model
+	 * currently glues the suffix onto the city).
+	 */
+	postcodeShape?: "conventional" | "as-source"
+	/**
+	 * How the NATIVE-order render joins street and house number. The OpenCage ES template comma-joins (`Calle Mayor, 12`
+	 * — the official Spanish convention); OA-derived feeds and our ES eval space-join (`CALLE MAYOR 12`, the observed
+	 * form on all 3,000 eval rows). `"template"` (default) keeps the template's own join; `"space"` collapses `<street>,
+	 * <house_number>` → `<street> <house_number>` after rendering. Countries whose template already space-joins
+	 * (DE/IT/NL) render identically under both. Mixing both stops an ES slice from teaching the comma as THE street→house
+	 * boundary signal (#241 format-diversity audit). International order ignores this (the US template is already
+	 * house-first space-joined).
+	 */
+	nativeHouseJoin?: "template" | "space"
+	/**
+	 * The string between rendered address lines. `", "` (default) is the template's own join. `" "` renders the
+	 * comma-free single-line register — dictation, a copy out of a one-field form — `Neusser Str. 12 Nippes 50733 Köln`
+	 * for the same components. Stage 2 segments the comma form into three and the comma-free form into ONE, and a single
+	 * segment starves the placetype-pair prior, which is how the comma-free form loses `Nippes` (#1946). Only the native
+	 * order reads it; the international layout keeps its own separator.
+	 */
+	separator?: ", " | " "
+}
+
+/**
+ * @deprecated Alias — use LocaleSynthesisOpts.
+ */
+export type GermanSynthesisOpts = LocaleSynthesisOpts
+
+/**
+ * ISO-3166 alpha-2 → BCP-47 tag for the emitted rows (primary language per country).
+ */
+const LOCALE_TAG: Record<string, string> = {
+	DE: "de-DE",
+	ES: "es-ES",
+	IT: "it-IT",
+	NL: "nl-NL",
+	GB: "en-GB",
+	FR: "fr-FR",
+	US: "en-US",
+	NZ: "en-NZ",
+}
+
+/**
+ * Canonicalize a postcode to the form the country's template renders, so the stored component aligns verbatim against
+ * `raw`. NL is the case that needs it: OA stores `1011AB` but the OpenCage NL template emits the conventional spaced
+ * `1011 AB` (4 digits + space + 2 letters), which otherwise fails verbatim alignment and drops the row. Other countries
+ * pass through unchanged.
+ */
+function normalizePostcode(postcode: string, country: string): string {
+	if (country === "NL") {
+		const m = /^(\d{4})\s*([A-Za-z]{2})$/.exec(postcode)
+
+		if (m) return `${m[1]} ${m[2]!.toUpperCase()}`
+	}
+
+	return postcode
+}
+
+/**
+ * True when `value` appears verbatim AND as a standalone token (so BIO alignment lands cleanly).
+ */
+function tokenPresent(raw: string, value: string): boolean {
+	if (!raw.includes(value)) return false
+	// Reject substring-of-a-larger-number collisions (e.g. house "2" inside postcode "12623").
+	const i = raw.indexOf(value)
+	const before = raw[i - 1]
+	const after = raw[i + value.length]
+	const isDigit = (c: string | undefined) => c !== undefined && c >= "0" && c <= "9"
+
+	if (/^\d+$/.test(value) && (isDigit(before) || isDigit(after))) return false
+
+	return true
+}
+
+/**
+ * Render one real tuple into an idiomatic, locale-ordered `{raw, components}` row via the OpenCage `country` template
+ * (DE → house-after-street + postcode-before-city; ES/IT the same; GB house-first; NL carries the `1012 LM` postcode),
+ * with light variation (drop house number / postcode some of the time). Returns `null` when the tuple is too thin or a
+ * component wouldn't align cleanly.
+ *
+ * Region handling is order-dependent: NATIVE order omits it (the native template absorbs the admin region into the
+ * postcode/city line, so it rarely renders verbatim and would break BIO alignment), while INTERNATIONAL order includes
+ * it in the tail ("City, Region Postcode" — the US/feed layout the eval uses; v0.9.3 / #327).
+ *
+ * Pass `opts.order: "international"` to render the same components house-first / postcode-after-city instead (see
+ * {@link LocaleSynthesisOpts.order}) — the layout international feeds impose on foreign addresses, and the one a
+ * native-order-trained model treats as a "collapse."
+ */
+export function synthesizeLocaleRow(
+	base: LocaleBaseTuple,
+	country: string,
+	opts: LocaleSynthesisOpts = {}
+): SynthesizedLocaleRow | null {
+	const random = opts.random ?? Math.random
+	const order = opts.order ?? "native"
+
+	if (!base.street || !base.locality) return null
+
+	const components: CanonicalRow["components"] = { street: base.street, locality: base.locality }
+
+	// Sub-locality (suburb / district) sits between street and locality and renders in BOTH orders — it's part
+	// of the address body, not the admin-region tail that native order drops. NZ needs it (suburb + city both on
+	// the envelope). The tokenPresent check below drops the row if the template didn't surface it verbatim.
+	if (base.dependent_locality) {
+		components.dependent_locality = base.dependent_locality
+	}
+
+	// ~80% keep the house number (the rest are street-only forms, also idiomatic).
+	if (base.house_number && random() < 0.8) {
+		components.house_number = base.house_number
+	}
+
+	// ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it). The
+	// `postcodeShape: "as-source"` rewrite happens AFTER the render: the OpenCage NL template
+	// normalizes the postcode itself, so a glued input can't survive rendering directly.
+	if (base.postcode && random() < 0.85) {
+		components.postcode = normalizePostcode(base.postcode, country)
+	}
+
+	// International order carries the REGION in the tail ("City, Region Postcode") — the layout real
+	// US/feed renderings (and our OA eval) use. v0.9.2 rendered international order WITHOUT the region,
+	// so the model never learned to segment the tail and mangled it at eval (region absorbed into the
+	// locality / locality dropped); v0.9.3 closes that gap (#327). Native order still drops the region
+	// (the native template absorbs it into the city line, which would break verbatim alignment).
+	if (order === "international" && base.region) {
+		components.region = base.region
+	}
+
+	// Native order uses the address's own country template; international order uses the US template —
+	// house-first, postcode-after-city, with a region slot for the tail. Neither branch consumes a
+	// `random()` draw for the template, so the RNG sequence existing callers/tests depend on is stable.
+	const renderCountry = order === "international" ? "US" : country
+	const separator = order === "native" ? (opts.separator ?? ", ") : ", "
+	let raw = formatAddress(components, renderCountry, { separator })
+
+	if (!raw) return null
+
+	// Native-order space-join (see {@link LocaleSynthesisOpts.nativeHouseJoin}): collapse the template's
+	// `<street>, <hn>` comma to a space — the OA/feed layout. A no-op for templates that already
+	// space-join (the substring isn't present), and skipped when the house number was dropped above.
+	if (order === "native" && separator === ", " && opts.nativeHouseJoin === "space" && components.house_number) {
+		raw = raw.replace(
+			`${components.street}, ${components.house_number}`,
+			`${components.street} ${components.house_number}`
+		)
+	}
+
+	// `postcodeShape: "as-source"` (see {@link LocaleSynthesisOpts.postcodeShape}): rewrite BOTH raw and
+	// the component back to the source's own surface (NL glued `1011AB`). Post-render because the
+	// OpenCage NL template normalizes the postcode to the spaced form no matter what it's given.
+	if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
+		const sourceForm = base.postcode.trim()
+
+		if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
+			raw = raw.replace(components.postcode, sourceForm)
+			components.postcode = sourceForm
+		}
+	}
+
+	// Every component must align — drop the row if the template didn't surface one verbatim, or a
+	// numeric component collides with a neighbouring digit run.
+	for (const value of Object.values(components)) {
+		if (!value || !tokenPresent(raw, value)) return null
+	}
+
+	return { raw, components, locale: LOCALE_TAG[country] ?? country.toLowerCase() }
+}
+
+/**
+ * German wrapper over {@link synthesizeLocaleRow}. Kept for the build-german-slice caller + tests.
+ */
+export function synthesizeGermanRow(
+	base: LocaleBaseTuple,
+	opts: LocaleSynthesisOpts = {}
+): SynthesizedLocaleRow | null {
+	return synthesizeLocaleRow(base, "DE", opts)
+}

package/lib/synthesizers/street.ts

@@ -21,9 +21,9 @@
 
 import { isPresent } from "@mailwoman/core/objects"
 
-import { decomposeStreet } from "#adapters/tiger/street-decompose"
 import { pick, tieredNumber } from "#synthesizers/utils"
 import type { CanonicalRow } from "#types"
+import { decomposeStreet } from "#us/adapters/tiger/street-decompose"
 
 // Hand-curated US street name pool. Real frequency-weighted street names — sampled
 // from US Census TIGER 2024 top-1000 by occurrence count. Keep ~50 entries so the

package/lib/synthesizers/utils.ts

@@ -26,6 +26,7 @@
  *   effect is measured; see the note there.
  */
 
+import type { BIOLabel, ComponentTag } from "@mailwoman/codex/component"
 import {
 	US_STREET_SUFFIX_PREFERRED_ABBR,
 	US_UNIT_DESIGNATOR_PREFERRED_ABBR,
@@ -36,7 +37,6 @@ import {
 import { isPresent } from "@mailwoman/core/objects"
 import { mulberry32 } from "@mailwoman/core/random"
 import { escapeRegExp } from "@mailwoman/core/strings/regexp"
-import type { BIOLabel, ComponentTag } from "@mailwoman/core/types"
 import { stripCombiningMarks } from "@mailwoman/normalize/fold"
 
 import type { CanonicalRow, LabeledRow, QuarantinedRow } from "#types"
@@ -158,7 +158,7 @@ export const accentStrip: Augmentation = (row) => {
 	return withAugmentation(row, "accent-strip", stripped, newComponents)
 }
 
-// --- typo injection (#530) -------------------------------------------------
+// The typo-injection helpers below keep generated rows reproducible.
 // The Phase-1 deferral asked for a "seed-aware API" so the corpus stays reproducible. Resolution:
 // seed the PRNG from the row's own `source_id` — deterministic per row, no global state, fits the
 // existing `(row) => CanonicalRow | null` signature unchanged.

package/lib/tools/audit.ts

@@ -15,9 +15,10 @@
  *   Emits warnings to stderr and the audit table to stdout; never throws on an empty corpus.
  */
 
-import { pathExists, readDirectory, readLocalBuffer, readLocalJSONFile } from "@mailwoman/core/fs/readers"
+import { pathExists, readLocalBuffer, readLocalJSONFile } from "@mailwoman/core/fs/readers"
 import { basename, join, type PathBuilderLike } from "path-ts"
 import { TextSpliterator } from "spliterator"
+import { Globerator } from "spliterator/node/fs"
 
 /**
  * Share of a slice one source may hold before the mix is flagged as dominated by it.
@@ -116,7 +117,7 @@ async function scanSlices(corpusDir: PathBuilderLike, sampleCount: number): Prom
 
 		if (!(await pathExists(splitDir))) continue
 
-		const files = (await readDirectory(splitDir)).filter((f) => f.endsWith(".parquet")).toSorted()
+		const files = await Globerator.files("parquet", { cwd: splitDir, absolute: false }).toSorted()
 
 		stats.totalFiles += files.length
 		const sampleEvery = Math.max(1, Math.floor(files.length / sampleCount))

package/lib/tools/corpus-stats.ts

@@ -37,9 +37,9 @@
  */
 
 import { ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { readDirectory, statPath } from "@mailwoman/core/fs/readers"
+import { statPath } from "@mailwoman/core/fs/readers"
 import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
-import { join } from "path-ts"
+import { Globerator } from "spliterator/node/fs"
 
 import { accumulateCooccurrences, createCooccurrenceStats, streamTokenLabelRows } from "#utils/slice-stats"
 
@@ -55,7 +55,7 @@ async function discoverSlices(slicesArg: string): Promise<string[]> {
 	const stat = await statPath(slicesArg)
 
 	if (stat.isDirectory()) {
-		return (await readDirectory(slicesArg)).filter((f) => f.endsWith(".parquet")).map((f) => join(slicesArg, f))
+		return await Globerator.files("parquet", { cwd: slicesArg }).toArray()
 	}
 
 	if (stat.isFile() && slicesArg.endsWith(".parquet")) return [slicesArg]

package/lib/tools/fetch/download/index.ts

@@ -0,0 +1,10 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Public entry point for corpus download utilities.
+ */
+
+export * from "#tools/fetch/download/manifest"
+export * from "#tools/fetch/download/network"
+export * from "#tools/fetch/download/retry"

package/lib/tools/fetch/download/manifest.ts

@@ -0,0 +1,102 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Reads and writes corpus-source manifests.
+ */
+
+import { pathExists, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
+import { movePath, writeLocalTextFile } from "@mailwoman/core/fs/writers"
+import { tryParsingJSON } from "@mailwoman/core/json"
+
+import { HTTPStatusError, type SourceManifest } from "#tools/fetch/download/network"
+
+/**
+ * Read a MANIFEST.json; `null` when missing or corrupt (callers re-fetch from scratch).
+ */
+export async function readManifest<T>(path: string): Promise<T | null> {
+	if (!(await pathExists(path))) return null
+
+	// A read failure (e.g. the file vanished after the existsSync probe) maps to null like corrupt
+	// JSON does; tryParsingJSON returns null for the non-string sentinel.
+	const text = await readLocalTextFile(path).catch(() => null)
+
+	return tryParsingJSON<T>(text)
+}
+
+/**
+ * Load manifest entries into a map so untouched keys survive a partial re-fetch.
+ */
+export async function loadManifestEntries<T>(path: string, key: (entry: T) => string): Promise<Map<string, T>> {
+	const entries = new Map<string, T>()
+	const parsed = await readManifest<T[]>(path)
+
+	for (const entry of parsed ?? []) {
+		entries.set(key(entry), entry)
+	}
+
+	return entries
+}
+
+/**
+ * Write a MANIFEST.json in the house shape: pretty-printed, trailing newline.
+ */
+export async function writeManifest(path: string, manifest: unknown): Promise<void> {
+	await writeLocalTextFile(JSON.stringify(manifest, null, 2) + "\n", path)
+}
+
+/**
+ * The sibling `MANIFEST.json` shape for a source that is a COLLECTION of files behind one portal (a monthly register
+ * published per region, per industry, or per first letter). It carries what a trained artifact has to be able to cite
+ * later: the license the portal labels the data with, the attribution wording it requires, and one
+ * {@link SourceManifest} per file.
+ */
+export interface SourceCollectionManifest {
+	source: string
+	source_url: string
+	license: string
+	attribution: string
+	downloaded_at: string
+	files: SourceManifest[]
+}
+
+/**
+ * Pipe a response body to `dest` and answer the byte count. The one primitive the portal fetchers share when the
+ * request is not a bare GET — a session cookie, a CSRF header, or a form POST stands between the listing and the file,
+ * so {@link streamDownload}'s URL-only contract does not fit and each module builds its own `Response` first. Writes a
+ * `.tmp` sibling and renames, so an interrupted transfer never lands at the final path looking complete.
+ */
+export async function streamBodyToFile(res: Response, dest: string): Promise<number> {
+	if (!res.body) throw new HTTPStatusError(res.status, `HTTP ${res.status} with no body — ${res.url}`)
+	let bytes = 0
+
+	const counter = new TransformStream<Uint8Array, Uint8Array>({
+		transform(chunk, controller) {
+			bytes += chunk.byteLength
+			controller.enqueue(chunk)
+		},
+	})
+
+	const tmp = dest + ".tmp"
+	await pipeline(Readable.fromWeb(res.body.pipeThrough(counter)), openWriteStream(tmp))
+	await movePath(tmp, dest)
+
+	return bytes
+}
+
+/**
+ * The per-file entries of a {@link SourceCollectionManifest} keyed by file name, or an empty map when the manifest is
+ * missing or not in the collection shape, so a re-run after an interruption fetches only what is missing. The
+ * single-file modules' `loadManifestEntries` reads a bare array and is not this.
+ */
+export async function loadCollectionFiles(path: string): Promise<Map<string, SourceManifest>> {
+	const parsed = await readManifest<SourceCollectionManifest>(path)
+	const entries = new Map<string, SourceManifest>()
+
+	for (const entry of Array.isArray(parsed?.files) ? parsed.files : []) {
+		entries.set(entry.filename, entry)
+	}
+
+	return entries
+}