@mailwoman/corpus 10.1.0 → 10.2.1

package/lib/synthesizers/utils.ts

@@ -309,6 +309,10 @@ const STATE_ABBR_TO_NAME: Record<string, string> = Object.fromEntries(
  * Expand state abbreviation to full name.
  */
 export const stateExpand: Augmentation = (row) => {
+	// country-branch: the state, directional, street-suffix, unit and ZIP+4
+	// augmentations below read US vocabulary tables.
+	// Per-country abbreviation tables in the codex would let
+	// `defaultAugmentationsForCountry` serve every country.
 	if (row.country !== "US") return null
 	const region = row.components.region
 
@@ -330,6 +334,7 @@ export const stateExpand: Augmentation = (row) => {
  * Abbreviate full state name to alpha-2.
  */
 export const stateAbbreviate: Augmentation = (row) => {
+	// country-branch: see `stateExpand`.
 	if (row.country !== "US") return null
 	const region = row.components.region
 
@@ -365,6 +370,7 @@ const DIRECTIONAL_ABBR_TO_FULL: Record<string, string> = Object.fromEntries(
  * Expand directional abbreviations.
  */
 export const directionalExpand: Augmentation = (row) => {
+	// country-branch: see `stateExpand`.
 	if (row.country !== "US") return null
 	const tagsToCheck: ComponentTag[] = ["street", "street_suffix", "street_prefix"]
 	let changed = false
@@ -393,6 +399,7 @@ export const directionalExpand: Augmentation = (row) => {
  * Abbreviate directional words.
  */
 export const directionalAbbreviate: Augmentation = (row) => {
+	// country-branch: see `stateExpand`.
 	if (row.country !== "US") return null
 	const tagsToCheck: ComponentTag[] = ["street", "street_suffix", "street_prefix"]
 	let changed = false
@@ -425,6 +432,7 @@ export const directionalAbbreviate: Augmentation = (row) => {
  * Abbreviate trailing US street suffix.
  */
 export const streetSuffixAbbreviate: Augmentation = (row) => {
+	// country-branch: see `stateExpand`.
 	if (row.country !== "US") return null
 	const street = row.components.street
 
@@ -454,6 +462,7 @@ export const streetSuffixAbbreviate: Augmentation = (row) => {
  * Expand trailing US street suffix.
  */
 export const streetSuffixExpand: Augmentation = (row) => {
+	// country-branch: see `stateExpand`.
 	if (row.country !== "US") return null
 	const street = row.components.street
 
@@ -482,6 +491,7 @@ export const streetSuffixExpand: Augmentation = (row) => {
  * Abbreviate leading US unit designator.
  */
 export const unitDesignatorAbbreviate: Augmentation = (row) => {
+	// country-branch: see `stateExpand`.
 	if (row.country !== "US") return null
 	const unit = row.components.unit
 
@@ -511,6 +521,7 @@ export const unitDesignatorAbbreviate: Augmentation = (row) => {
  * Expand leading US unit designator.
  */
 export const unitDesignatorExpand: Augmentation = (row) => {
+	// country-branch: see `stateExpand`.
 	if (row.country !== "US") return null
 	const unit = row.components.unit
 
@@ -539,6 +550,7 @@ export const unitDesignatorExpand: Augmentation = (row) => {
  * Drop dash from ZIP+4.
  */
 export const zipPlus4DashDrop: Augmentation = (row) => {
+	// country-branch: see `stateExpand`.
 	if (row.country !== "US") return null
 	const postcode = row.components.postcode
 
@@ -557,6 +569,8 @@ export const zipPlus4DashDrop: Augmentation = (row) => {
  * Drop French street particle.
  */
 export const particleStrip: Augmentation = (row) => {
+	// country-branch: the particle list is French.
+	// A per-language particle table in the codex would replace this.
 	if (row.country !== "FR") return null
 	const particle = row.components.street_prefix_particle
 
@@ -604,6 +618,7 @@ export function defaultAugmentationsForCountry(country: string): readonly Augmen
 	const universal = [caseUpper, caseLower, dropCommas, doubleSpace]
 
 	switch (country) {
+		// country-branch: the US and FR augmentation lists exist because the two tier-1 locales have vocabulary tables. See `stateExpand`.
 		case "US":
 			return [
 				...universal,
@@ -617,6 +632,7 @@ export function defaultAugmentationsForCountry(country: string): readonly Augmen
 				unitDesignatorExpand,
 				zipPlus4DashDrop,
 			]
+		// country-branch: see the US case above.
 		case "FR":
 			return [...universal, accentStrip, particleStrip]
 		default:
@@ -731,7 +747,8 @@ export interface ComposeAdversarialOptions {
 
 	/**
 	 * Separator between venue and address raw.
-	 * Default `", "`.
+	 *
+	 * @defaultValue `", "`
 	 */
 	separator?: string
 
@@ -799,7 +816,7 @@ export function composeAdversarialRow(
 	// Shift address spans by venue + separator and prepend one venue span.
 	const { span_starts: addrStarts, span_ends: addrEnds, span_tags: addrTags } = addressAligned.row
 
-	if (addrStarts === undefined || addrEnds === undefined || addrTags === undefined) {
+	if (!addrStarts || !addrEnds || !addrTags) {
 		throw new Error(
 			`composeAdversarialRow: alignRow returned a labeled row without the span triple ` +
 				`(source=${address.source}, source_id=${address.source_id}) — alignment interface violation`

package/lib/test-kit.ts

@@ -36,7 +36,7 @@ export interface ScratchDir {
  * or a platform holding a handle open must not turn a passing assertion into a failing suite.
  */
 export function useScratchDir(slug: string): ScratchDir {
-	let owned: TemporaryDirectory | undefined
+	let owned: TemporaryDirectory | null = null
 
 	beforeEach(async () => {
 		owned = await temporaryDirectory(`mailwoman-${slug}-`)
@@ -49,7 +49,7 @@ export function useScratchDir(slug: string): ScratchDir {
 			await owned?.[Symbol.asyncDispose]()
 		} catch {}
 
-		owned = undefined
+		owned = null
 	})
 
 	return {

package/lib/tools/audit.ts

@@ -15,7 +15,8 @@
  *   Emits warnings to stderr and the audit table to stdout. never throws on an empty corpus.
  */
 
-import { pathExists, readLocalJSONFile, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { readLocalJSONFile, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { pathExists } from "@mailwoman/core/fs/readers/stat"
 import { basename, type PathBuilderLike, resolvePathBuilder } from "path-ts"
 import { Globerator } from "spliterator/node/fs"
 
@@ -43,11 +44,12 @@ export interface AuditOpts {
 	/**
 	 * Sample at most N parquet files per split when counting sources.
 	 *
-	 * Default 100 for speed.
 	 * Bump to read the full set on a slow run.
 	 *
 	 * The first row of each file determines its source — corpus-v0.2.0+ files are 100%
 	 * source-segregated, so a one-row read is authoritative.
+	 *
+	 * @defaultValue `100` for speed
 	 */
 	sampleFileCount?: number
 }
@@ -134,7 +136,7 @@ function inferSourceFromFilename(filename: string): string {
 	// Return "<unknown>" so the caller flags this case.
 	const m = basename(filename).match(/part-([\w-]+)-\d+\.parquet$/)
 
-	if (m && m[1] !== undefined) return m[1]
+	if (m && m[1]) return m[1]
 
 	return "<unknown>"
 }
@@ -303,7 +305,7 @@ function formatPct(v: number | "—"): string {
 
 function printReport(
 	corpusDir: PathBuilderLike,
-	configPath: PathBuilderLike | undefined,
+	configPath: PathBuilderLike | null,
 	stats: FileCountStats,
 	rows: AuditRow[]
 ): void {
@@ -395,5 +397,5 @@ export async function audit(opts: AuditOpts): Promise<void> {
 
 	const trainStats = stats.bySplit["train"] ?? {}
 	const rows = buildAuditRows(trainStats, config?.sourceWeights ?? {})
-	printReport(opts.corpusDir, opts.configPath, stats, rows)
+	printReport(opts.corpusDir, opts.configPath ?? null, stats, rows)
 }

package/lib/tools/corpus-stats.ts

@@ -37,7 +37,7 @@
  */
 
 import { ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { statPath } from "@mailwoman/core/fs/readers"
+import { statPath } from "@mailwoman/core/fs/readers/stat"
 import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
 import { stringifyJSON } from "@mailwoman/core/json"
 import { PathBuilder } from "path-ts"

package/lib/tools/fetch/atom.ts

@@ -10,12 +10,12 @@
  *   that feed's entries point at files, and both are `<entry>` with `<link>` children.
  *
  *   Parsed with `@mailwoman/core/html/elements` rather than matched with a pattern, because a feed
- *   needs the two things a pattern handles badly. An entry's open tag carries attributes
+ *   needs the two things a pattern handles badly. An entry's open tag holds attributes
  *   (`xml:lang`), and an href escapes the ampersand between its query parameters, so a reader has to
  *   keep attributes and decode entities. XML mode is required: a feed's links self-close, and HTML
  *   mode leaves them open.
  *
- *   Feed-level links are told from entry-level links by the parser rather than by position. Asking
+ *   Feed-level links are told from entry-level links by the parser rather than by position. One request
  *   for `link` and `entry` together yields a `link` only where it is not already inside an `entry`,
  *   because capture runs to the outermost requested element's close.
  */
@@ -86,7 +86,8 @@ export interface AtomEntry {
  */
 export interface AtomFeed {
 	/**
-	 * The feed's own links, which are the ones not inside an entry.
+	 * The feed's own links.
+	 * These sit outside every entry.
 	 */
 	links: readonly AtomLink[]
 	entries: readonly AtomEntry[]
@@ -100,7 +101,7 @@ function atomLinkOf(element: MarkupElement): AtomLink {
 		rel: (element.attributes.rel ?? "").toLowerCase(),
 		type: element.attributes.type ?? null,
 		title: element.attributes.title ?? null,
-		length: length !== undefined && /^\d+$/u.test(length) ? Number(length) : null,
+		length: length && /^\d+$/u.test(length) ? Number(length) : null,
 	}
 }
 
@@ -118,8 +119,7 @@ function atomEntryOf(element: MarkupElement): AtomEntry {
 /**
  * Read a feed's links and entries out of its markup.
  *
- * Streaming rather than whole-document, because a service document may be large:
- * ČÚZK's lists one entry per municipality in 8,373,817 bytes, measured 2026-10-02.
+ * The reader streams because a service document may be large: ČÚZK's lists one entry per municipality.
  *
  * @param chunks The feed's bytes or text, in order.
  * A Node `Readable` satisfies this.
@@ -154,11 +154,11 @@ export async function* feedChunks(text: string): AsyncIterable<string> {
 }
 
 /**
- * The first link carrying `rel`, or `undefined` where the collection holds none.
+ * The first link carrying `rel`, or `null` where the collection holds none.
  *
- * A feed's entry commonly carries several: ČÚZK writes a `describedby` link to its ISO 19139
+ * A feed's entry commonly holds several: ČÚZK writes a `describedby` link to its ISO 19139
  * record beside the `alternate` link that is the data, and either would download as a dataset.
  */
-export function linkWithRel(links: readonly AtomLink[], rel: string): AtomLink | undefined {
-	return links.find((link) => link.rel === rel.toLowerCase())
+export function linkWithRel(links: readonly AtomLink[], rel: string): AtomLink | null {
+	return links.find((link) => link.rel === rel.toLowerCase()) ?? null
 }

package/lib/tools/fetch/cordis.ts

@@ -29,7 +29,7 @@
 
 import { APIClient } from "@mailwoman/core/api"
 import { ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { tryStat } from "@mailwoman/core/fs/readers"
+import { tryStat } from "@mailwoman/core/fs/readers/stat"
 import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
 import { listZipEntries } from "@mailwoman/core/fs/zip"
 import { sha256File } from "@mailwoman/core/hash"
@@ -57,7 +57,8 @@ export function cordisArchiveURL(programme: string): string {
 }
 
 /**
- * The file name one programme's archive is kept under, which is the URL's own last segment.
+ * The file name one programme's archive is kept under.
+ * It is the URL's own last segment.
  */
 export function cordisArchiveFilename(programme: string): string {
 	return `cordis-${programme}projects-csv.zip`
@@ -69,7 +70,8 @@ export function cordisArchiveFilename(programme: string): string {
 export const CORDIS_LEGAL_NOTICE_URL = "https://cordis.europa.eu/about/legal"
 
 /**
- * The directory the archives are written under, which is the adapter's `inputPath`.
+ * The directory the archives are written under.
+ * It is the adapter's `inputPath`.
  */
 const SLUG = CORDIS_ADAPTER_ID
 
@@ -110,7 +112,7 @@ export async function readCORDISArchiveHead(
 
 	return {
 		lastModified: headers?.["last-modified"] ?? null,
-		contentLength: length !== undefined && /^\d+$/u.test(length) ? Number(length) : null,
+		contentLength: length && /^\d+$/u.test(length) ? Number(length) : null,
 	}
 }
 
@@ -120,16 +122,16 @@ export async function readCORDISArchiveHead(
  * The `HEAD` decides first, then the file's own length, then, when asked, its digest.
  */
 export async function isCORDISArchiveCurrent(
-	recorded: CORDISArchiveManifest | undefined,
+	recorded: CORDISArchiveManifest | null,
 	head: CORDISArchiveHead,
 	path: PathBuilderLike,
 	verifyDigest: boolean
 ): Promise<boolean> {
 	if (!recorded) return false
 
-	if (head.lastModified === null && head.contentLength === null) return false
+	if (!head.lastModified && head.contentLength === null) return false
 
-	if (head.lastModified !== null && head.lastModified !== recorded.last_modified) return false
+	if (head.lastModified && head.lastModified !== recorded.last_modified) return false
 
 	if (head.contentLength !== null && head.contentLength !== recorded.bytes) return false
 
@@ -150,7 +152,7 @@ export interface DownloadCORDISOptions {
 	outputDir: PathBuilderLike
 	/**
 	 * Which programmes to fetch.
-	 * Defaults to {@linkcode CORDIS_PROGRAMMES}.
+	 * @defaultValue {@linkcode CORDIS_PROGRAMMES}
 	 */
 	programmes?: readonly string[]
 	/**
@@ -225,7 +227,7 @@ export async function downloadCORDIS(
 
 		const current =
 			!options.force &&
-			(await isCORDISArchiveCurrent(recorded.get(filename), head, archivePath, options.verifyDigest ?? false))
+			(await isCORDISArchiveCurrent(recorded.get(filename) ?? null, head, archivePath, options.verifyDigest ?? false))
 
 		if (current) {
 			report?.(`  ✓ ${programme} already current — no download.`)

package/lib/tools/fetch/download/manifest.ts

@@ -5,7 +5,8 @@
  * @file Reads and writes corpus-source manifests.
  */
 
-import { pathExists, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { pathExists } from "@mailwoman/core/fs/readers/stat"
 import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
 import { movePath, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
 import { tryParsingJSON } from "@mailwoman/core/json"

package/lib/tools/fetch/download/network.ts

@@ -46,7 +46,7 @@ export interface BaseFetchOptions {
 	/**
 	 * Pause between transfer retries, in milliseconds.
 	 *
-	 * Defaults to {@linkcode DEFAULT_RETRY_DELAY_MS}.
+	 * @defaultValue {@linkcode DEFAULT_RETRY_DELAY_MS}
 	 *
 	 * A test that exercises the failure path pays this delay once per retry in real time.
 	 * Measured at 20.1 s for the two failing-transfer cases in `geonames-postal.test.ts`,
@@ -115,19 +115,19 @@ export interface DownloadOptions {
 	/**
 	 * Per-attempt timeout.
 	 *
-	 * Default 10 minutes — these are multi-GB government dumps.
+	 * @defaultValue `10` minutes — these are multi-GB government dumps
 	 */
 	timeoutMs?: number
 	/**
 	 * Extra attempts after the first, taken only on transient statuses or network errors.
 	 *
-	 * Default 0.
+	 * @defaultValue `0`
 	 */
 	retries?: number
 	/**
 	 * Delay between attempts.
 	 *
-	 * Default 5s.
+	 * @defaultValue `5s`
 	 */
 	retryDelayMs?: number
 	headers?: Record<string, string>

package/lib/tools/fetch/download/retry.ts

@@ -5,7 +5,7 @@
  * @file Retries and resumes interrupted corpus downloads.
  */
 
-import { tryStat } from "@mailwoman/core/fs/readers"
+import { tryStat } from "@mailwoman/core/fs/readers/stat"
 import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
 import { movePath } from "@mailwoman/core/fs/writers"
 import { sleep } from "@mailwoman/core/utils/sleep"
@@ -80,10 +80,10 @@ export async function resumableDownload(options: {
 	const { url, dest, headers = {}, maxConnections = 400, retryDelayMs = DEFAULT_RETRY_DELAY_MS, report } = options
 	const tmp = dest + ".tmp"
 	let have = await bytesOnDisk(tmp)
-	let total: number | undefined
+	let total: number | null = null
 
 	for (let connection = 0; connection < maxConnections; connection++) {
-		if (total !== undefined && have >= total) break
+		if (total !== null && have >= total) break
 
 		let res: Response
 
@@ -137,7 +137,7 @@ export async function resumableDownload(options: {
 		}
 	}
 
-	if (total === undefined || have < total) {
+	if (total === null || have < total) {
 		throw new Error(`resumable download stalled at ${have} of ${total ?? "?"} bytes — ${url}`)
 	}
 

package/lib/tools/fetch/geonames/dump.ts

@@ -18,7 +18,8 @@
  *   the basename, so a postal export receives `wrong_format_present` and does not count as coverage.
  */
 
-import { pathExists, readFileHead, readLocalBuffer, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { readFileHead, readLocalBuffer, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { pathExists } from "@mailwoman/core/fs/readers/stat"
 import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
 import { extractZipEntry } from "@mailwoman/core/fs/zip"
 import { sha256File } from "@mailwoman/core/hash"
@@ -50,14 +51,15 @@ export interface FetchGeonamesDumpOptions extends BaseFetchOptions {
 	/**
 	 * Dump directory to read from.
 	 *
-	 * Defaults to GeoNames' own.
 	 * Exists so the 404 and coverage behavior can be exercised against a local server.
+	 *
+	 * @defaultValue GeoNames' own.
 	 */
 	baseURL?: string
 	/**
 	 * Refetch a dump whose `<CC>.txt` already exists.
 	 *
-	 * Default false — the tool fills gaps.
+	 * @defaultValue `false` — the tool fills gaps
 	 */
 	force?: boolean
 }

package/lib/tools/fetch/geonames/postal.ts

@@ -42,13 +42,13 @@ export interface FetchGeonamesPostalOptions extends BaseFetchOptions {
 	/**
 	 * ISO alpha-2 codes in any casing.
 	 *
-	 * Defaults to `{@linkcode GEONAMES_POSTAL_DEFAULT_COUNTRIES}`.
+	 * @defaultValue {@linkcode GEONAMES_POSTAL_DEFAULT_COUNTRIES}
 	 */
 	countries?: readonly string[]
 	/**
 	 * Export directory to read from.
 	 *
-	 * Defaults to GeoNames' own, so the 404-is-coverage behavior can be exercised against a local server.
+	 * @defaultValue GeoNames' own, so the 404-is-coverage behavior can be exercised against a local server.
 	 */
 	baseURL?: string
 }

package/lib/tools/fetch/gleif.ts

@@ -28,12 +28,12 @@
  * streams the member out of the archive rather than reading an extracted copy.
  *
  * The receipt written beside the archive records the URL, the publish date, the byte count and the
- * sha256 of the archive, which identify the bytes a corpus row was read from.
+ * sha256 of the archive. Together they identify the bytes a corpus row was read from.
  */
 
 import { APIClient } from "@mailwoman/core/api"
 import { ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { tryStat } from "@mailwoman/core/fs/readers"
+import { tryStat } from "@mailwoman/core/fs/readers/stat"
 import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
 import { listZipEntries } from "@mailwoman/core/fs/zip"
 import { sha256File } from "@mailwoman/core/hash"
@@ -56,7 +56,8 @@ import { readManifest, resumableDownload, writeManifest } from "#tools/fetch/dow
 export const GLEIF_LATEST_PUBLISH_URL = "https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"
 
 /**
- * The directory the archive and its receipt are written under, which is the adapter's `inputPath`.
+ * The directory the archive and its receipt are written under.
+ * It is the adapter's `inputPath`.
  */
 const SLUG = "gleif"
 
@@ -155,7 +156,8 @@ export async function readGLEIFLatestPublish(client: Pick<APIClient, "fetch">):
 }
 
 /**
- * The archive's file name, which is the last path segment of the URL the API names.
+ * The archive's file name.
+ * It is the last path segment of the URL the API returns.
  */
 export function gleifArchiveFilename(url: string): string {
 	const name = new URL(url).pathname.split("/").at(-1) ?? ""

package/lib/tools/fetch/header.ts

@@ -3,7 +3,7 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- * Reading a delimited file's header line, and refusing one that stops naming a column an adapter
+ * This module reads a delimited file's header line and refuses one that drops a column an adapter
  * indexes.
  *
  * A fetch module checks the file it just wrote against the columns its adapter reads by name,
@@ -22,11 +22,12 @@ import { CSVSpliterator } from "spliterator"
  * Read through `CSVSpliterator`, which is the reader the adapters use, rather than through
  * a split of a fetch module's own: which columns a publisher quotes is a property of
  * the edition, and a second reader would be a second answer to that question.
- * `columnScan: "rows"` keeps the first row from decoding the whole file,
- * which the default `"auto"` does even for one row.
+ * `columnScan: "rows"` keeps the first row from decoding the whole file.
+ *
+ * The default `"auto"` decodes the whole file even for one row.
  *
  * @param path The written file.
- * @param columnDelimiter The publisher's delimiter, which defaults to a comma.
+ * @param columnDelimiter [","] The publisher's delimiter.
  * @throws When the file holds no row at all, so an empty or truncated file reports itself
  * rather than reading as a file with no columns.
  */
@@ -37,7 +38,7 @@ export async function readDelimitedHeader(path: PathBuilderLike, columnDelimiter
 			mode: "array",
 			columnScan: "rows",
 			take: 1,
-			...(columnDelimiter === undefined ? {} : { columnDelimiter }),
+			...(columnDelimiter ? { columnDelimiter } : {}),
 		})
 	)
 
@@ -53,7 +54,8 @@ export async function readDelimitedHeader(path: PathBuilderLike, columnDelimiter
  *
  * @param columns The header {@linkcode readDelimitedHeader} returned.
  * @param required The columns the adapter indexes by name.
- * @param context What is being checked, which opens the message.
+ * @param context What is being checked.
+ * This value opens the message.
  * @throws Naming the columns that are absent, so a renamed column is a reported failure
  * rather than an empty string on every row the adapter emits.
  */