@mailwoman/corpus 8.6.0 → 9.1.0

package/src/tools/fetch/nppes.ts

@@ -24,19 +24,16 @@
 /* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
    procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
 
-import { execFile } from "node:child_process"
 import { existsSync, mkdirSync, statSync } from "node:fs"
 import { rm } from "node:fs/promises"
-import { join } from "node:path"
-import { promisify } from "node:util"
+import { basename, join } from "node:path"
 
+import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
 import { sha256File } from "@mailwoman/core/utils"
 
 import type { BaseFetchOptions, FetchSummary } from "./download.ts"
 import { downloadToFile, readManifest, writeManifest } from "./download.ts"
 
-const execFileAsync = promisify(execFile)
-
 const INDEX_URL = "https://download.cms.gov/nppes/NPI_Files.html"
 const BASE_URL = "https://download.cms.gov/nppes"
 const SLUG = "usgov-nppes"
@@ -74,18 +71,13 @@ async function discoverLatestZip(): Promise<string | undefined> {
 }
 
 /**
- * Extract the main registry CSV name (npidata_pfile_*.csv) from a ZIP's `unzip -l` listing.
+ * The main registry CSV (npidata_pfile_*.csv), which the archive also carries alongside a header file and a per-month
+ * change file.
  */
 async function findNpidataCSV(zipPath: string): Promise<string | undefined> {
-	const listing = await execFileAsync("unzip", ["-l", zipPath])
-
-	for (const line of listing.stdout.split("\n")) {
-		const match = /npidata_pfile\S+\.csv/i.exec(line)
-
-		if (match?.[0]) return match[0]
-	}
+	const entries = await listZipEntries(zipPath)
 
-	return undefined
+	return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name
 }
 
 export async function fetchNPPES(options: FetchNPPESOptions, report?: (line: string) => void): Promise<FetchSummary> {
@@ -148,9 +140,10 @@ export async function fetchNPPES(options: FetchNPPESOptions, report?: (line: str
 	}
 
 	report?.(`  Extracting: ${csvName}`)
-	await execFileAsync("unzip", ["-o", "-j", zipDest, csvName, "-d", destDir])
 
-	const csvDest = join(destDir, csvName)
+	const csvDest = join(destDir, basename(csvName))
+
+	await extractZipEntry(zipDest, csvName, csvDest)
 	const csvSize = statSync(csvDest).size
 	const csvSha = await sha256File(csvDest)
 	report?.(`  CSV size: ${(csvSize / 1024 / 1024).toFixed(1)} MB`)

package/src/tools/fetch/openaddresses.ts

@@ -44,7 +44,7 @@
  *   ```sh
  *   # With token (preferred). Default country: ca. Supports any OA country code (us-west, fr, …)
  *   OA_BATCH_TOKEN=<token> mailwoman corpus fetch openaddresses --country ca \
- *     --out-root /mnt/playpen/mailwoman-data/corpus/sources
+ *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
  *
  *   # Without token (will detect + print instructions, then report the failure):
  *   mailwoman corpus fetch openaddresses --country ca

package/src/tools/fetch/ourairports.ts

@@ -0,0 +1,166 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the OurAirports CSV dumps — the VENUE side of the sub-venue corpus arc (#35).
+ *
+ *   Source : https://davidmegginson.github.io/ourairports-data/ (the project's own GitHub Pages
+ *            mirror of the nightly export; `ourairports.com/data/` redirects here).
+ *   License: PUBLIC DOMAIN. OurAirports places its data in the public domain and asks only for a
+ *            courtesy credit — no attribution obligation rides on a derived shard, which makes this
+ *            the one transport source in the arc with no licensing question at all. Tier A.
+ *
+ *   ## What it is good for, and what it is not
+ *
+ *   `airports.csv` is every airport on earth with ICAO/IATA codes, coordinates, `municipality`, and
+ *   `iso_country` — 12.7 MB, ~83,000 rows as of 2026-08-04. That is the CONTAINING VENUE for a
+ *   `<sub-venue>, <venue>, <street>, <locality>, <postcode>` corpus line, and it is better at that job
+ *   than OSM: every row is named, the name is canonical, and `municipality` gives the locality without
+ *   a spatial join.
+ *
+ *   It carries NO interior structure. There is no terminal, concourse, gate or pier table — the
+ *   corpus task says as much ("Good for the venue side of each pair, weaker on interior structure")
+ *   and a row-level read confirms it. Pair it with the OSM `aeroway` extractor
+ *   (`@mailwoman/osm/sdk`'s `extractOSMSubVenues`), which is where the sub-venue half comes from.
+ *
+ *   ## Why `downloadToFile` and not `APIClient`
+ *
+ *   `AGENTS.md` routes HTTP through `APIClient`, and that rule is about API REQUESTS — small bodies,
+ *   repeated calls, rate-limited hosts. This is four static file transfers against a GitHub Pages CDN
+ *   with no rate limit and nothing to pace, run once per refresh. It uses the same `downloadToFile`
+ *   every other module in this `fetch/` family uses, which is where the retry and timeout live.
+ *   The Wikidata sibling (`wikidata-subvenue.ts`) IS an API client and is built on `APIClient`
+ *   accordingly; the split between the two is the one `AGENTS.md` draws.
+ *
+ *   Invoke via `mailwoman corpus fetch ourairports --out-root <path>`.
+ */
+
+import { mkdirSync } from "node:fs"
+import { join } from "node:path"
+
+import { sha256File } from "@mailwoman/core/utils"
+
+import type { BaseFetchOptions, FetchSummary } from "./download.ts"
+import { downloadToFile, writeManifest } from "./download.ts"
+
+const SLUG = "ourairports"
+
+/**
+ * The GitHub Pages mirror the project itself publishes. `ourairports.com/data/*.csv` 302s here, so pointing at the
+ * mirror directly saves a redirect and is the URL the project's own README gives.
+ */
+const BASE_URL = "https://davidmegginson.github.io/ourairports-data"
+
+/**
+ * The files worth having, and why each one.
+ *
+ * `airports.csv` is the payload. The other three are small joins that turn its codes into text: `countries.csv` and
+ * `regions.csv` expand `iso_country`/`iso_region` into names (a corpus line needs "Germany", not "DE"), and
+ * `runways.csv` is the only file carrying per-airport sub-structure of any kind — runway designators, which are NOT
+ * sub-venue designators (nobody addresses mail to a runway) but are worth having on disk as the negative class if the
+ * shard ever needs one.
+ */
+const FILES = ["airports.csv", "countries.csv", "regions.csv", "runways.csv"] as const
+
+export type FetchOurAirportsOptions = BaseFetchOptions
+
+interface OurAirportsFileEntry {
+	filename: string
+	source_url: string
+	sha256: string
+	bytes: number
+	/**
+	 * The upstream `Last-Modified`, when the CDN gave one. This is the DATA's vintage; `downloaded_at` is only when we
+	 * asked. `corpus/AGENTS.md` has the standing warning that a file's mtime is not its data's vintage — recording the
+	 * upstream header is how a later refresh decision gets made on the right number.
+	 */
+	last_modified: string | null
+}
+
+interface OurAirportsManifest {
+	source: string
+	base_url: string
+	license: string
+	downloaded_at: string
+	files: OurAirportsFileEntry[]
+}
+
+/**
+ * Read the upstream `Last-Modified` with a HEAD. Returns `null` on any failure — provenance metadata is nice to have
+ * and must never fail a download that otherwise succeeded.
+ */
+async function readLastModified(url: string): Promise<string | null> {
+	try {
+		const res = await fetch(url, { method: "HEAD", signal: AbortSignal.timeout(30_000) })
+
+		return res.ok ? res.headers.get("last-modified") : null
+	} catch {
+		return null
+	}
+}
+
+/**
+ * Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `MANIFEST.json` carrying each file's
+ * origin URL, sha256, byte count and upstream `Last-Modified`.
+ */
+export async function fetchOurAirports(
+	options: FetchOurAirportsOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	mkdirSync(destDir, { recursive: true })
+
+	const entries: OurAirportsFileEntry[] = []
+	const failedCodes: string[] = []
+	let fetched = 0
+	let failed = 0
+
+	for (const filename of FILES) {
+		const url = `${BASE_URL}/${filename}`
+		const dest = join(destDir, filename)
+
+		report?.(`=== ${SLUG} / ${filename}`)
+
+		try {
+			const [{ bytes }, lastModified] = await Promise.all([
+				downloadToFile({
+					url,
+					dest,
+					timeoutMs: 600_000,
+					retries: 2,
+					headers: { "Accept-Encoding": "gzip, br" },
+					report,
+				}),
+				readLastModified(url),
+			])
+
+			entries.push({
+				filename,
+				source_url: url,
+				sha256: await sha256File(dest),
+				bytes,
+				last_modified: lastModified,
+			})
+
+			fetched++
+		} catch (error) {
+			report?.(`✗ ${filename}: ${error instanceof Error ? error.message : String(error)}`)
+			failedCodes.push(filename)
+
+			failed++
+		}
+	}
+
+	const manifest: OurAirportsManifest = {
+		source: "OurAirports",
+		base_url: BASE_URL,
+		license: "public domain (courtesy credit requested)",
+		downloaded_at: new Date().toISOString(),
+		files: entries,
+	}
+
+	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+
+	return { fetched, skipped: 0, failed, failedCodes }
+}

package/src/tools/fetch/wikidata-subvenue.ts

@@ -0,0 +1,386 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the multilingual sub-venue designator vocabulary from Wikidata (#35 wave 1).
+ *
+ *   Source : https://query.wikidata.org/sparql (the Wikidata Query Service).
+ *   License: CC0. Wikidata's data is public-domain dedicated, so nothing rides on a derived shard.
+ *            Tier A.
+ *
+ *   ## The pull is CLASS labels, not instance names — and that inversion is the whole design
+ *
+ *   The obvious read of "Wikidata for localized designators" is: fetch every airport terminal entity
+ *   and read its name in each language. That was tried first and it is the WRONG query. Wikidata
+ *   knows 246 items that are `instance of / subclass of*` airport terminal, carrying 775 labels
+ *   between them (measured 2026-08-04), and most of those labels are proper names that translate
+ *   verbatim — `TWA Flight Center` is spelled `TWA Flight Center` in fifteen languages. The
+ *   instance layer is thin and its localization is mostly a no-op.
+ *
+ *   The DESIGNATOR is the label of the CLASS. `wd:Q849706` ("airport terminal") is labelled `Terminal`
+ *   in German, `terminal aéroportuaire` in French, `ターミナルビル` in Japanese, `航站楼` in Chinese —
+ *   and `skos:altLabel` adds the aliases (`Abfertigungsgebäude`, `Flughafenterminal`, `aerostazione`).
+ *   Eight concept ids yield 877 label+alias rows across 174 languages, which is the vocabulary the
+ *   corpus task asked for and the instance query does not contain. {@link SUBVENUE_CONCEPTS} is that
+ *   list of ids; {@link buildDesignatorLabelQuery} is that query.
+ *
+ *   Instance labels are fetched too ({@link buildTerminalInstanceQuery}), for a different job: they
+ *   are ATTESTED USAGE — evidence of how a designator combines with a modifier or an identifier in
+ *   running text. 775 rows is small, and it is a validation set, not a vocabulary.
+ *
+ *   ## What a caller must NOT do with the output
+ *
+ *   A class label is a CONCEPT NAME, not a designator as written in an address. Q849706's Spanish
+ *   label is `terminal aeroportuaria` and its French is `terminal d'aéroport`; nobody writes either on
+ *   an envelope, they write `Terminal`. Q247739's Spanish is `puerta de embarque` where the addressed
+ *   form is `Puerta`. So this fetch produces CANDIDATE SURFACES that need a head-noun/curation pass
+ *   before any of them reaches `neural/venue-structure.ts`'s designator vocabulary — the lexicon
+ *   builder marks every one `curated: false` and the burden of promotion is on a human. Wiring the raw
+ *   pull straight into the span proposer would admit multi-word phrases that match nothing and, worse,
+ *   admit `hall` in a language where it names an ordinary room.
+ *
+ *   ## Why `APIClient` here when the OurAirports sibling uses `downloadToFile`
+ *
+ *   This is the API-request side of `AGENTS.md`'s split: small JSON bodies, several calls per run, and
+ *   a host that publishes a rate policy and enforces it with 429s. Pacing, bounded `Retry-After`-aware
+ *   retry, response caching and `ResourceError` mapping all earn their keep, so it extends
+ *   {@link APIClient}. `ourairports.ts` is four static file transfers off a CDN and correctly does not.
+ *
+ *   WDQS also REQUIRES a descriptive `User-Agent` naming the tool and a contact — an anonymous or
+ *   library-default agent is blocked outright by the Wikimedia user-agent policy. See
+ *   {@link WIKIDATA_USER_AGENT}.
+ *
+ *   Invoke via `mailwoman corpus fetch wikidata-subvenue --out-root <path>`.
+ */
+
+import { mkdirSync } from "node:fs"
+import { writeFile } from "node:fs/promises"
+import { join } from "node:path"
+
+import { APIClient, type ClockLike } from "@mailwoman/core/api"
+import { buildDiskStorage } from "@mailwoman/core/api/disk-storage"
+import { sha256File } from "@mailwoman/core/utils"
+
+import type { BaseFetchOptions, FetchSummary } from "./download.ts"
+import { writeManifest } from "./download.ts"
+
+const SLUG = "wikidata-subvenue"
+
+/**
+ * The SPARQL endpoint. Public, no credential.
+ */
+export const WDQS_ENDPOINT = "https://query.wikidata.org/sparql"
+
+/**
+ * The `User-Agent` every request carries.
+ *
+ * NOT decoration. The Wikimedia user-agent policy blocks requests whose agent is absent, generic, or a library default,
+ * and WDQS enforces it — an unidentified client gets a 403 that no amount of retrying fixes. The policy asks for a tool
+ * name, a URL, and a contact address, all three of which are here.
+ */
+export const WIKIDATA_USER_AGENT =
+	"mailwoman/1.0 (https://github.com/sister-software/mailwoman; teffen@sister.software) corpus-subvenue-fetch"
+
+/**
+ * Minimum spacing between dispatches, in milliseconds.
+ *
+ * WDQS's published limit is expressed as processing-time budget rather than a request rate, and this run issues fewer
+ * than a dozen queries total, so the number is chosen for politeness rather than to sit against a ceiling: one query
+ * per second is far inside anything WDQS objects to, and at this volume the whole fetch still completes in seconds.
+ *
+ * Set as `minRequestIntervalMs` rather than `requestsPerMinute` deliberately — `AGENTS.md` records that
+ * `requestsPerMinute` is a BUDGET model whose cooldown lets N requests go out back to back, so it does not deliver N
+ * per minute and is not the gate that holds a rate. The interval is.
+ */
+const WDQS_MIN_REQUEST_INTERVAL_MS = 1000
+
+/**
+ * Per-attempt socket-inactivity timeout. WDQS's own query timeout is 60 seconds and it answers with a 500 when a query
+ * exceeds it, so a client timeout below that would turn a server-side timeout into a client-side one and lose the error
+ * body that says which query was too expensive.
+ */
+const WDQS_REQUEST_TIMEOUT_MS = 90_000
+
+/**
+ * Total attempts (including the first) before giving up on a 429/5xx.
+ */
+const WDQS_MAX_ATTEMPTS = 3
+
+/**
+ * How long a cached SPARQL response stays fresh. A week: the class labels this pulls change on the timescale at which
+ * someone edits a Wikidata concept's German alias, which is to say rarely, and a re-run inside a working session should
+ * not re-ask.
+ */
+const WDQS_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000
+
+/**
+ * One Wikidata concept whose labels are a designator's multilingual surface set.
+ *
+ * `designatorID` is the mailwoman-side vocabulary term, matching `neural/venue-structure.ts`'s
+ * `VENUE_STRUCTURE_DESIGNATORS` wherever the two overlap. `qid` was resolved by `wbsearchentities` and hand-checked
+ * against the entity's English description (recorded below) on 2026-08-04 — a QID picked by search alone is how you end
+ * up pulling the labels of a Bronx neighbourhood called Concourse.
+ *
+ * `wing` is ABSENT and that is a finding, not an oversight: Wikidata has no clean concept for "wing of a building".
+ * `wbsearchentities` for "wing" returns a surname, two English villages, a rugby position and a drone company. Since
+ * `wing` is the single most valuable designator in the arc — `West Wing` is the one modifier case that already parses,
+ * and `East Wing` is the one that does not — its localized surfaces have to come from somewhere else. See the wave-1
+ * report.
+ */
+export interface SubVenueConcept {
+	designatorID: string
+	qid: string
+	/**
+	 * The entity's English description, recorded so a future reader can tell at a glance whether the QID still names what
+	 * we think it names.
+	 */
+	gloss: string
+}
+
+/**
+ * The concept table. Eight ids, each verified against its English description on 2026-08-04.
+ */
+export const SUBVENUE_CONCEPTS: readonly SubVenueConcept[] = [
+	{ designatorID: "terminal", qid: "Q849706", gloss: "airport terminal — part of an airport" },
+	{ designatorID: "gate", qid: "Q247739", gloss: "gate — airport facility for passenger loading/unloading" },
+	{ designatorID: "concourse", qid: "Q862212", gloss: "concourse — place where pathways or roads meet" },
+	{ designatorID: "campus", qid: "Q209465", gloss: "campus — cluster of buildings used by an educational institution" },
+	{ designatorID: "building", qid: "Q41176", gloss: "building — structure with a roof and walls" },
+	{ designatorID: "arcade", qid: "Q186637", gloss: "arcade — covered walk enclosed by a line of arches" },
+	{ designatorID: "hall", qid: "Q240854", gloss: "hall — large enclosed room" },
+	{ designatorID: "satellite", qid: "Q15990706", gloss: "satellite terminal — detached airport building" },
+]
+
+/**
+ * The class whose instances are fetched for attested-usage evidence: airport terminal.
+ */
+const TERMINAL_CLASS_QID = "Q849706"
+
+/**
+ * Build the class-label query: `rdfs:label` and `skos:altLabel` for every concept, in every language, tagged with which
+ * of the two it came from so the lexicon can rank a label above an alias.
+ *
+ * `VALUES` rather than a property path over the whole class tree — the concept list is closed and hand-verified, and a
+ * `wdt:P279*` walk from `building` would drag in every structure type on earth.
+ */
+export function buildDesignatorLabelQuery(concepts: readonly SubVenueConcept[] = SUBVENUE_CONCEPTS): string {
+	const values = concepts.map((c) => `wd:${c.qid}`).join(" ")
+
+	return `SELECT ?item ?lang ?label ?kind WHERE {
+  VALUES ?item { ${values} }
+  { ?item rdfs:label ?label . BIND("label" AS ?kind) }
+  UNION
+  { ?item skos:altLabel ?label . BIND("alt" AS ?kind) }
+  BIND(LANG(?label) AS ?lang)
+}`
+}
+
+/**
+ * Build the instance-label query — every item that is an `instance of` (through any `subclass of` chain) an airport
+ * terminal, with all of its labels. Measured at 246 items / 775 labels on 2026-08-04, well inside WDQS's 60-second
+ * budget.
+ *
+ * A caveat worth knowing before trusting a row: Wikidata's P31 on these is not clean. `Q1322696` (Kigali International
+ * Airport) is typed as an airport terminal, so the result set mixes AIRPORTS in with terminals. The consumer filters;
+ * this module fetches what the query returns.
+ */
+export function buildTerminalInstanceQuery(classQID: string = TERMINAL_CLASS_QID): string {
+	return `SELECT ?item ?lang ?label WHERE {
+  ?item wdt:P31/wdt:P279* wd:${classQID} .
+  ?item rdfs:label ?label .
+  BIND(LANG(?label) AS ?lang)
+}`
+}
+
+/**
+ * The SPARQL JSON results shape, narrowed to the two column types these queries produce.
+ */
+export interface SPARQLResults {
+	results: {
+		bindings: Array<Record<string, { type: string; value: string; "xml:lang"?: string }>>
+	}
+}
+
+/**
+ * Whether a decoded body is a SPARQL results envelope. Used as the cache's write validator so an HTML error page served
+ * under a 200 is never persisted for the next run to destructure into `undefined`.
+ */
+export function isSPARQLResults(value: unknown): value is SPARQLResults {
+	return typeof value === "object" && value !== null && Array.isArray((value as SPARQLResults).results?.bindings)
+}
+
+export interface CreateWikidataClientOptions {
+	/**
+	 * On-disk response-cache root. Defaults to a `http-cache` directory beside the fetch output.
+	 */
+	cacheDir: string
+	/**
+	 * Time source powering the pacer and the retry backoff. Defaults to the system clock; tests inject a fake so no suite
+	 * ever sleeps a real second.
+	 */
+	clock?: ClockLike
+	/**
+	 * Axios overrides, merged over this client's defaults. THE TEST SEAM — pass an `adapter` and no live call is made.
+	 * Overriding `headers` wholesale would drop the required `User-Agent`, so don't.
+	 */
+	axios?: ConstructorParameters<typeof APIClient>[0]["axios"]
+}
+
+/**
+ * A Wikidata Query Service client: paced, retrying, disk-cached, and correctly identified.
+ */
+export class WikidataClient extends APIClient {
+	/**
+	 * Run one SPARQL query and return its results envelope.
+	 */
+	public async query(sparql: string): Promise<SPARQLResults> {
+		const url = new URL(WDQS_ENDPOINT)
+		url.searchParams.set("query", sparql)
+
+		const response = await this.fetch<SPARQLResults>({ url: url.toString() })
+
+		return response.data
+	}
+}
+
+/**
+ * Construct a {@link WikidataClient} with every default resolved.
+ */
+export function createWikidataClient(options: CreateWikidataClientOptions): WikidataClient {
+	return new WikidataClient({
+		displayName: "Wikidata Query Service",
+		minRequestIntervalMs: WDQS_MIN_REQUEST_INTERVAL_MS,
+		retry: { maxAttempts: WDQS_MAX_ATTEMPTS },
+		clock: options.clock,
+		caching: {
+			storage: buildDiskStorage({
+				directory: options.cacheDir,
+				// `value.data` is the cached RESPONSE; `value.data.data` is its body. Passing the response here
+				// instead of the body is a silent-failure trap — the predicate returns false for every entry and
+				// every run re-fetches while logging "rejected by the configured validate() predicate". Caught
+				// on the first live run, 2026-08-04.
+				validate: (value) => isSPARQLResults(value.data?.data),
+			}),
+			ttl: WDQS_CACHE_TTL_MS,
+			// The TTL is chosen against Wikidata's edit cadence; letting a CDN header override it would
+			// silently replace that reasoning with whatever varnish in front of WDQS happens to send.
+			interpretHeader: false,
+		},
+		axios: {
+			headers: {
+				"User-Agent": WIKIDATA_USER_AGENT,
+				Accept: "application/sparql-results+json",
+			},
+			timeout: WDQS_REQUEST_TIMEOUT_MS,
+			responseType: "json",
+			// Axios hands back the RAW STRING when a body fails to parse unless this is off. WDQS serves an
+			// HTML error page under some failures, and returning that as `SPARQLResults` would surface as an
+			// `undefined` destructure far from the cause.
+			transitional: { silentJSONParsing: false },
+			...options.axios,
+		},
+	})
+}
+
+export type FetchWikidataSubVenueOptions = BaseFetchOptions
+
+interface WikidataFileEntry {
+	filename: string
+	query: string
+	rows: number
+	sha256: string
+	bytes: number
+}
+
+interface WikidataManifest {
+	source: string
+	endpoint: string
+	license: string
+	user_agent: string
+	downloaded_at: string
+	concepts: readonly SubVenueConcept[]
+	files: WikidataFileEntry[]
+}
+
+/**
+ * Write a JSON payload and return its manifest entry.
+ */
+async function writePayload(
+	destDir: string,
+	filename: string,
+	query: string,
+	results: SPARQLResults
+): Promise<WikidataFileEntry> {
+	const path = join(destDir, filename)
+	const body = JSON.stringify(results, null, 2) + "\n"
+	await writeFile(path, body)
+
+	return {
+		filename,
+		query,
+		rows: results.results.bindings.length,
+		sha256: await sha256File(path),
+		bytes: Buffer.byteLength(body),
+	}
+}
+
+/**
+ * Run both queries and write their raw SPARQL JSON into `<outRoot>/wikidata-subvenue/`, with a `MANIFEST.json` carrying
+ * the endpoint, the exact queries, the concept table, row counts and sha256s.
+ *
+ * The RAW envelope is written rather than a reshaped one on purpose: the lexicon build is a separate, pure step
+ * (`sub-venue-lexicon.ts`) and keeping the fetch output byte-faithful to what WDQS served means a lexicon regeneration
+ * never needs the network.
+ */
+export async function fetchWikidataSubVenue(
+	options: FetchWikidataSubVenueOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	mkdirSync(destDir, { recursive: true })
+
+	await using client = createWikidataClient({ cacheDir: join(destDir, "http-cache") })
+
+	const jobs: Array<{ filename: string; query: string }> = [
+		{ filename: "designator-labels.json", query: buildDesignatorLabelQuery() },
+		{ filename: "terminal-instance-labels.json", query: buildTerminalInstanceQuery() },
+	]
+
+	const files: WikidataFileEntry[] = []
+	const failedCodes: string[] = []
+	let fetched = 0
+	let failed = 0
+
+	for (const job of jobs) {
+		report?.(`=== ${SLUG} / ${job.filename}`)
+
+		try {
+			const results = await client.query(job.query)
+			const entry = await writePayload(destDir, job.filename, job.query, results)
+			report?.(`  ${entry.rows} rows, ${entry.bytes} bytes`)
+			files.push(entry)
+
+			fetched++
+		} catch (error) {
+			report?.(`✗ ${job.filename}: ${error instanceof Error ? error.message : String(error)}`)
+			failedCodes.push(job.filename)
+
+			failed++
+		}
+	}
+
+	const manifest: WikidataManifest = {
+		source: "Wikidata Query Service",
+		endpoint: WDQS_ENDPOINT,
+		license: "CC0",
+		user_agent: WIKIDATA_USER_AGENT,
+		downloaded_at: new Date().toISOString(),
+		concepts: SUBVENUE_CONCEPTS,
+		files,
+	}
+
+	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+
+	return { fetched, skipped: 0, failed, failedCodes }
+}

package/src/tools/golden-expand.ts

@@ -51,7 +51,9 @@ import { dirname } from "node:path"
 
 import { ParquetReader } from "@dsnp/parquetjs"
 import { $private } from "@mailwoman/core/env"
-import { dataRootPath, writeJSONL } from "@mailwoman/core/utils"
+import { tryParsingJSON } from "@mailwoman/core/objects"
+import { dataRootPath } from "@mailwoman/core/utils"
+import { createNewlineWriter } from "spliterator"
 
 // ── Types ─────────────────────────────────────────────────────────────────
 
@@ -406,21 +408,17 @@ function parseCandidates(text: string): Candidate[] {
 	// Strip markdown fences the model sometimes wraps around JSON
 	const cleaned = text.replaceAll(/^```(?:json)?\n?|\n?```$/g, "").trim()
 
-	try {
-		const parsed = JSON.parse(cleaned) as unknown
+	const parsed = tryParsingJSON(cleaned)
 
-		if (Array.isArray(parsed)) return parsed as Candidate[]
+	if (Array.isArray(parsed)) return parsed as Candidate[]
 
-		// Some providers wrap in {"variants": [...]} or {"candidates": [...]}
-		if (typeof parsed === "object" && parsed !== null) {
-			for (const key of ["variants", "candidates", "results"]) {
-				const v = (parsed as Record<string, unknown>)[key]
+	// Some providers wrap in {"variants": [...]} or {"candidates": [...]}
+	if (typeof parsed === "object" && parsed !== null) {
+		for (const key of ["variants", "candidates", "results"]) {
+			const v = (parsed as Record<string, unknown>)[key]
 
-				if (Array.isArray(v)) return v as Candidate[]
-			}
+			if (Array.isArray(v)) return v as Candidate[]
 		}
-	} catch {
-		// fall through
 	}
 
 	return []
@@ -555,7 +553,14 @@ export async function expandGolden(
 
 	await Promise.all(workers)
 
-	writeJSONL(outputPath, outRows)
+	{
+		await using out = createNewlineWriter(outputPath)
+
+		for (const row of outRows) {
+			await out.write(JSON.stringify(row))
+		}
+	}
+
 	report?.(`=== summary ===`)
 	report?.(`seeds processed:  ${seeds.length}`)
 	report?.(`candidates kept:  ${kept}`)