@mailwoman/corpus 9.3.0 → 10.0.0

package/out/international/recipes/locale.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../lib/international/recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAQH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAI9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAuB,MAAM,sBAAsB,CAAA;AAI3G;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,CAAC,EAAE,eAAe,CAAA;IACrB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,eAAe,CAAA;IACtB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;CACjB;AAED,MAAM,WAAW,mBAAmB;IACnC,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,UAAU,EAAE,CAAA;CAC3B;AA6FD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM1D;AAgBD;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAgJhG;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CACjC,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,MAAM,EACf,eAAe,EAAE,MAAM,EACvB,MAAM,EAAE,MAAM,MAAM,GAClB,IAAI,CAgBN;AAED;;;;GAIG;AACH,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAE3G;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,aAAa,EAAE,mBAAmB,EAAE,QAAQ,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,EAAE,CAElH;AAED;;;GAGG;AACH,eAAO,MAAM,YAAY,EAAE,YAmK1B,CAAA"}

package/out/international/recipes/locale.js

@@ -0,0 +1,458 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `locale` slice recipe — the multi-locale generalization of the `german` recipe. Reads REAL
+ *   OpenAddresses tuples for a `--country` (DE/FR/NL/IT/ES), renders each via
+ *   {@link synthesizeLocaleRow} in BOTH orders (`--intl-fraction`, default 0.4 international / the
+ *   rest country-native), aligns to BIO, and emits a labeled JSONL. Generate-mode: it STREAMS each
+ *   source CSV (a streamed zip member for cached zips, plain `createReadStream` for extracted CSVs) and
+ *   reservoir-samples to {@link RESERVOIR_CAP} (so FR/ES countrywide work in bounded memory), then
+ *   draws `--count` rows from the pool with the passed `random`. Ported from
+ *   scripts/build-locale-slice.mjs.
+ *
+ *   The reservoir uses its OWN seeded PRNG ({@link makeMulberry32}, per part), independent of the
+ *   emit `random`, so the input sample is reproducible WITHOUT perturbing the synth/order draws.
+ *
+ *   SURFACE DIVERSITY (#241): two per-country shape draws ride the emit loop, sized by the
+ *   2026-07-02 format-diversity audit against the `openaddresses-{es,nl,it}-sample.jsonl` observed
+ *   forms. ES: the OpenCage template comma-joins the house number (`CALLE MAYOR, 12`) but all 3,000
+ *   eval rows space-join (`CALLE MAYOR 12`) — {@link ES_SPACE_JOIN_FRACTION} of native rows collapse
+ *   the comma. NL: OA (and the eval, 3,000/3,000) glue the postcode (`1187LM`) while the national
+ *   convention spaces it (`1187 LM`) — {@link NL_GLUED_POSTCODE_FRACTION} of rows keep the glued
+ *   source shape, the rest the spaced conventional one. These draws are consumed ONLY for their
+ *   country, so DE/FR emit streams are unchanged for a given seed.
+ */
+import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country";
+import { dataRootPath } from "@mailwoman/core/data-root";
+import { openReadStream } from "@mailwoman/core/fs/streams";
+import { readZipEntry } from "@mailwoman/core/fs/zip";
+import { isPresent } from "@mailwoman/core/objects";
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils";
+import { CSVSpliterator } from "spliterator";
+import { stableSourceID } from "#adapters/utils";
+import { synthesizeLocaleRow } from "#synthesizers/locale";
+import { pick } from "#synthesizers/utils";
+import { alignRow } from "#utils";
+/**
+ * Per-country OA sources + the source name used in the corpus. DE/FR read their historical build inputs, the cached
+ * zips under `$MAILWOMAN_DATA_ROOT/oa-cache` — materialize them there to regenerate. ES/NL read the extracted
+ * countrywide CSVs and IT the cached national zip under `$MAILWOMAN_DATA_ROOT` (#241; the fresh ES extract is
+ * OA-conformed, so the old raw-CNIG conform map is gone). DE carries a per-part `region` fallback (its REGION column is
+ * empty; the international-order tail needs it, #327). FR/NL/IT/ES REGION is populated per-row (ES = comunidad
+ * autónoma, IT = regione, NL = province).
+ */
+const COUNTRY_SOURCES = {
+    DE: {
+        source: "synth-german",
+        corpusVersion: "0.4.0",
+        parts: [
+            { zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", region: "Berlin" },
+            { zip: dataRootPath("oa-cache", "de__sn__statewide.zip"), csv: "de/sn/statewide.csv", region: "Sachsen" },
+        ],
+    },
+    FR: {
+        source: "synth-fr",
+        corpusVersion: "0.4.0",
+        parts: [{ zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }],
+    },
+    NL: {
+        source: "synth-nl",
+        corpusVersion: "0.9.9",
+        parts: [{ path: dataRootPath("openaddresses", "extracted", "nl", "countrywide.csv") }],
+    },
+    IT: {
+        source: "synth-it",
+        corpusVersion: "0.9.9",
+        parts: [{ zip: dataRootPath("oa-cache", "it__countrywide.zip"), csv: "it/countrywide.csv" }],
+    },
+    ES: {
+        source: "synth-es",
+        corpusVersion: "0.9.9",
+        parts: [{ path: dataRootPath("openaddresses", "extracted", "es", "countrywide.csv") }],
+        // Pedanía slice (`synth-es-pedania`, `--district-as-locality`). Reads the RAW (un-conformed) CNIG export
+        // cached at oa-cache/es__countrywide.zip — the `parts` CSV above lost `poblacion` in OA's own conform step
+        // (see {@link LocalePart.cnigRaw}). districtAsLocality is pinned true here (this part only exists to be
+        // read pedanía-style); the CLI override still applies harmlessly on top.
+        pedaniaParts: [
+            {
+                zip: dataRootPath("oa-cache", "es__countrywide.zip"),
+                csv: "es_addresses.csv",
+                cnigRaw: true,
+                districtAsLocality: true,
+            },
+        ],
+    },
+    NZ: {
+        // LINZ-derived OA countrywide extract (2.12M rows). `districtAsLocality` inverts the CITY/DISTRICT
+        // mapping: DISTRICT holds the city (Auckland), CITY the suburb (Birkenhead). NZ OA carries no postcode.
+        source: "synth-nz",
+        corpusVersion: "0.9.9",
+        // Eval holdout: a probe build excludes ~12% of NZ localities (a locality-bucket split) for a source-disjoint
+        // coord board; that split is a BUILD-TIME concern (scratchpad), so the committed recipe reads the full CSV.
+        parts: [{ path: dataRootPath("openaddresses", "extracted", "nz", "countrywide.csv"), districtAsLocality: true }],
+    },
+    GB: {
+        // HM Land Registry Price Paid Data tuples (25.67M rows out of the PPD ingest). PPD's DISTRICT is the
+        // postal town (locality) and CITY is the dependent locality — legitimately EMPTY on the majority of rows
+        // (most GB addresses have no dependent locality). `districtAsLocality` maps DISTRICT→locality and, when
+        // present, CITY→dependent_locality; the `readTuples` check above only drops a row when BOTH are empty, so
+        // the majority empty-CITY rows survive.
+        source: "synth-gb",
+        corpusVersion: "0.9.9",
+        parts: [{ path: dataRootPath("ppd", "2026-07-22", "gb-tuples.csv"), districtAsLocality: true }],
+    },
+};
+/**
+ * Per-part reservoir cap. Streaming + Algorithm-R reservoir sampling to this size keeps memory bounded regardless of
+ * source size, where buffering the whole CSV OOMs / overflows on FR/ES countrywide (~2.5 GB, ~25M rows). DE/NL-scale
+ * sources (≤ ~1.2M) fit entirely, so they're sampled losslessly.
+ */
+const RESERVOIR_CAP = 1_200_000;
+/**
+ * Fraction of NATIVE-order ES rows whose street→house join is space-collapsed (`CALLE MAYOR 12`) instead of the
+ * template's comma (`CALLE MAYOR, 12`). Both are real Spanish surfaces — the comma is the official convention, the
+ * space is what OA-derived feeds (and all 3,000 `openaddresses-es-sample.jsonl` rows) carry. 0.5 teaches both.
+ */
+const ES_SPACE_JOIN_FRACTION = 0.5;
+/**
+ * Fraction of NL rows whose postcode keeps OA's glued shape (`1187LM`) instead of the spaced national convention (`1187
+ * LM`). The eval sample is 100% glued; the conventional spaced form is the `1012 LM` two-letter-suffix shape the model
+ * currently glues onto the city (#241). 0.5 teaches both.
+ */
+const NL_GLUED_POSTCODE_FRACTION = 0.5;
+/**
+ * OA CITY-noise normalization (#241) — the documented cleaning step, derived from the 2026-07-02 FULL-STREAM audit of
+ * the ES (15.6M rows), IT (13.9M), and NL (9.1M) sources (not a hand-list). Returns the cleaned city, or `null` to drop
+ * the tuple.
+ *
+ * Cleaned classes:
+ *
+ * 1. DROP pseudo-localities — the ES cadastral aggregates (`Comunidad de 09076, 09150 y 09578`, `Ledanía de …`; 0.06% of
+ *    ES rows): any CITY containing a comma or a ≥4-digit run is a land-register aggregate, not a renderable city.
+ *    Structural, locale-safe — NL's genuine `2e Valthermond` (one digit) survives; IT/NL have zero hits.
+ * 2. STRIP a trailing parenthesized 1–3-letter admin code — the NL BAG province disambiguator (`Bergen (NH)`, `Rijswijk
+ *    (GLD)` → `Bergen`, `Rijswijk`; 0.13% of NL rows). The analogue of the German Kreis/region-suffix class (#241 names
+ *    `Rabenau Sachs` / `Weißwasser /O.L.`): an admin-region gloss glued onto the locality value that dirties locality
+ *    labels.
+ *
+ * Audit-verified NON-noise, deliberately NOT cleaned (a naive suffix rule would mangle real names):
+ *
+ * - ES/IT city-ends-with-province (`Alhama de Almería`, `GENZANO DI ROMA`; ~0.8% each): genuine toponyms whose linking
+ *   `de`/`di` makes them full names, unlike the German glued-abbreviation class.
+ * - ES bilingual slash names (`Laudio/Llodio`; 2.16%): official co-names — the eval expects them verbatim.
+ * - IT ALL-CAPS city casing (98.79% of the source, and the eval's observed form): casing is the #829 case-augmentation
+ *   change, not this slice's.
+ */
+export function cleanCityNoise(city) {
+    if (/,|\d{4}/.test(city))
+        return null;
+    const stripped = city.replace(/\s*\(\p{L}{1,3}\)\s*$/u, "").trim();
+    return stripped || null;
+}
+/**
+ * Stream real tuples out of an OA source part and reservoir-sample to {@link RESERVOIR_CAP}. Reads the CSV row-by-row —
+ * `readZipEntry | CSVSpliterator` for zip parts, `createReadStream | CSVSpliterator` for extracted parts (both bounded
+ * memory) — and keeps a uniform random sample (Algorithm R) seeded by `rng`, separate from the emit loop's PRNG. NO
+ * global dedup (a 25M-key Set would OOM; OA rows are near-unique). The city passes through {@link cleanCityNoise}; the
+ * region falls back to `part.region` when the row's REGION cell is empty (DE).
+ *
+ * Exported for {@link locale.test.ts} — the CSV read path (quote handling, CRLF, region fallback) has no other test.
+ */
+export async function readTuples(part, rng) {
+    // No `encoding` on the file path — CSVSpliterator delimits raw bytes and decodes utf-8 itself; a string stream
+    // (from `{ encoding: "utf8" }`) would defeat its byte-range scanner.
+    const input = part.path
+        ? openReadStream(part.path)
+        : readZipEntry(part.zip, part.csv);
+    const get = (cells, i) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "");
+    const reservoir = [];
+    let cols = null;
+    let header = null;
+    let seen = 0;
+    let dropped = 0;
+    try {
+        // CSVSpliterator handles OA's quoted fields (embedded commas/newlines) and CRLF row terminators
+        // (spliterator ≥ 3.2.0); `header: false` yields the header row too, so the column index is built
+        // from that first row rather than from a header option.
+        for await (const cells of CSVSpliterator.fromAsync(input, {
+            header: false,
+        })) {
+            if (header === null) {
+                header = cells.map((h) => h.trim().toLowerCase());
+                // oxlint-disable-next-line no-loop-func -- the binding is per-iteration (for-of/for-await) and the batch is awaited before the next
+                const ix = (name) => header.indexOf(name);
+                // `cnigRaw` (ES pedanía only): the RAW CNIG header has no NUMBER/STREET/CITY/DISTRICT/REGION/POSTCODE
+                // at all — `numero`/`nombre_via`/`poblacion`/`municipio`/`comunidad_autonoma`/`cod_postal` instead.
+                // See {@link LocalePart.cnigRaw}.
+                cols = part.cnigRaw
+                    ? {
+                        num: ix("numero"),
+                        street: ix("nombre_via"),
+                        tipoVial: ix("tipo_vial"),
+                        city: ix("poblacion"),
+                        district: ix("municipio"),
+                        region: ix("comunidad_autonoma"),
+                        post: ix("cod_postal"),
+                    }
+                    : {
+                        num: ix("number"),
+                        street: ix("street"),
+                        tipoVial: -1,
+                        city: ix("city"),
+                        district: ix("district"),
+                        region: ix("region"),
+                        post: ix("postcode"),
+                    };
+                continue;
+            }
+            if (cols === null)
+                continue;
+            // cnigRaw: STREET is split into a road-type column (`tipo_vial`, e.g. "CARRETERA") and the name
+            // (`nombre_via`) — rejoin them exactly as OA's own conform step does (verified byte-identical to the
+            // conformed STREET column for the same source row). Every other part's header already carries the
+            // pre-joined STREET column, so `cols.tipoVial === -1` and this is a no-op there.
+            const street = cols.tipoVial >= 0
+                ? [get(cells, cols.tipoVial), get(cells, cols.street)].filter(isPresent).join(" ")
+                : get(cells, cols.street);
+            const rawCity = get(cells, cols.city);
+            if (!street)
+                continue;
+            // Default: CITY → locality. NZ/GB (`districtAsLocality`) inverts it — the OA DISTRICT holds the city
+            // (`Auckland`) and CITY holds the suburb (`Birkenhead`), so DISTRICT → locality and CITY →
+            // dependent_locality. When DISTRICT is empty (~18% of NZ rows), fall back to CITY → locality with no
+            // sub-locality. GB PPD tuples flip which side is legitimately empty — on the MAJORITY of GB rows CITY
+            // (the dependent_locality) is empty and DISTRICT (the locality) is populated, so the check below only
+            // drops a `districtAsLocality` row when BOTH are empty, never when CITY alone is — filtering on CITY
+            // alone silently discards most of the GB source. See {@link LocalePart.districtAsLocality}.
+            let locality;
+            let dependent_locality;
+            if (part.districtAsLocality) {
+                const rawDistrict = get(cells, cols.district);
+                if (!rawCity && !rawDistrict)
+                    continue;
+                const cleanedDistrict = cleanCityNoise(rawDistrict);
+                if (cleanedDistrict) {
+                    locality = cleanedDistrict;
+                    const cleanedCity = cleanCityNoise(rawCity);
+                    // ES pedanía lesson (2026-07-22): the CNIG `poblacion` column is filled on ~93% of rows but
+                    // EQUALS `municipio` on the majority of those (the address point sits in the municipio's own
+                    // main town, not a below-municipio pedanía) — only ~32.6% of ES rows carry a genuinely
+                    // DISTINCT poblacion. GB/NZ never hit this (CITY/DISTRICT name the same place only by rare
+                    // coincidence), but the guard is general: a dependent_locality equal to its own locality is
+                    // never a real sub-locality, so drop it rather than emit a same-value pair (would fail the
+                    // dep_loc≠locality invariant every recipe otherwise upholds).
+                    dependent_locality =
+                        cleanedCity && cleanedCity.localeCompare(locality, undefined, { sensitivity: "base" }) !== 0
+                            ? cleanedCity
+                            : undefined;
+                }
+                else {
+                    locality = cleanCityNoise(rawCity);
+                }
+            }
+            else {
+                if (!rawCity)
+                    continue;
+                locality = cleanCityNoise(rawCity);
+            }
+            if (!locality) {
+                dropped++;
+                continue;
+            }
+            const tuple = {
+                house_number: get(cells, cols.num),
+                street,
+                locality,
+                region: get(cells, cols.region) || part.region || "",
+                postcode: get(cells, cols.post),
+                ...(dependent_locality ? { dependent_locality } : {}),
+            };
+            seen++;
+            if (reservoir.length < RESERVOIR_CAP) {
+                reservoir.push(tuple);
+            }
+            else {
+                const j = Math.floor(rng() * seen);
+                // 0 .. seen-1
+                if (j < RESERVOIR_CAP) {
+                    reservoir[j] = tuple;
+                }
+            }
+        }
+    }
+    catch (error) {
+        console.error(`  WARN: read failed for ${part.path ?? part.zip}: ${error.message}`);
+        return [];
+    }
+    console.error(`  ${part.path ?? part.csv}: ${reservoir.length} sampled of ${seen} rows (${dropped} city-noise drops)`);
+    return reservoir;
+}
+/**
+ * Country-append fraction (the fr-admin-split #728 pattern, generalized to the locale recipe): mutates `synth` in
+ * place, `countryFraction` of the time appending an explicit country surface form ("United Kingdom") to `raw` + a
+ * `country` component — the model relearns to emit country WHEN the token is present without over-firing it on the
+ * (still-majority) country-less rows. `countryFraction <= 0` (the default) short-circuits the `random()` draw away
+ * entirely — no `synth` mutation and no RNG consumption — so every existing locale's emit stream stays byte-identical
+ * to before this option existed. Exported for {@link locale.test.ts}.
+ */
+export function applyCountryAppend(synth, country, countryFraction, random) {
+    if (countryFraction > 0 && random() < countryFraction) {
+        const forms = COUNTRY_SURFACE_FORMS[country];
+        if (!forms?.length) {
+            // The BR/NZ lesson: a missing table entry must never silently no-op a requested fraction —
+            // it must raise so the gap is caught at build time, not discovered later as a 0% check failure.
+            throw new Error(`No COUNTRY_SURFACE_FORMS entry for ${country} — add it to codex/country/country.ts before using --country-fraction`);
+        }
+        const form = pick(forms, random);
+        synth.raw = `${synth.raw}, ${form}`;
+        synth.components = { ...synth.components, country: form };
+    }
+}
+/**
+ * Merge the `--district-as-locality` CLI override onto one part. `undefined` (flag absent) returns `part` unchanged
+ * (same object — no allocation, no behavior change); `true`/`false` returns a shallow copy with `districtAsLocality`
+ * forced to that value for this invocation only. Exported for {@link locale.test.ts}.
+ */
+export function applyDistrictAsLocalityOverride(part, override) {
+    return override === undefined ? part : { ...part, districtAsLocality: override };
+}
+/**
+ * Pick which part list a `--country` run reads: {@link LocaleCountrySource.pedaniaParts} when the override is explicitly
+ * `true` AND the country registers one (ES only, so far), else the default `parts` — unchanged for every other
+ * country/override combination. Exported for {@link locale.test.ts}.
+ */
+export function resolveLocaleParts(countrySource, override) {
+    return override === true && countrySource.pedaniaParts ? countrySource.pedaniaParts : countrySource.parts;
+}
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const localeRecipe = {
+    name: "locale",
+    description: "Per-locale coverage rows (DE/FR/NL/IT/ES/NZ/GB) from real OA tuples, both orders → synthesizeLocaleRow",
+    mode: "generate",
+    options: [
+        { flag: "--country <cc>", description: "Target country (DE|FR|NL|IT|ES|NZ|GB). Default DE" },
+        { flag: "--intl-fraction <f>", description: "Fraction rendered international order. Default 0.4" },
+        {
+            flag: "--country-fraction <f>",
+            description: "Fraction of rows that append an explicit country surface form (`, United Kingdom`) + a `country` component (fr-admin-split pattern). Default 0 — byte-identical to before when unset.",
+        },
+        {
+            flag: "--district-as-locality / --no-district-as-locality",
+            description: "Override the per-part districtAsLocality mapping for this run. Unset (default) leaves each COUNTRY_SOURCES part's own value untouched — every existing build stays byte-identical. ES additionally switches to the pedanía (poblacion→dependent_locality) source when passed as true — combine with --source-name synth-es-pedania.",
+        },
+    ],
+    async run(opts, write) {
+        // Emit PRNG: the legacy build-locale-slice.mjs seeded mulberry32(opts.seed). The reservoir uses a
+        // SEPARATE per-part mulberry32 (below) so input sampling never perturbs this emit stream.
+        const random = makeMulberry32(opts.seed);
+        const country = (opts.country ?? "DE").toUpperCase();
+        const countrySource = COUNTRY_SOURCES[country];
+        if (!countrySource) {
+            throw new Error(`No OA sources registered for --country ${country}. Known: ${Object.keys(COUNTRY_SOURCES).join(", ")}.`);
+        }
+        const intlFraction = opts.intlFraction ?? 0.4;
+        if (!(intlFraction >= 0 && intlFraction <= 1)) {
+            throw new Error(`--intl-fraction must be in [0, 1], got ${intlFraction}`);
+        }
+        // Default 0 → the `random() < countryFraction` draw below is short-circuited away entirely (never
+        // consumed), so every existing locale's emit stream is byte-identical to before this option existed.
+        const countryFraction = opts.countryFraction ?? 0;
+        if (!(countryFraction >= 0 && countryFraction <= 1)) {
+            throw new Error(`--country-fraction must be in [0, 1], got ${countryFraction}`);
+        }
+        const source = opts.sourceName ?? countrySource.source;
+        const count = opts.count ?? 4000;
+        // Tri-state: `undefined` (flag absent) touches nothing below — `parts` stays the default list and each
+        // part keeps its own `districtAsLocality`, so every existing locale build is byte-identical to before this
+        // option existed. `true` additionally selects `pedaniaParts` when the country registers one (ES); `false`
+        // forces the mapping off on every part read this run (a debugging override for GB/NZ).
+        const districtAsLocalityOverride = opts.districtAsLocality;
+        const parts = resolveLocaleParts(countrySource, districtAsLocalityOverride);
+        const pool = [];
+        for (let pi = 0; pi < parts.length; pi++) {
+            // A reservoir PRNG per part, seeded but independent of the emit loop's `random`, so the sample is
+            // reproducible without perturbing the synth/order draws.
+            const reservoirRng = makeMulberry32((opts.seed ^ (0x9e_37_79_b9 * (pi + 1))) >>> 0);
+            const effectivePart = applyDistrictAsLocalityOverride(parts[pi], districtAsLocalityOverride);
+            const t = await readTuples(effectivePart, reservoirRng);
+            for (const x of t) {
+                pool.push(x);
+            } // NOT pool.push(...t) — spreading huge arrays overflows the stack
+        }
+        if (!pool.length) {
+            throw new Error(`No ${country} tuples found — are the source CSVs/zips present? (see COUNTRY_SOURCES)`);
+        }
+        let emitted = 0;
+        let skipped = 0;
+        let guard = 0;
+        const N = pool.length;
+        while (emitted < count && guard++ < count * 6) {
+            const base = pool[Math.floor(random() * N)];
+            const order = random() < intlFraction ? "international" : "native";
+            // Per-country surface-shape draws (#241) — consumed ONLY for that country, so the DE/FR emit
+            // streams for a given seed are unchanged by their existence.
+            const nativeHouseJoin = country === "ES" ? (random() < ES_SPACE_JOIN_FRACTION ? "space" : "template") : undefined;
+            const postcodeShape = country === "NL"
+                ? random() < NL_GLUED_POSTCODE_FRACTION
+                    ? "as-source"
+                    : "conventional"
+                : undefined;
+            const synth = synthesizeLocaleRow(base, country, { random, order, nativeHouseJoin, postcodeShape });
+            if (!synth) {
+                skipped++;
+                continue;
+            }
+            applyCountryAppend(synth, country, countryFraction, random);
+            if (opts.golden) {
+                // Golden rows must round-trip through alignRow exactly like training rows (#241 done-when): a
+                // render that can't be BIO-labeled can't serve as a parser golden either. Consumes no RNG draw.
+                const goldenCanonical = {
+                    raw: synth.raw,
+                    components: synth.components,
+                    country,
+                    locale: synth.locale,
+                    source,
+                    source_id: "golden:align-check",
+                };
+                const goldenAligned = alignRow(goldenCanonical);
+                if (goldenAligned.kind !== "labeled" || !goldenAligned.row) {
+                    skipped++;
+                    continue;
+                }
+                write(JSON.stringify({ raw: synth.raw, components: synth.components, country, order }) + "\n");
+                emitted++;
+                continue;
+            }
+            const sourceID = stableSourceID(source, {
+                street: synth.components.street,
+                house_number: synth.components.house_number,
+                locality: synth.components.locality,
+                postcode: synth.components.postcode,
+            });
+            const canonical = {
+                raw: synth.raw,
+                components: synth.components,
+                country,
+                locale: synth.locale,
+                source,
+                source_id: sourceID,
+                corpus_version: countrySource.corpusVersion,
+                license: `OpenAddresses ${country} tuples, rendered ${order}-order — see ingest SOURCES`,
+            };
+            const aligned = alignRow(canonical);
+            if (aligned.kind !== "labeled" || !aligned.row) {
+                skipped++;
+                continue;
+            }
+            write(JSON.stringify({ ...aligned.row, synth_method: source, synth_order: order, synth_base_id: null }) + "\n");
+            emitted++;
+        }
+        return { emitted, skipped };
+    },
+};
+//# sourceMappingURL=locale.js.map

package/out/international/recipes/locale.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"locale.js","sourceRoot":"","sources":["../../../lib/international/recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AACnD,OAAO,EAAE,UAAU,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAEpE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAEhD,OAAO,EAAmD,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC3G,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAA;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAkDjC;;;;;;;GAOG;AACH,MAAM,eAAe,GAAwC;IAC5D,EAAE,EAAE;QACH,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE;YACN,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,gBAAgB,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE;YAC3F,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,uBAAuB,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE;SACzG;KACD;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,qBAAqB,CAAC,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAC;KAC5F;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,CAAC,EAAE,CAAC;KACtF;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,qBAAqB,CAAC,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAC;KAC5F;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,CAAC,EAAE,CAAC;QACtF,yGAAyG;QACzG,2GAA2G;QAC3G,wGAAwG;QACxG,yEAAyE;QACzE,YAAY,EAAE;YACb;gBACC,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,qBAAqB,CAAC;gBACpD,GAAG,EAAE,kBAAkB;gBACvB,OAAO,EAAE,IAAI;gBACb,kBAAkB,EAAE,IAAI;aACxB;SACD;KACD;IACD,EAAE,EAAE;QACH,mGAAmG;QACnG,wGAAwG;QACxG,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,6GAA6G;QAC7G,4GAA4G;QAC5G,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;KAChH;IACD,EAAE,EAAE;QACH,qGAAqG;QACrG,yGAAyG;QACzG,wGAAwG;QACxG,0GAA0G;QAC1G,wCAAwC;QACxC,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,eAAe,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;KAC/F;CACD,CAAA;AAED;;;;GAIG;AACH,MAAM,aAAa,GAAG,SAAS,CAAA;AAE/B;;;;GAIG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;;;GAIG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAEtC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAElE,OAAO,QAAQ,IAAI,IAAI,CAAA;AACxB,CAAC;AAgBD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAgB,EAAE,GAAiB;IACnE,+GAA+G;IAC/G,qEAAqE;IACrE,MAAM,KAAK,GAAsD,IAAI,CAAC,IAAI;QACzE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;QAC3B,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,GAAI,EAAE,IAAI,CAAC,GAAI,CAAC,CAAA;IAErC,MAAM,GAAG,GAAG,CAAC,KAAe,EAAE,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC/G,MAAM,SAAS,GAAsB,EAAE,CAAA;IACvC,IAAI,IAAI,GAAuB,IAAI,CAAA;IACnC,IAAI,MAAM,GAAoB,IAAI,CAAA;IAClC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,CAAC;QACJ,gGAAgG;QAChG,iGAAiG;QACjG,wDAAwD;QACxD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,cAAc,CAAC,SAAS,CAAW,KAAK,EAAE;YACnE,MAAM,EAAE,KAAK;SACb,CAAC,EAAE,CAAC;YACJ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACrB,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;gBACjD,oIAAoI;gBACpI,MAAM,EAAE,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,MAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;gBAE1D,sGAAsG;gBACtG,oGAAoG;gBACpG,kCAAkC;gBAClC,IAAI,GAAG,IAAI,CAAC,OAAO;oBAClB,CAAC,CAAC;wBACA,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC;wBACjB,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC;wBACxB,QAAQ,EAAE,EAAE,CAAC,WAAW,CAAC;wBACzB,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC;wBACrB,QAAQ,EAAE,EAAE,CAAC,WAAW,CAAC;wBACzB,MAAM,EAAE,EAAE,CAAC,oBAAoB,CAAC;wBAChC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC;qBACtB;oBACF,CAAC,CAAC;wBACA,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC;wBACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC;wBACpB,QAAQ,EAAE,CAAC,CAAC;wBACZ,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC;wBAChB,QAAQ,EAAE,EAAE,CAAC,UAAU,CAAC;wBACxB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC;wBACpB,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC;qBACpB,CAAA;gBAEH,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,KAAK,IAAI;gBAAE,SAAQ;YAE3B,gGAAgG;YAChG,qGAAqG;YACrG,kGAAkG;YAClG,iFAAiF;YACjF,MAAM,MAAM,GACX,IAAI,CAAC,QAAQ,IAAI,CAAC;gBACjB,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBAClF,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;YAE3B,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YAErC,IAAI,CAAC,MAAM;gBAAE,SAAQ;YAErB,qGAAqG;YACrG,2FAA2F;YAC3F,qGAAqG;YACrG,sGAAsG;YACtG,sGAAsG;YACtG,qGAAqG;YACrG,4FAA4F;YAC5F,IAAI,QAAuB,CAAA;YAC3B,IAAI,kBAAsC,CAAA;YAE1C,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC7B,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;gBAE7C,IAAI,CAAC,OAAO,IAAI,CAAC,WAAW;oBAAE,SAAQ;gBAEtC,MAAM,eAAe,GAAG,cAAc,CAAC,WAAW,CAAC,CAAA;gBAEnD,IAAI,eAAe,EAAE,CAAC;oBACrB,QAAQ,GAAG,eAAe,CAAA;oBAC1B,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;oBAE3C,4FAA4F;oBAC5F,6FAA6F;oBAC7F,uFAAuF;oBACvF,2FAA2F;oBAC3F,4FAA4F;oBAC5F,2FAA2F;oBAC3F,8DAA8D;oBAC9D,kBAAkB;wBACjB,WAAW,IAAI,WAAW,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC;4BAC3F,CAAC,CAAC,WAAW;4BACb,CAAC,CAAC,SAAS,CAAA;gBACd,CAAC;qBAAM,CAAC;oBACP,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;gBACnC,CAAC;YACF,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,OAAO;oBAAE,SAAQ;gBACtB,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;YACnC,CAAC;YAED,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,KAAK,GAAoB;gBAC9B,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;gBAClC,MAAM;gBACN,QAAQ;gBACR,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE;gBACpD,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC/B,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrD,CAAA;YAED,IAAI,EAAE,CAAA;YAEN,IAAI,SAAS,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;gBACtC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAA;gBAElC,cAAc;gBACd,IAAI,CAAC,GAAG,aAAa,EAAE,CAAC;oBACvB,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAA;gBACrB,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;QAE9F,OAAO,EAAE,CAAA;IACV,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,MAAM,eAAe,IAAI,UAAU,OAAO,oBAAoB,CAAC,CAAA;IAEtH,OAAO,SAAS,CAAA;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CACjC,KAA2B,EAC3B,OAAe,EACf,eAAuB,EACvB,MAAoB;IAEpB,IAAI,eAAe,GAAG,CAAC,IAAI,MAAM,EAAE,GAAG,eAAe,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,qBAAqB,CAAC,OAA6C,CAAC,CAAA;QAElF,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;YACpB,2FAA2F;YAC3F,gGAAgG;YAChG,MAAM,IAAI,KAAK,CACd,sCAAsC,OAAO,uEAAuE,CACpH,CAAA;QACF,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QAChC,KAAK,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,GAAG,KAAK,IAAI,EAAE,CAAA;QACnC,KAAK,CAAC,UAAU,GAAG,EAAE,GAAG,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAC1D,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,+BAA+B,CAAC,IAAgB,EAAE,QAA6B;IAC9F,OAAO,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAA;AACjF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,aAAkC,EAAE,QAA6B;IACnG,OAAO,QAAQ,KAAK,IAAI,IAAI,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAA;AAC1G,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAiB;IACzC,IAAI,EAAE,QAAQ;IACd,WAAW,EAAE,wGAAwG;IACrH,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACR,EAAE,IAAI,EAAE,gBAAgB,EAAE,WAAW,EAAE,mDAAmD,EAAE;QAC5F,EAAE,IAAI,EAAE,qBAAqB,EAAE,WAAW,EAAE,oDAAoD,EAAE;QAClG;YACC,IAAI,EAAE,wBAAwB;YAC9B,WAAW,EACV,uLAAuL;SACxL;QACD;YACC,IAAI,EAAE,oDAAoD;YAC1D,WAAW,EACV,qUAAqU;SACtU;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,kGAAkG;QAClG,0FAA0F;QAC1F,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,WAAW,EAAE,CAAA;QACpD,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QAE9C,IAAI,CAAC,aAAa,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACd,0CAA0C,OAAO,YAAY,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACvG,CAAA;QACF,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,GAAG,CAAA;QAE7C,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,IAAI,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAA;QAC1E,CAAC;QAED,kGAAkG;QAClG,qGAAqG;QACrG,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;QAEjD,IAAI,CAAC,CAAC,eAAe,IAAI,CAAC,IAAI,eAAe,IAAI,CAAC,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,6CAA6C,eAAe,EAAE,CAAC,CAAA;QAChF,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,aAAa,CAAC,MAAM,CAAA;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAA;QAChC,uGAAuG;QACvG,2GAA2G;QAC3G,0GAA0G;QAC1G,uFAAuF;QACvF,MAAM,0BAA0B,GAAG,IAAI,CAAC,kBAAkB,CAAA;QAC1D,MAAM,KAAK,GAAG,kBAAkB,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAA;QAE3E,MAAM,IAAI,GAAsB,EAAE,CAAA;QAElC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YAC1C,kGAAkG;YAClG,yDAAyD;YACzD,MAAM,YAAY,GAAG,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,aAAa,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;YACnF,MAAM,aAAa,GAAG,+BAA+B,CAAC,KAAK,CAAC,EAAE,CAAE,EAAE,0BAA0B,CAAC,CAAA;YAC7F,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,aAAa,EAAE,YAAY,CAAC,CAAA;YAEvD,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACb,CAAC,CAAC,kEAAkE;QACrE,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,MAAM,OAAO,yEAAyE,CAAC,CAAA;QACxG,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAA;QAErB,OAAO,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAE,CAAA;YAC5C,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAA;YAElE,6FAA6F;YAC7F,6DAA6D;YAC7D,MAAM,eAAe,GACpB,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,sBAAsB,CAAC,CAAC,CAAE,OAAiB,CAAC,CAAC,CAAE,UAAoB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;YAEhH,MAAM,aAAa,GAClB,OAAO,KAAK,IAAI;gBACf,CAAC,CAAC,MAAM,EAAE,GAAG,0BAA0B;oBACtC,CAAC,CAAE,WAAqB;oBACxB,CAAC,CAAE,cAAwB;gBAC5B,CAAC,CAAC,SAAS,CAAA;YAEb,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC,CAAA;YAEnG,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,CAAA;YAE3D,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,8FAA8F;gBAC9F,gGAAgG;gBAChG,MAAM,eAAe,GAAG;oBACvB,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO;oBACP,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM;oBACN,SAAS,EAAE,oBAAoB;iBAC/B,CAAA;gBAED,MAAM,aAAa,GAAG,QAAQ,CAAC,eAAiD,CAAC,CAAA;gBAEjF,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC;oBAC5D,OAAO,EAAE,CAAA;oBAET,SAAQ;gBACT,CAAC;gBAED,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAE9F,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE;gBACvC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM;gBAC/B,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,YAAY;gBAC3C,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;gBACnC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;aACnC,CAAC,CAAA;YAEF,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO;gBACP,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,aAAa,CAAC,aAAa;gBAC3C,OAAO,EAAE,iBAAiB,OAAO,qBAAqB,KAAK,6BAA6B;aACxF,CAAA;YAED,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;YAE/G,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}

package/out/jp/adapters/wof-admin/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../lib/jp/adapters/wof-admin/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAOH,OAAO,KAAK,EAAkB,YAAY,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAOzE;;;GAGG;AACH,eAAO,MAAM,uBAAuB,iBAAiB,CAAA;AAErD,UAAU,QAAQ;IACjB,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CACf;AAoCD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAClC,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC3B;IACF,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;CACtC,GAAG,IAAI,CAyBP;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,IAAI,aAAa,CAyEvD;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,eAA4B,CAAA"}

package/out/jp/adapters/wof-admin/adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../lib/jp/adapters/wof-admin/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAIzD;;GAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAE5B;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,cAAc,CAAA;AAUrD;;;;;;;GAOG;AACH,SAAS,OAAO,CACf,IAA2B,EAC3B,OAA6C,EAC7C,OAAe;IAEf,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,IAAI,EAAE,GAAG,OAAO,CAAA;IAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,kBAAkB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,EAAE,CAAC,CAAA;QAEvC,IAAI,CAAC,GAAG;YAAE,MAAK;QACf,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACb,EAAE,GAAG,GAAG,CAAC,SAAS,CAAA;IACnB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CAAC,GAAa,EAAE,QAA6B;IAC7D,OAAO,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,CAAA;AACxC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAClC,KAAiB,EACjB,QAA6B;IAK7B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAA;IAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAA;IAE1F,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,eAAe,CAAC,CAAA;IAExE,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;IAC7C,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IACjD,MAAM,iBAAiB,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAElF,MAAM,UAAU,GAA+B;QAC9C,MAAM,EAAE,UAAU;QAClB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,IAAI;KACb,CAAA;IAED,IAAI,iBAAiB,EAAE,CAAC;QACvB,UAAU,CAAC,kBAAkB,GAAG,iBAAiB,CAAA;IAClD,CAAC;IAED,MAAM,GAAG,GAAG,CAAC,UAAU,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEpF,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB;IACtC,OAAO;QACN,EAAE,EAAE,uBAAuB;QAC3B,cAAc,EAAE,WAAW;QAC3B,WAAW,EAAE,kFAAkF;QAE/F,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,wDAAwD,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;YACxF,CAAC;YAED,MAAM,EAAE,GAAG,IAAI,cAAc,CAAc,IAAI,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;YAE9E,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC,mDAAmD,CAAC,CAAA;YACpF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;YAE1C,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,GAAG,EAAoC,EAAE,CAAC;gBACxE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAC3B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAA;gBAC/B,CAAC;YACF,CAAC;YAED,4FAA4F;YAC5F,4DAA4D;YAC5D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAoB,CAAA;YAExC,KAAK,MAAM,GAAG,IAAI,OAAO,CACxB,EAAE,CAAC,OAAO,CAAC,4EAA4E,CAAC,CACxF,EAAE,CAAC;gBACH,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;YACtB,CAAC;YAED,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,sEAAsE,CAAC,CAAA;YAEtG,MAAM,cAAc,GAAG,CAAC,EAAU,EAAwB,EAAE;gBAC3D,MAAM,GAAG,GAAG,MAAM,CAAW,WAAW,EAAE,EAAE,CAAC,CAAA;gBAE7C,IAAI,GAAG,EAAE,CAAC;oBACT,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;gBAClB,CAAC;gBAED,OAAO,GAAG,CAAA;YACX,CAAC,CAAA;YAED,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,eAAe,CAAC,CAAA;YAEnF,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAK;gBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,MAAK;gBAE5D,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,CAAA;gBACpD,MAAM,KAAK,GAAG,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;gBAElD,IAAI,CAAC,KAAK;oBAAE,SAAQ;gBAEpB,MAAM;oBACL,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,uBAAuB;oBAC/B,SAAS,EAAE,GAAG,uBAAuB,IAAI,IAAI,CAAC,EAAE,EAAE;oBAClD,cAAc,EAAE,EAAE;oBAClB,OAAO,EAAE,WAAW;iBACpB,CAAA;gBAED,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,uBAAuB,EAAE,CAAA"}

package/out/jp/tools/fetch/houjin.d.ts

@@ -0,0 +1,24 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the National Tax Agency's corporate-number register (法人番号公表サイト 全件データ): every
+ *   corporation assigned a number, about 5,000,000 rows, with the head-office address as one string
+ *   (国内所在地) plus the prefecture and municipality split out and the postcode. The nationwide CSV
+ *   in Unicode is one zip of about 255 MB. This is the NOISY source for Japanese addresses; the
+ *   Overture rows the JP corpus is built from are the LABEL half.
+ *
+ *   The agency states the three published fields may be used freely by anyone ("どなたでも自由にご利用
+ *   いただくことができます"), with no attribution condition on the download page.
+ *
+ *   The file sits behind a form: the page hands out a per-session token, and a POST with that token,
+ *   `event=download` and the file number of the nationwide Unicode CSV answers the zip. The file
+ *   number is read off the page rather than pinned, because the agency re-issues the files monthly.
+ *
+ *   Invoke via `mailwoman corpus fetch houjin-jp --out-root <path>`.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index";
+export type FetchHoujinJPOptions = BaseFetchOptions;
+export declare function fetchHoujinJP(options: FetchHoujinJPOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=houjin.d.ts.map

package/out/jp/tools/fetch/houjin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"houjin.d.ts","sourceRoot":"","sources":["../../../../lib/jp/tools/fetch/houjin.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAMH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAA4B,MAAM,6BAA6B,CAAA;AAiB3G,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,CAAA;AAenD,wBAAsB,aAAa,CAClC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAgFvB"}

package/out/jp/tools/fetch/houjin.js

@@ -0,0 +1,102 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the National Tax Agency's corporate-number register (法人番号公表サイト 全件データ): every
+ *   corporation assigned a number, about 5,000,000 rows, with the head-office address as one string
+ *   (国内所在地) plus the prefecture and municipality split out and the postcode. The nationwide CSV
+ *   in Unicode is one zip of about 255 MB. This is the NOISY source for Japanese addresses; the
+ *   Overture rows the JP corpus is built from are the LABEL half.
+ *
+ *   The agency states the three published fields may be used freely by anyone ("どなたでも自由にご利用
+ *   いただくことができます"), with no attribution condition on the download page.
+ *
+ *   The file sits behind a form: the page hands out a per-session token, and a POST with that token,
+ *   `event=download` and the file number of the nationwide Unicode CSV answers the zip. The file
+ *   number is read off the page rather than pinned, because the agency re-issues the files monthly.
+ *
+ *   Invoke via `mailwoman corpus fetch houjin-jp --out-root <path>`.
+ */
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { attachmentFilename, cookieHeader, streamBodyToFile, withRetries, writeManifest, } from "#tools/fetch/download/index";
+const SLUG = "houjin-jp";
+const PAGE_URL = "https://www.houjin-bangou.nta.go.jp/download/zenken/";
+const FORM_URL = `${PAGE_URL}index.html`;
+const TOKEN_FIELD = "jp.go.nta.houjin_bangou.framework.web.common.CNSFWTokenProcessor.request.token";
+const LICENSE = "国税庁法人番号公表サイト — the three published fields are free for anyone to use (利用規約)";
+const ATTRIBUTION = "国税庁 法人番号公表サイト (National Tax Agency, Corporate Number Publication Site)";
+const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch";
+/**
+ * The nationwide row of the "CSV形式・Unicode" table carries the file number in its `doDownload(N)` handler.
+ */
+function nationwideUnicodeFileNumber(html) {
+    const start = html.indexOf('id="csv-unicode"');
+    if (start === -1)
+        return undefined;
+    const section = html.slice(start, html.indexOf('id="xml-unicode"', start));
+    const row = /全国[\s\S]{0,600}?doDownload\((\d+)\)/.exec(section);
+    return row?.[1];
+}
+export async function fetchHoujinJP(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const page = await fetch(PAGE_URL, { headers: { "user-agent": USER_AGENT, accept: "text/html" } });
+    if (!page.ok)
+        throw new Error(`houjin-jp: the download page answered HTTP ${page.status}`);
+    const html = await page.text();
+    const token = new RegExp(`${TOKEN_FIELD.replaceAll(".", "\\.")}" value="([^"]+)"`).exec(html)?.[1];
+    const fileNumber = nationwideUnicodeFileNumber(html);
+    if (!token || !fileNumber) {
+        report?.(`  ✗ the page carried ${token ? "a token" : "no token"} and ${fileNumber ? `file ${fileNumber}` : "no nationwide Unicode CSV row"}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: ["zenkoku-csv-unicode"] };
+    }
+    report?.(`=== ${SLUG}: nationwide CSV (Unicode), file ${fileNumber}`);
+    let filename = `zenkoku_all_${fileNumber}.zip`;
+    let bytes;
+    try {
+        bytes = await withRetries(async () => {
+            // A fresh page per attempt: the token is bound to the session cookie, and both may have lapsed.
+            const attemptPage = await fetch(PAGE_URL, { headers: { "user-agent": USER_AGENT, accept: "text/html" } });
+            const attemptToken = new RegExp(`${TOKEN_FIELD.replaceAll(".", "\\.")}" value="([^"]+)"`).exec(await attemptPage.text())?.[1];
+            if (!attemptToken)
+                throw new Error("the download page carried no token");
+            const res = await fetch(FORM_URL, {
+                method: "POST",
+                headers: {
+                    "user-agent": USER_AGENT,
+                    cookie: cookieHeader(attemptPage),
+                    referer: PAGE_URL,
+                    "content-type": "application/x-www-form-urlencoded",
+                },
+                body: new URLSearchParams({ [TOKEN_FIELD]: attemptToken, event: "download", selDlFileNo: fileNumber }),
+                signal: AbortSignal.timeout(1_800_000),
+            });
+            if (!res.ok || !(res.headers.get("content-type") ?? "").includes("octet-stream")) {
+                throw new Error(`HTTP ${res.status} ${res.headers.get("content-type") ?? ""}`);
+            }
+            filename = attachmentFilename(res, filename);
+            return streamBodyToFile(res, join(destDir, filename));
+        }, { report, label: filename });
+    }
+    catch (error) {
+        report?.(`  ✗ ${error.message}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: ["zenkoku-csv-unicode"] };
+    }
+    const dest = join(destDir, filename);
+    const sha = await sha256File(dest);
+    report?.(`  ✓ ${filename} ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
+    const manifest = {
+        source: SLUG,
+        source_url: PAGE_URL,
+        license: LICENSE,
+        attribution: ATTRIBUTION,
+        downloaded_at: new Date().toISOString(),
+        files: [{ source_url: FORM_URL, downloaded_at: new Date().toISOString(), filename, sha256: sha, bytes }],
+    };
+    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=houjin.js.map