@mailwoman/corpus 9.2.0 → 9.4.0

package/out/tools/fetch/imls-pls.js

@@ -0,0 +1,108 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the IMLS Public Libraries Survey (PLS) outlet-level data. Each US public library branch
+ *   (outlet) is one row, ~17K rows with address fields. Source for the `usgov-imls-pls` adapter. US
+ *   Public Domain (federal statistical survey).
+ *
+ *   The FY 2023 release is the most current as of 2026-05. IMLS ships a single ZIP containing CSV,
+ *   SAS, and SPSS variants. We extract the outlet-level CSV (pls_fy*_outlet*.csv or similar) and
+ *   discard the rest. The administrative-entity (system-level) CSV is intentionally skipped — it has
+ *   no per-branch address detail.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum. The
+ *   ZIP is unpacked with the `unzip` binary via `node:child_process` (no clean Node equivalent for
+ *   member listing + selective extraction).
+ *
+ *   Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
+ *   and sha matches MANIFEST, skips download.
+ */
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { statPath, pathExists } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip";
+import { sha256File } from "@mailwoman/core/hash";
+import { basename, join } from "path-ts";
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download";
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+/**
+ * The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
+ */
+const ZIP_URL = "https://www.imls.gov/sites/default/files/2025-08/pls_fy2023_csv.zip";
+const SLUG = "usgov-imls-pls";
+export async function fetchIMLSPLS(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const zipDest = join(destDir, basename(ZIP_URL));
+    const manifestPath = join(destDir, "MANIFEST.json");
+    report?.(`=== ${SLUG}`);
+    // MARK: Idempotency check
+    const recorded = await readManifest(manifestPath);
+    if (recorded?.sha256 && recorded.filename) {
+        const recordedPath = join(destDir, recorded.filename);
+        if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
+            report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.");
+            return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
+        }
+    }
+    // MARK: Download ZIP
+    report?.(`  Downloading ${ZIP_URL} ...`);
+    const { bytes: zipSize } = await downloadToFile({
+        url: ZIP_URL,
+        dest: zipDest,
+        timeoutMs: 600_000,
+        headers: { "Accept-Encoding": "gzip, br" },
+        report,
+    });
+    report?.(`  Downloaded: ${ByteFormatter.formatIEC(zipSize)}`);
+    if (zipSize < BYTES_PER_KIB) {
+        report?.(`  ✗ Response too small (${zipSize} bytes) — probable error page`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    // Discover the outlet-level CSV inside the ZIP.
+    // Outlet files match: pls_fy*outlet*.csv (case-insensitive)
+    // Administrative-entity files match: pls_fy*ae*.csv — we skip those.
+    report?.("  Inspecting ZIP contents ...");
+    const entries = (await listZipEntries(zipDest)).map((entry) => entry.name);
+    let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name));
+    // Fallback: if IMLS renames the file, grab any CSV that is NOT the ae file.
+    if (!csvName) {
+        csvName = entries.find((name) => /\.csv$/i.test(name) && !/system|state|_ae\b|_se\b/i.test(name));
+    }
+    if (!csvName) {
+        report?.("  Available files in ZIP:");
+        for (const name of entries) {
+            report?.(`    ${name}`);
+        }
+        report?.("  ✗ Could not identify outlet CSV — inspect above listing and update this module");
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    report?.(`  Extracting outlet CSV: ${csvName}`);
+    const csvDest = join(destDir, basename(csvName));
+    await extractZipEntry(zipDest, csvName, csvDest);
+    const csvSize = (await statPath(csvDest)).size;
+    const csvSha = await sha256File(csvDest);
+    // MARK: Remove ZIP (small, but keep destDir clean)
+    await removePathIfPresent(zipDest);
+    report?.("  Removed ZIP (CSV kept)");
+    // MARK: Write MANIFEST
+    const manifest = {
+        source_url: ZIP_URL,
+        downloaded_at: new Date().toISOString(),
+        filename: basename(csvName),
+        sha256: csvSha,
+        bytes: csvSize,
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`);
+    report?.(`  MANIFEST written to ${manifestPath}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=imls-pls.js.map

package/out/tools/fetch/imls-pls.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"imls-pls.js","sourceRoot":"","sources":["../../../lib/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;kGACkG;AAElG,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC5E,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAGxC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEnF;;;GAGG;AAEH;;GAEG;AACH,MAAM,OAAO,GAAG,qEAAqE,CAAA;AACrF,MAAM,IAAI,GAAG,gBAAgB,CAAA;AAI7B,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,OAA4B,EAC5B,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAChD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,0BAA0B;IAE1B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9F,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,qBAAqB;IAErB,MAAM,EAAE,CAAC,iBAAiB,OAAO,MAAM,CAAC,CAAA;IAExC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,OAAO;QACZ,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAE7D,IAAI,OAAO,GAAG,aAAa,EAAE,CAAC;QAC7B,MAAM,EAAE,CAAC,2BAA2B,OAAO,+BAA+B,CAAC,CAAA;QAE3E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,gDAAgD;IAChD,4DAA4D;IAC5D,qEAAqE;IACrE,MAAM,EAAE,CAAC,+BAA+B,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,CAAC,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAE1E,IAAI,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAEzE,4EAA4E;IAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAClG,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,2BAA2B,CAAC,CAAA;QAErC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;QAED,MAAM,EAAE,CAAC,kFAAkF,CAAC,CAAA;QAE5F,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAA;IAE/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAEhD,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,mDAAmD;IAEnD,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,uBAAuB;IAEvB,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,OAAO;QACnB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC;QAC3B,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/tools/fetch/index.d.ts

@@ -0,0 +1,171 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The `mailwoman corpus fetch <source>` family — reproducible bulk-download recovery modules for
+ *   the open-data sources the corpus build pipeline consumes. Each module writes the raw download
+ *   files **plus** a sibling `MANIFEST.json` capturing the origin URL, fetch timestamp, byte count,
+ *   and sha256 so downstream adapters can verify provenance.
+ *
+ *   The corpus build pipeline itself does NOT call these modules — the existing convention is for
+ *   operators to pre-download into an out-root and point adapters at the resulting files. These
+ *   modules exist for **reproducibility** (disk-loss recovery, weekly refresh, fresh-environment
+ *   bootstrap).
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   # Default: writes under ./data/corpus/sources/ relative to the working directory
+ *   mailwoman corpus fetch state-sources
+ *   mailwoman corpus fetch hrsa
+ *
+ *   # Or point at the standard mailwoman data root
+ *   mailwoman corpus fetch state-sources --out-root /data/corpus/sources
+ *   ```
+ *
+ *   Each adapter under `corpus/src/adapters/<adapter>/README.md` documents the specific URL its
+ *   input was pulled from; these modules mirror those URLs in a single executable place.
+ *
+ *   ## Coverage
+ *
+ *   - `ban` — French BAN (Base Adresse Nationale), all départements incl. DOM/TOM. Tier B (Licence
+ *       Ouverte 2.0).
+ *   - `nad` — US DOT National Address Database (~97M address points, ArcGIS FeatureServer). Tier A
+ *       (US PD).
+ *   - `geonames-postal` — GeoNames per-country postal exports (~80 countries). Tier B (CC-BY-4.0,
+ *       attribute "GeoNames"). The only source in this family carrying `(postcode, locality, region)`
+ *       with the names INLINE, which is why it exists: the `parent_id` join route covers exactly five
+ *       countries and the nearest-centroid fallback measured under 50% agreement on four of those five.
+ *       GeoNames does not publish every country — Venezuela 404s.
+ *   - `hrsa` — HRSA Health Center Service Delivery Sites (federal). Tier A (US PD).
+ *   - `imls-pls` — IMLS Public Libraries Survey, outlet-level (~17K library branches, FY 2023).
+ *       Tier A (US PD).
+ *   - `nppes` — NPPES NPI registry, full monthly dissemination (~7M provider venue+address rows).
+ *       Tier A (US PD).
+ *   - `openaddresses` — OpenAddresses country collections (default: Canada / `ca`). Tier B/C mixed
+ *       — per-row filter.
+ *   - `ourairports` — OurAirports global airport CSVs (~83K airports + the country/region/runway
+ *       joins). Tier A (public domain). The VENUE half of the sub-venue arc (#35); it carries no
+ *       interior structure, which comes from `@mailwoman/osm/sdk`'s `extractOSMSubVenues`.
+ *   - `wikidata-subvenue` — the multilingual sub-venue DESIGNATOR vocabulary, pulled as the labels
+ *       and aliases of eight Wikidata concepts (terminal, gate, concourse, campus, …) plus the
+ *       airport-terminal instance labels as attested usage. Tier A (CC0). The only module in this
+ *       family built on `APIClient` — see its docstring for why.
+ *   - `state-sources` — NY/TX/DE/OR notaries, IA contractors, WA health providers, HI lobbyists.
+ *       Tier A (state PD-equivalent).
+ *   - `state-hi-schools` — Hawaii DOE school directory (original XLSX; HIDOE + PCS sheets). Tier A
+ *       (state PD-equivalent).
+ *   - `tiger-full` — US Census TIGER 2024 ADDRFEAT, all US counties. Tier A (US PD).
+ *   - `juso-kr` — the Korean road-name address register (주소DB, plus the English road-name DB) from
+ *       business.juso.go.kr, the LABEL source for Korean. Tier B ("이용허락범위 제한 없음", attribution).
+ *   - `localdata-kr` — Korea's local-government permit registry, one CSV per category, each row carrying
+ *       the lot-number AND road-name address of one premises plus a coordinate. The NOISY source for
+ *       Korean. Tier B (same label as `juso-kr`).
+ *   - `gcis-tw` — Taiwan's company and business registers from the economic ministry's open-data
+ *       platform, 148 files by region and industry. The NOISY source for Taiwanese. Tier B (Open
+ *       Government Data License v1.0 — the prescribed attribution is a condition of the grant).
+ *   - `acra-sg` — Singapore's ACRA corporate-entity register, 27 CSVs with the address FIELDED. The
+ *       NOISY source for Singapore. Tier B (Singapore Open Data Licence 1.0).
+ *   - `houjin-jp` — the National Tax Agency's corporate-number register, the nationwide Unicode CSV.
+ *       The NOISY source for Japanese. Tier A-equivalent (free use, no attribution condition stated).
+ *
+ *   License tiers per `docs/licensing-strategy.md` (or the playpen knowledge base mirror at
+ *   `docs/docs/projects/mailwoman/licensing-strategy.md`). `openaddresses` is a **Tier-mixed**
+ *   source: the downloaded collection includes CC0, CC-BY, OGL, and ODbL/CC-BY-SA rows. The per-row
+ *   `LICENSE` filter in the `openaddresses` adapter is essential — Tier-C (ODbL, CC-BY-SA) rows are
+ *   dropped at ingest by default to protect proprietary-weights training.
+ *
+ *   ### OpenAddresses authentication (as of 2026-05-18)
+ *
+ *   `batch.openaddresses.io` now requires a free registered account for bulk downloads (auth check
+ *   prevents CDN abuse; data remains openly licensed). `fetchOpenAddresses` reads `OA_BATCH_TOKEN`
+ *   from the environment:
+ *
+ *   ```sh
+ *   # One-time: register at https://batch.openaddresses.io/register
+ *   # Log in → Profile → "Create Token" → copy token
+ *   export OA_BATCH_TOKEN=<your-token>
+ *
+ *   # Download Canada (~2 GiB compressed, ~7 GiB uncompressed)
+ *   mailwoman corpus fetch openaddresses --country ca \
+ *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
+ *
+ *   # Or any other OA country code
+ *   mailwoman corpus fetch openaddresses --country fr
+ *   ```
+ *
+ *   Without a token the command prints setup instructions and reports the failure.
+ *
+ *   ## Adding a new source
+ *
+ *   1. Pick the right module (or create a sibling one if the source is from a meaningfully
+ *      different family).
+ *   2. Append to the `SOURCES` array: `{ slug, filename, url }`.
+ *   3. Confirm the destination URL via `curl -sI -L <url> | head` before committing — state
+ *      open-data portals occasionally rotate Socrata view IDs.
+ *   4. Run the command against a scratch `--out-root` to verify the download succeeds + the
+ *      MANIFEST is well-formed.
+ *   5. Add the source's adapter (or extend an existing one) under `corpus/src/adapters/`.
+ */
+import { fetchACRASG } from "#tools/fetch/acra-sg";
+import { fetchBan } from "#tools/fetch/ban";
+import { fetchGCISTW } from "#tools/fetch/gcis-tw";
+import { fetchGeonamesDumps } from "#tools/fetch/geonames-dump";
+import { fetchGeonamesPostal } from "#tools/fetch/geonames-postal";
+import { fetchHoujinJP } from "#tools/fetch/houjin-jp";
+import { fetchHRSA } from "#tools/fetch/hrsa";
+import { fetchIMLSPLS } from "#tools/fetch/imls-pls";
+import { fetchJusoKR } from "#tools/fetch/juso-kr";
+import { fetchLocaldataKR } from "#tools/fetch/localdata-kr";
+import { fetchNAD } from "#tools/fetch/nad";
+import { fetchNPPES } from "#tools/fetch/nppes";
+import { fetchOpenAddresses } from "#tools/fetch/openaddresses";
+import { fetchOurAirports } from "#tools/fetch/ourairports";
+import { fetchStateHISchools } from "#tools/fetch/state-hi-schools";
+import { fetchStateSources } from "#tools/fetch/state-sources";
+import { fetchTigerFull } from "#tools/fetch/tiger-full";
+import { fetchWikidataSubVenue } from "#tools/fetch/wikidata-subvenue";
+export * from "#tools/fetch/acra-sg";
+export * from "#tools/fetch/ban";
+export * from "#tools/fetch/gcis-tw";
+export * from "#tools/fetch/geonames-dump";
+export * from "#tools/fetch/geonames-postal";
+export * from "#tools/fetch/houjin-jp";
+export * from "#tools/fetch/hrsa";
+export * from "#tools/fetch/imls-pls";
+export * from "#tools/fetch/juso-kr";
+export * from "#tools/fetch/localdata-kr";
+export * from "#tools/fetch/nad";
+export * from "#tools/fetch/nppes";
+export * from "#tools/fetch/openaddresses";
+export * from "#tools/fetch/ourairports";
+export * from "#tools/fetch/state-hi-schools";
+export * from "#tools/fetch/state-sources";
+export * from "#tools/fetch/tiger-full";
+export * from "#tools/fetch/wikidata-subvenue";
+/**
+ * The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
+ */
+export declare const FETCH_SOURCES: {
+    readonly "acra-sg": typeof fetchACRASG;
+    readonly ban: typeof fetchBan;
+    readonly "gcis-tw": typeof fetchGCISTW;
+    readonly "houjin-jp": typeof fetchHoujinJP;
+    readonly "juso-kr": typeof fetchJusoKR;
+    readonly "localdata-kr": typeof fetchLocaldataKR;
+    readonly nad: typeof fetchNAD;
+    readonly "geonames-dump": typeof fetchGeonamesDumps;
+    readonly "geonames-postal": typeof fetchGeonamesPostal;
+    readonly hrsa: typeof fetchHRSA;
+    readonly "imls-pls": typeof fetchIMLSPLS;
+    readonly nppes: typeof fetchNPPES;
+    readonly openaddresses: typeof fetchOpenAddresses;
+    readonly ourairports: typeof fetchOurAirports;
+    readonly "state-sources": typeof fetchStateSources;
+    readonly "state-hi-schools": typeof fetchStateHISchools;
+    readonly "tiger-full": typeof fetchTigerFull;
+    readonly "wikidata-subvenue": typeof fetchWikidataSubVenue;
+};
+export type FetchSourceID = keyof typeof FETCH_SOURCES;
+//# sourceMappingURL=index.d.ts.map

package/out/tools/fetch/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6GG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAA;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAA;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAA;AAEtE,cAAc,sBAAsB,CAAA;AACpC,cAAc,kBAAkB,CAAA;AAChC,cAAc,sBAAsB,CAAA;AACpC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,wBAAwB,CAAA;AACtC,cAAc,mBAAmB,CAAA;AACjC,cAAc,uBAAuB,CAAA;AACrC,cAAc,sBAAsB,CAAA;AACpC,cAAc,2BAA2B,CAAA;AACzC,cAAc,kBAAkB,CAAA;AAChC,cAAc,oBAAoB,CAAA;AAClC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,0BAA0B,CAAA;AACxC,cAAc,+BAA+B,CAAA;AAC7C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,gCAAgC,CAAA;AAE9C;;GAEG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;CAmBhB,CAAA;AAEV,MAAM,MAAM,aAAa,GAAG,MAAM,OAAO,aAAa,CAAA"}

package/out/tools/fetch/index.js

@@ -0,0 +1,170 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The `mailwoman corpus fetch <source>` family — reproducible bulk-download recovery modules for
+ *   the open-data sources the corpus build pipeline consumes. Each module writes the raw download
+ *   files **plus** a sibling `MANIFEST.json` capturing the origin URL, fetch timestamp, byte count,
+ *   and sha256 so downstream adapters can verify provenance.
+ *
+ *   The corpus build pipeline itself does NOT call these modules — the existing convention is for
+ *   operators to pre-download into an out-root and point adapters at the resulting files. These
+ *   modules exist for **reproducibility** (disk-loss recovery, weekly refresh, fresh-environment
+ *   bootstrap).
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   # Default: writes under ./data/corpus/sources/ relative to the working directory
+ *   mailwoman corpus fetch state-sources
+ *   mailwoman corpus fetch hrsa
+ *
+ *   # Or point at the standard mailwoman data root
+ *   mailwoman corpus fetch state-sources --out-root /data/corpus/sources
+ *   ```
+ *
+ *   Each adapter under `corpus/src/adapters/<adapter>/README.md` documents the specific URL its
+ *   input was pulled from; these modules mirror those URLs in a single executable place.
+ *
+ *   ## Coverage
+ *
+ *   - `ban` — French BAN (Base Adresse Nationale), all départements incl. DOM/TOM. Tier B (Licence
+ *       Ouverte 2.0).
+ *   - `nad` — US DOT National Address Database (~97M address points, ArcGIS FeatureServer). Tier A
+ *       (US PD).
+ *   - `geonames-postal` — GeoNames per-country postal exports (~80 countries). Tier B (CC-BY-4.0,
+ *       attribute "GeoNames"). The only source in this family carrying `(postcode, locality, region)`
+ *       with the names INLINE, which is why it exists: the `parent_id` join route covers exactly five
+ *       countries and the nearest-centroid fallback measured under 50% agreement on four of those five.
+ *       GeoNames does not publish every country — Venezuela 404s.
+ *   - `hrsa` — HRSA Health Center Service Delivery Sites (federal). Tier A (US PD).
+ *   - `imls-pls` — IMLS Public Libraries Survey, outlet-level (~17K library branches, FY 2023).
+ *       Tier A (US PD).
+ *   - `nppes` — NPPES NPI registry, full monthly dissemination (~7M provider venue+address rows).
+ *       Tier A (US PD).
+ *   - `openaddresses` — OpenAddresses country collections (default: Canada / `ca`). Tier B/C mixed
+ *       — per-row filter.
+ *   - `ourairports` — OurAirports global airport CSVs (~83K airports + the country/region/runway
+ *       joins). Tier A (public domain). The VENUE half of the sub-venue arc (#35); it carries no
+ *       interior structure, which comes from `@mailwoman/osm/sdk`'s `extractOSMSubVenues`.
+ *   - `wikidata-subvenue` — the multilingual sub-venue DESIGNATOR vocabulary, pulled as the labels
+ *       and aliases of eight Wikidata concepts (terminal, gate, concourse, campus, …) plus the
+ *       airport-terminal instance labels as attested usage. Tier A (CC0). The only module in this
+ *       family built on `APIClient` — see its docstring for why.
+ *   - `state-sources` — NY/TX/DE/OR notaries, IA contractors, WA health providers, HI lobbyists.
+ *       Tier A (state PD-equivalent).
+ *   - `state-hi-schools` — Hawaii DOE school directory (original XLSX; HIDOE + PCS sheets). Tier A
+ *       (state PD-equivalent).
+ *   - `tiger-full` — US Census TIGER 2024 ADDRFEAT, all US counties. Tier A (US PD).
+ *   - `juso-kr` — the Korean road-name address register (주소DB, plus the English road-name DB) from
+ *       business.juso.go.kr, the LABEL source for Korean. Tier B ("이용허락범위 제한 없음", attribution).
+ *   - `localdata-kr` — Korea's local-government permit registry, one CSV per category, each row carrying
+ *       the lot-number AND road-name address of one premises plus a coordinate. The NOISY source for
+ *       Korean. Tier B (same label as `juso-kr`).
+ *   - `gcis-tw` — Taiwan's company and business registers from the economic ministry's open-data
+ *       platform, 148 files by region and industry. The NOISY source for Taiwanese. Tier B (Open
+ *       Government Data License v1.0 — the prescribed attribution is a condition of the grant).
+ *   - `acra-sg` — Singapore's ACRA corporate-entity register, 27 CSVs with the address FIELDED. The
+ *       NOISY source for Singapore. Tier B (Singapore Open Data Licence 1.0).
+ *   - `houjin-jp` — the National Tax Agency's corporate-number register, the nationwide Unicode CSV.
+ *       The NOISY source for Japanese. Tier A-equivalent (free use, no attribution condition stated).
+ *
+ *   License tiers per `docs/licensing-strategy.md` (or the playpen knowledge base mirror at
+ *   `docs/docs/projects/mailwoman/licensing-strategy.md`). `openaddresses` is a **Tier-mixed**
+ *   source: the downloaded collection includes CC0, CC-BY, OGL, and ODbL/CC-BY-SA rows. The per-row
+ *   `LICENSE` filter in the `openaddresses` adapter is essential — Tier-C (ODbL, CC-BY-SA) rows are
+ *   dropped at ingest by default to protect proprietary-weights training.
+ *
+ *   ### OpenAddresses authentication (as of 2026-05-18)
+ *
+ *   `batch.openaddresses.io` now requires a free registered account for bulk downloads (auth check
+ *   prevents CDN abuse; data remains openly licensed). `fetchOpenAddresses` reads `OA_BATCH_TOKEN`
+ *   from the environment:
+ *
+ *   ```sh
+ *   # One-time: register at https://batch.openaddresses.io/register
+ *   # Log in → Profile → "Create Token" → copy token
+ *   export OA_BATCH_TOKEN=<your-token>
+ *
+ *   # Download Canada (~2 GiB compressed, ~7 GiB uncompressed)
+ *   mailwoman corpus fetch openaddresses --country ca \
+ *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
+ *
+ *   # Or any other OA country code
+ *   mailwoman corpus fetch openaddresses --country fr
+ *   ```
+ *
+ *   Without a token the command prints setup instructions and reports the failure.
+ *
+ *   ## Adding a new source
+ *
+ *   1. Pick the right module (or create a sibling one if the source is from a meaningfully
+ *      different family).
+ *   2. Append to the `SOURCES` array: `{ slug, filename, url }`.
+ *   3. Confirm the destination URL via `curl -sI -L <url> | head` before committing — state
+ *      open-data portals occasionally rotate Socrata view IDs.
+ *   4. Run the command against a scratch `--out-root` to verify the download succeeds + the
+ *      MANIFEST is well-formed.
+ *   5. Add the source's adapter (or extend an existing one) under `corpus/src/adapters/`.
+ */
+import { fetchACRASG } from "#tools/fetch/acra-sg";
+import { fetchBan } from "#tools/fetch/ban";
+import { fetchGCISTW } from "#tools/fetch/gcis-tw";
+import { fetchGeonamesDumps } from "#tools/fetch/geonames-dump";
+import { fetchGeonamesPostal } from "#tools/fetch/geonames-postal";
+import { fetchHoujinJP } from "#tools/fetch/houjin-jp";
+import { fetchHRSA } from "#tools/fetch/hrsa";
+import { fetchIMLSPLS } from "#tools/fetch/imls-pls";
+import { fetchJusoKR } from "#tools/fetch/juso-kr";
+import { fetchLocaldataKR } from "#tools/fetch/localdata-kr";
+import { fetchNAD } from "#tools/fetch/nad";
+import { fetchNPPES } from "#tools/fetch/nppes";
+import { fetchOpenAddresses } from "#tools/fetch/openaddresses";
+import { fetchOurAirports } from "#tools/fetch/ourairports";
+import { fetchStateHISchools } from "#tools/fetch/state-hi-schools";
+import { fetchStateSources } from "#tools/fetch/state-sources";
+import { fetchTigerFull } from "#tools/fetch/tiger-full";
+import { fetchWikidataSubVenue } from "#tools/fetch/wikidata-subvenue";
+export * from "#tools/fetch/acra-sg";
+export * from "#tools/fetch/ban";
+export * from "#tools/fetch/gcis-tw";
+export * from "#tools/fetch/geonames-dump";
+export * from "#tools/fetch/geonames-postal";
+export * from "#tools/fetch/houjin-jp";
+export * from "#tools/fetch/hrsa";
+export * from "#tools/fetch/imls-pls";
+export * from "#tools/fetch/juso-kr";
+export * from "#tools/fetch/localdata-kr";
+export * from "#tools/fetch/nad";
+export * from "#tools/fetch/nppes";
+export * from "#tools/fetch/openaddresses";
+export * from "#tools/fetch/ourairports";
+export * from "#tools/fetch/state-hi-schools";
+export * from "#tools/fetch/state-sources";
+export * from "#tools/fetch/tiger-full";
+export * from "#tools/fetch/wikidata-subvenue";
+/**
+ * The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
+ */
+export const FETCH_SOURCES = {
+    "acra-sg": fetchACRASG,
+    ban: fetchBan,
+    "gcis-tw": fetchGCISTW,
+    "houjin-jp": fetchHoujinJP,
+    "juso-kr": fetchJusoKR,
+    "localdata-kr": fetchLocaldataKR,
+    nad: fetchNAD,
+    "geonames-dump": fetchGeonamesDumps,
+    "geonames-postal": fetchGeonamesPostal,
+    hrsa: fetchHRSA,
+    "imls-pls": fetchIMLSPLS,
+    nppes: fetchNPPES,
+    openaddresses: fetchOpenAddresses,
+    ourairports: fetchOurAirports,
+    "state-sources": fetchStateSources,
+    "state-hi-schools": fetchStateHISchools,
+    "tiger-full": fetchTigerFull,
+    "wikidata-subvenue": fetchWikidataSubVenue,
+};
+//# sourceMappingURL=index.js.map

package/out/tools/fetch/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../lib/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6GG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAA;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAA;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAA;AAEtE,cAAc,sBAAsB,CAAA;AACpC,cAAc,kBAAkB,CAAA;AAChC,cAAc,sBAAsB,CAAA;AACpC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,wBAAwB,CAAA;AACtC,cAAc,mBAAmB,CAAA;AACjC,cAAc,uBAAuB,CAAA;AACrC,cAAc,sBAAsB,CAAA;AACpC,cAAc,2BAA2B,CAAA;AACzC,cAAc,kBAAkB,CAAA;AAChC,cAAc,oBAAoB,CAAA;AAClC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,0BAA0B,CAAA;AACxC,cAAc,+BAA+B,CAAA;AAC7C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,gCAAgC,CAAA;AAE9C;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B,SAAS,EAAE,WAAW;IACtB,GAAG,EAAE,QAAQ;IACb,SAAS,EAAE,WAAW;IACtB,WAAW,EAAE,aAAa;IAC1B,SAAS,EAAE,WAAW;IACtB,cAAc,EAAE,gBAAgB;IAChC,GAAG,EAAE,QAAQ;IACb,eAAe,EAAE,kBAAkB;IACnC,iBAAiB,EAAE,mBAAmB;IACtC,IAAI,EAAE,SAAS;IACf,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE,UAAU;IACjB,aAAa,EAAE,kBAAkB;IACjC,WAAW,EAAE,gBAAgB;IAC7B,eAAe,EAAE,iBAAiB;IAClC,kBAAkB,EAAE,mBAAmB;IACvC,YAAY,EAAE,cAAc;IAC5B,mBAAmB,EAAE,qBAAqB;CACjC,CAAA"}

package/out/tools/fetch/juso-kr.d.ts

@@ -0,0 +1,30 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the Korean road-name address register (도로명주소 주소DB) from the ministry's address-industry
+ *   portal, business.juso.go.kr. This is the LABEL source for Korean addresses: about 6,000,000
+ *   road-name addresses and 8,000,000 lot-number (지번) records, split by the publisher into four
+ *   pipe-delimited CP949 text files per region (도로명코드 / 주소 / 지번 / 부가정보), with the postcode in
+ *   the supplementary file. No coordinates. The public-data portal labels it "이용허락범위 제한 없음"
+ *   (no restriction on the scope of use), and it is a direct download with no application step — the
+ *   distinction that matters against the ministry's entrance-coordinate products, which are provided
+ *   only after a purpose-of-use review (`docs/superpowers/plans/counsel-dossier.md` §4).
+ *
+ *   The portal is a single-page app; the file behind it is reached in two calls. `selectAttrbDBDwldList`
+ *   lists the monthly full files for one product (`rtlDtaDtlSn` 8 is the 주소DB, 2 the English
+ *   road-name DB), and `/api/jst/download` streams one by the parameters the listing carried. The
+ *   English DB rides along because it is the Latin-script half of the same register.
+ *
+ *   Invoke via `mailwoman corpus fetch juso-kr --out-root <path>`.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export interface FetchJusoKROptions extends BaseFetchOptions {
+    /**
+     * The month to fetch as `YYYYMM`. Defaults to the latest month the portal lists as present.
+     */
+    month?: string;
+}
+export declare function fetchJusoKR(options: FetchJusoKROptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=juso-kr.d.ts.map

package/out/tools/fetch/juso-kr.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"juso-kr.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/juso-kr.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAMH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAA4C,MAAM,uBAAuB,CAAA;AAoCrH,MAAM,WAAW,kBAAmB,SAAQ,gBAAgB;IAC3D;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAsCD,wBAAsB,WAAW,CAAC,OAAO,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CA2DrH"}

package/out/tools/fetch/juso-kr.js

@@ -0,0 +1,120 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the Korean road-name address register (도로명주소 주소DB) from the ministry's address-industry
+ *   portal, business.juso.go.kr. This is the LABEL source for Korean addresses: about 6,000,000
+ *   road-name addresses and 8,000,000 lot-number (지번) records, split by the publisher into four
+ *   pipe-delimited CP949 text files per region (도로명코드 / 주소 / 지번 / 부가정보), with the postcode in
+ *   the supplementary file. No coordinates. The public-data portal labels it "이용허락범위 제한 없음"
+ *   (no restriction on the scope of use), and it is a direct download with no application step — the
+ *   distinction that matters against the ministry's entrance-coordinate products, which are provided
+ *   only after a purpose-of-use review (`docs/superpowers/plans/counsel-dossier.md` §4).
+ *
+ *   The portal is a single-page app; the file behind it is reached in two calls. `selectAttrbDBDwldList`
+ *   lists the monthly full files for one product (`rtlDtaDtlSn` 8 is the 주소DB, 2 the English
+ *   road-name DB), and `/api/jst/download` streams one by the parameters the listing carried. The
+ *   English DB rides along because it is the Latin-script half of the same register.
+ *
+ *   Invoke via `mailwoman corpus fetch juso-kr --out-root <path>`.
+ */
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { resumableDownload, writeManifest } from "#tools/fetch/download";
+const SLUG = "juso-kr";
+const PORTAL = "https://business.juso.go.kr";
+const LIST_URL = `${PORTAL}/api/jst/selectAttrbDBDwldList`;
+const DOWNLOAD_URL = `${PORTAL}/api/jst/download`;
+const REFERER = `${PORTAL}/jst/jstAddressDownload?menu=30`;
+/**
+ * The portal's product ids for the two registers this module fetches.
+ */
+const PRODUCTS = [
+    { sn: "8", name: "주소DB" },
+    { sn: "2", name: "도로명주소 영문" },
+];
+const LICENSE = "이용허락범위 제한 없음 (data.go.kr label for 행정안전부_도로명주소 주소DB); 공공누리 attribution to 행정안전부";
+const ATTRIBUTION = "행정안전부 (Ministry of the Interior and Safety), 도로명주소 (juso.go.kr)";
+const BROWSER_HEADERS = {
+    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch",
+    referer: REFERER,
+};
+/**
+ * The monthly full files the portal lists for one product over the twelve months ending at `year`/`month`.
+ */
+async function listMonthlyFiles(sn, year, month) {
+    const res = await fetch(LIST_URL, {
+        method: "POST",
+        headers: { ...BROWSER_HEADERS, accept: "application/json", "content-type": "application/json;charset=UTF-8" },
+        body: JSON.stringify({ rtlDtaDtlSn: sn, year, month, expand: "Y" }),
+    });
+    if (!res.ok)
+        throw new Error(`juso-kr: listing product ${sn} answered HTTP ${res.status}`);
+    const json = (await res.json());
+    return (json.results?.allMonthFileList ?? []).filter((file) => file.isExist === "Y");
+}
+function downloadURL(file, regYmd) {
+    const query = new URLSearchParams({
+        reqType: file.fileTypeNm,
+        ctprvnCd: file.ctpvClsfCd,
+        stdde: file.crtrYm,
+        fileName: file.fileNm,
+        realFileName: file.tmprFileNm,
+        intFileNo: file.fileSn ?? "0",
+        intNum: file.atflNo ?? "0",
+        regYmd,
+    });
+    return `${DOWNLOAD_URL}?${query.toString()}`;
+}
+export async function fetchJusoKR(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const now = new Date();
+    const files = [];
+    const failedCodes = [];
+    for (const product of PRODUCTS) {
+        const listed = await listMonthlyFiles(product.sn, now.getFullYear(), now.getMonth() + 1);
+        const file = options.month ? listed.find((entry) => entry.crtrYm === options.month) : listed.at(-1);
+        if (!file) {
+            report?.(`  ✗ ${product.name}: no full file listed${options.month ? ` for ${options.month}` : ""}`);
+            failedCodes.push(product.name);
+            continue;
+        }
+        const url = downloadURL(file, String(now.getFullYear()));
+        const dest = join(destDir, file.tmprFileNm);
+        report?.(`=== ${SLUG} / ${product.name} ${file.crtrYm} → ${file.tmprFileNm}`);
+        let bytes;
+        try {
+            // The portal drops a connection every few megabytes and answers Range with 206, so the transfer resumes
+            // from what landed rather than starting the 181 MB over.
+            bytes = await resumableDownload({ url, dest, headers: BROWSER_HEADERS, report });
+        }
+        catch (error) {
+            report?.(`  ✗ ${error.message}`);
+            failedCodes.push(product.name);
+            continue;
+        }
+        const sha = await sha256File(dest);
+        files.push({
+            source_url: url,
+            downloaded_at: new Date().toISOString(),
+            filename: file.tmprFileNm,
+            sha256: sha,
+            bytes,
+        });
+        report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
+    }
+    const manifest = {
+        source: SLUG,
+        source_url: REFERER,
+        license: LICENSE,
+        attribution: ATTRIBUTION,
+        downloaded_at: new Date().toISOString(),
+        files,
+    };
+    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
+    return { fetched: files.length, skipped: 0, failed: failedCodes.length, failedCodes };
+}
+//# sourceMappingURL=juso-kr.js.map

package/out/tools/fetch/juso-kr.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"juso-kr.js","sourceRoot":"","sources":["../../../lib/tools/fetch/juso-kr.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAExE,MAAM,IAAI,GAAG,SAAS,CAAA;AACtB,MAAM,MAAM,GAAG,6BAA6B,CAAA;AAC5C,MAAM,QAAQ,GAAG,GAAG,MAAM,gCAAgC,CAAA;AAC1D,MAAM,YAAY,GAAG,GAAG,MAAM,mBAAmB,CAAA;AACjD,MAAM,OAAO,GAAG,GAAG,MAAM,iCAAiC,CAAA;AAE1D;;GAEG;AACH,MAAM,QAAQ,GAAG;IAChB,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE;IACzB,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE;CACpB,CAAA;AAEV,MAAM,OAAO,GACZ,iFAAiF,CAAA;AAClF,MAAM,WAAW,GAAG,iEAAiE,CAAA;AAwBrF,MAAM,eAAe,GAAG;IACvB,YAAY,EAAE,wDAAwD;IACtE,OAAO,EAAE,OAAO;CAChB,CAAA;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,EAAU,EAAE,IAAY,EAAE,KAAa;IACtE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,GAAG,eAAe,EAAE,MAAM,EAAE,kBAAkB,EAAE,cAAc,EAAE,gCAAgC,EAAE;QAC7G,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;KACnE,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,EAAE,kBAAkB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;IAC1F,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAiB,CAAA;IAE/C,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,GAAG,CAAC,CAAA;AACrF,CAAC;AAED,SAAS,WAAW,CAAC,IAAgB,EAAE,MAAc;IACpD,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC;QACjC,OAAO,EAAE,IAAI,CAAC,UAAU;QACxB,QAAQ,EAAE,IAAI,CAAC,UAAU;QACzB,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,YAAY,EAAE,IAAI,CAAC,UAAU;QAC7B,SAAS,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG;QAC7B,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG;QAC1B,MAAM;KACN,CAAC,CAAA;IAEF,OAAO,GAAG,YAAY,IAAI,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAA2B,EAAE,MAA+B;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAA;IACtB,MAAM,KAAK,GAAqB,EAAE,CAAA;IAClC,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAA;QACxF,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAEnG,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,MAAM,EAAE,CAAC,OAAO,OAAO,CAAC,IAAI,wBAAwB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAE9B,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;QACxD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;QAC3C,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAA;QAC7E,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,wGAAwG;YACxG,yDAAyD;YACzD,KAAK,GAAG,MAAM,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAA;QACjF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAQ,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC3C,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAE9B,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAElC,KAAK,CAAC,IAAI,CAAC;YACV,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ,EAAE,IAAI,CAAC,UAAU;YACzB,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAC,CAAA;QACF,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;IACtE,CAAC;IAED,MAAM,QAAQ,GAA6B;QAC1C,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,OAAO;QACnB,OAAO,EAAE,OAAO;QAChB,WAAW,EAAE,WAAW;QACxB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK;KACL,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,CAAA;AACtF,CAAC"}