@mailwoman/corpus 9.2.0 → 9.3.0

package/out/src/tools/fetch/tiger-full.js

@@ -1,248 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
- *
- *   TIGER ADDRFEAT 2024 source:
- *
- *   - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
- *   - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
- *
- *   Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
- *   `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
- *   already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
- *   this module is download + provenance only.)
- *
- *   Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
- *   county ZIP to disk (no curl subprocess).
- */
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-import { createWriteStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
-import { basename, join } from "node:path";
-import { Readable } from "node:stream";
-import { pipeline } from "node:stream/promises";
-import { setTimeout as sleep } from "node:timers/promises";
-import { APIClient, pluckResponseData } from "@mailwoman/core/api";
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/utils";
-import { sha256File } from "@mailwoman/core/utils";
-import { isTransientStatus, readManifest, writeManifest } from "./download.js";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-/**
- * Lowest 2xx status; anything below is informational.
- */
-const HTTP_OK = 200;
-/**
- * Lowest 3xx status; at or above it the response is a redirect or an error, not a body.
- */
-const HTTP_REDIRECT = 300;
-const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT";
-/**
- * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries).
- *
- * NOTE(phase1): kept local instead of the shared `downloadToFile` — this one streams each county ZIP to disk (the
- * shared util buffers via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-county result
- * collector consumes.
- */
-async function streamDownload(url, dest, opts) {
-    for (let attempt = 0; attempt <= opts.retries; attempt++) {
-        try {
-            const res = await fetch(url, { redirect: "follow", signal: AbortSignal.timeout(opts.timeoutMs) });
-            if (res.ok && res.body) {
-                await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
-                return res.status;
-            }
-            if (attempt < opts.retries && isTransientStatus(res.status)) {
-                await sleep(opts.retryDelayMs);
-                continue;
-            }
-            return res.status;
-        }
-        catch {
-            if (attempt < opts.retries) {
-                await sleep(opts.retryDelayMs);
-                continue;
-            }
-            return 0;
-        }
-    }
-    return 0;
-}
-/**
- * Read a per-state MANIFEST.json into a filename → entry map.
- */
-async function readCountyManifest(manifestPath) {
-    const map = new Map();
-    const parsed = await readManifest(manifestPath);
-    for (const c of parsed?.counties ?? []) {
-        if (c.filename) {
-            map.set(c.filename, { filename: c.filename, sha256: c.sha256, bytes: c.bytes });
-        }
-    }
-    return map;
-}
-/**
- * Check whether a file already matches a recorded sha256 and byte count.
- */
-async function fileMatchesSha(path, expectedSha, expectedBytes) {
-    if (!existsSync(path))
-        return false;
-    if (statSync(path).size !== expectedBytes)
-        return false;
-    return (await sha256File(path)) === expectedSha;
-}
-/**
- * Download one county ZIP (size sanity check + sha256).
- */
-async function downloadCounty(url, dest) {
-    const filename = basename(dest);
-    const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 });
-    if (status < HTTP_OK || status >= HTTP_REDIRECT) {
-        return { ok: false, filename, reason: `HTTP ${status}` };
-    }
-    const bytes = statSync(dest).size;
-    if (bytes < BYTES_PER_KIB) {
-        rmSync(dest, { force: true });
-        return { ok: false, filename, reason: `too small (${bytes} bytes)` };
-    }
-    const sha256 = await sha256File(dest);
-    return { ok: true, filename, sha256, bytes };
-}
-export async function fetchTigerFull(options, report) {
-    const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length > 0);
-    const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000);
-    const maxParallel = options.maxParallel ?? 4;
-    const dryRun = options.dryRun ?? false;
-    const addrfeatDir = join(options.outRoot, "tiger", "addrfeat");
-    mkdirSync(addrfeatDir, { recursive: true });
-    // MARK: Step 1 — discover the county file list
-    report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`);
-    // `responseType: "text"` — an Apache directory index, scraped below. The per-county ARCHIVE downloads
-    // stay on raw `fetch` (they stream to disk; see `downloadOne`).
-    const listingRes = await new APIClient({
-        displayName: "tiger-listing",
-        retry: true,
-        axios: { headers: { "Accept-Encoding": "gzip, br" } },
-    })
-        .fetch({ url: `${TIGER_BASE_URL}/`, responseType: "text", timeout: 60_000 })
-        .then(pluckResponseData);
-    const html = listingRes;
-    const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted();
-    const totalCounties = allZips.length;
-    report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`);
-    // Build a map: state_fips -> list of filenames.
-    // tl_2024_SSCCC_addrfeat.zip — SS = 2-digit state FIPS (chars 8-9), CCC = county FIPS.
-    const stateFiles = new Map();
-    for (const fname of allZips) {
-        const stateFips = fname.slice(8, 10);
-        const list = stateFiles.get(stateFips) ?? [];
-        list.push(fname);
-        stateFiles.set(stateFips, list);
-    }
-    report?.(`  Spans ${stateFiles.size} state/territory FIPS codes.`);
-    // MARK: Step 2 — download the county ZIPs
-    let totalFetched = 0;
-    let totalSkipped = 0;
-    let totalSkippedState = 0;
-    let totalFailed = 0;
-    let totalBytesFetched = 0;
-    const failedCodes = [];
-    // Process states in sorted FIPS order for predictable output.
-    const sortedStates = [...stateFiles.keys()].toSorted();
-    for (const stateFips of sortedStates) {
-        const countyFiles = stateFiles.get(stateFips) ?? [];
-        // --- Skip entire state if requested ------------------------------------------
-        if (skipStateFips.includes(stateFips)) {
-            report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`);
-            totalSkippedState += countyFiles.length;
-            continue;
-        }
-        const stateDir = join(addrfeatDir, `state-${stateFips}`);
-        mkdirSync(stateDir, { recursive: true });
-        const manifestPath = join(stateDir, "MANIFEST.json");
-        // Load existing manifest for O(1) verified-skip lookup.
-        const manifest = await readCountyManifest(manifestPath);
-        report?.(`--- State ${stateFips} — ${countyFiles.length} counties`);
-        // Build a list of URLs+dests that need fetching.
-        const pending = [];
-        for (const fname of countyFiles) {
-            const dest = join(stateDir, fname);
-            const url = `${TIGER_BASE_URL}/${fname}`;
-            const known = manifest.get(fname);
-            // Skip if already verified via MANIFEST.
-            if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
-                report?.(`  skip (verified) ${fname}`);
-                totalSkipped++;
-                continue;
-            }
-            if (dryRun) {
-                report?.(`  would fetch: ${url}`);
-                totalFetched++;
-                continue;
-            }
-            pending.push({ url, dest });
-        }
-        if (dryRun)
-            continue;
-        if (!pending.length)
-            continue;
-        // --- Download pending files with bounded parallelism + rate-limit spacing ---
-        const results = new Array(pending.length);
-        let cursor = 0;
-        const workers = Array.from({ length: Math.min(maxParallel, pending.length) }, async () => {
-            while (true) {
-                const i = cursor++;
-                if (i >= pending.length)
-                    return;
-                const item = pending[i];
-                // Rate-limit: polite spacing before each fetch.
-                await sleep(rateSleepMs);
-                results[i] = await downloadCounty(item.url, item.dest);
-            }
-        });
-        await Promise.all(workers);
-        // Collect results from this state.
-        for (const result of results) {
-            if (result.ok) {
-                report?.(`  ok ${result.filename}  ${ByteFormatter.formatIEC(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`);
-                manifest.set(result.filename, { filename: result.filename, sha256: result.sha256, bytes: result.bytes });
-                totalFetched++;
-                totalBytesFetched += result.bytes;
-            }
-            else {
-                report?.(`  FAIL ${result.filename} -- ${result.reason}`);
-                totalFailed++;
-                failedCodes.push(result.filename);
-            }
-        }
-        // Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
-        const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename));
-        const manifestDoc = {
-            state_fips: stateFips,
-            updated_at: new Date().toISOString().replace(/\.\d{3}Z$/, "Z"),
-            tiger_base_url: TIGER_BASE_URL,
-            counties,
-        };
-        await writeManifest(manifestPath, manifestDoc);
-    }
-    // MARK: Summary
-    report?.(`=== Summary ===`);
-    report?.(`  Total counties in index   : ${totalCounties}`);
-    report?.(`  State(s) fully skipped    : ${totalSkippedState} (--skip-state-fips "${skipStateFips.join(" ")}")`);
-    report?.(`  Counties already present  : ${totalSkipped}`);
-    report?.(`  Counties fetched this run : ${totalFetched}`);
-    report?.(`  Counties failed           : ${totalFailed}`);
-    if (totalBytesFetched > 0) {
-        report?.(`  Bytes fetched this run    : ${ByteFormatter.formatIEC(totalBytesFetched)} (${totalBytesFetched})`);
-    }
-    if (totalFailed > 0) {
-        report?.(`WARNING: ${totalFailed} download(s) failed. Re-run to retry.`);
-    }
-    return { fetched: totalFetched, skipped: totalSkipped + totalSkippedState, failed: totalFailed, failedCodes };
-}
-//# sourceMappingURL=tiger-full.js.map

package/out/src/tools/fetch/tiger-full.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"tiger-full.js","sourceRoot":"","sources":["../../../../src/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;kGACkG;AAElG,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACpF,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAA;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE9E;;;GAGG;AACH;;GAEG;AACH,MAAM,OAAO,GAAG,GAAG,CAAA;AAEnB;;GAEG;AACH,MAAM,aAAa,GAAG,GAAG,CAAA;AAEzB,MAAM,cAAc,GAAG,sDAAsD,CAAA;AA4B7E;;;;;;GAMG;AACH,KAAK,UAAU,cAAc,CAC5B,GAAW,EACX,IAAY,EACZ,IAAkE;IAElE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;YAEjG,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAA;gBAEnE,OAAO,GAAG,CAAC,MAAM,CAAA;YAClB,CAAC;YAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,GAAG,CAAC,MAAM,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,kBAAkB,CAAC,YAAoB;IACrD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAuB,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,YAAY,CAA+B,YAAY,CAAC,CAAA;IAE7E,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,QAAQ,IAAI,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YAChB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;QAChF,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,WAAmB,EAAE,aAAqB;IACrF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAA;IAEnC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,aAAa;QAAE,OAAO,KAAK,CAAA;IAEvD,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,WAAW,CAAA;AAChD,CAAC;AAMD;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY;IACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAC/B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtG,IAAI,MAAM,GAAG,OAAO,IAAI,MAAM,IAAI,aAAa,EAAE,CAAC;QACjD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,MAAM,EAAE,EAAE,CAAA;IACzD,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAA;IAEjC,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAE7B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,KAAK,SAAS,EAAE,CAAA;IACrE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAErC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,aAAa,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACtG,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;IACjE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IAEtC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,CAAA;IAC9D,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,+CAA+C;IAE/C,MAAM,EAAE,CAAC,uDAAuD,CAAC,CAAA;IAEjE,sGAAsG;IACtG,gEAAgE;IAChE,MAAM,UAAU,GAAG,MAAM,IAAI,SAAS,CAAC;QACtC,WAAW,EAAE,eAAe;QAC5B,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;KACrD,CAAC;SACA,KAAK,CAAS,EAAE,GAAG,EAAE,GAAG,cAAc,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SACnF,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,MAAM,IAAI,GAAG,UAAU,CAAA;IACvB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;IACtF,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAA;IACpC,MAAM,EAAE,CAAC,WAAW,aAAa,gDAAgD,CAAC,CAAA;IAElF,gDAAgD;IAChD,uFAAuF;IACvF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE9C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACpC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAC5C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChB,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAChC,CAAC;IAED,MAAM,EAAE,CAAC,WAAW,UAAU,CAAC,IAAI,8BAA8B,CAAC,CAAA;IAElE,0CAA0C;IAE1C,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,8DAA8D;IAC9D,MAAM,YAAY,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEtD,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAEnD,gFAAgF;QAChF,IAAI,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,MAAM,EAAE,CAAC,aAAa,SAAS,qCAAqC,WAAW,CAAC,MAAM,YAAY,CAAC,CAAA;YACnG,iBAAiB,IAAI,WAAW,CAAC,MAAM,CAAA;YAEvC,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,SAAS,SAAS,EAAE,CAAC,CAAA;QACxD,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACxC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAA;QAEpD,wDAAwD;QACxD,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,CAAA;QAEvD,MAAM,EAAE,CAAC,aAAa,SAAS,MAAM,WAAW,CAAC,MAAM,WAAW,CAAC,CAAA;QAEnE,iDAAiD;QACjD,MAAM,OAAO,GAAyC,EAAE,CAAA;QAExD,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YAClC,MAAM,GAAG,GAAG,GAAG,cAAc,IAAI,KAAK,EAAE,CAAA;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAEjC,yCAAyC;YACzC,IAAI,KAAK,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACtE,MAAM,EAAE,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAA;gBAEtC,YAAY,EAAE,CAAA;gBAEd,SAAQ;YACT,CAAC;YAED,IAAI,MAAM,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAA;gBAEjC,YAAY,EAAE,CAAA;gBAEd,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;QAC5B,CAAC;QAED,IAAI,MAAM;YAAE,SAAQ;QAEpB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,SAAQ;QAE7B,+EAA+E;QAC/E,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACzD,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;YACxF,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;gBAElB,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM;oBAAE,OAAM;gBAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;gBACxB,gDAAgD;gBAChD,MAAM,KAAK,CAAC,WAAW,CAAC,CAAA;gBACxB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACvD,CAAC;QACF,CAAC,CAAC,CAAA;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAE1B,mCAAmC;QACnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,EAAE,CACP,QAAQ,MAAM,CAAC,QAAQ,KAAK,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAC5G,CAAA;gBAED,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;gBAExG,YAAY,EAAE,CAAA;gBACd,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAA;YAClC,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,UAAU,MAAM,CAAC,QAAQ,OAAO,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;gBAEzD,WAAW,EAAE,CAAA;gBACb,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YAClC,CAAC;QACF,CAAC;QAED,yFAAyF;QACzF,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;QAEhG,MAAM,WAAW,GAAG;YACnB,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;YAC9D,cAAc,EAAE,cAAc;YAC9B,QAAQ;SACR,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;IAC/C,CAAC;IAED,gBAAgB;IAEhB,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,iCAAiC,aAAa,EAAE,CAAC,CAAA;IAC1D,MAAM,EAAE,CAAC,iCAAiC,iBAAiB,wBAAwB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,WAAW,EAAE,CAAC,CAAA;IAExD,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,CAAC,iCAAiC,aAAa,CAAC,SAAS,CAAC,iBAAiB,CAAC,KAAK,iBAAiB,GAAG,CAAC,CAAA;IAC/G,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,EAAE,CAAC,YAAY,WAAW,uCAAuC,CAAC,CAAA;IACzE,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,GAAG,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAA;AAC9G,CAAC"}

package/out/src/tools/fetch/wikidata-subvenue.d.ts

@@ -1,171 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the multilingual sub-venue designator vocabulary from Wikidata (#35 wave 1).
- *
- *   Source : https://query.wikidata.org/sparql (the Wikidata Query Service).
- *   License: CC0. Wikidata's data is public-domain dedicated, so nothing rides on a derived shard.
- *            Tier A.
- *
- *   ## The pull is CLASS labels, not instance names — and that inversion is the whole design
- *
- *   The obvious read of "Wikidata for localized designators" is: fetch every airport terminal entity
- *   and read its name in each language. That was tried first and it is the WRONG query. Wikidata
- *   knows 246 items that are `instance of / subclass of*` airport terminal, carrying 775 labels
- *   between them (measured 2026-08-04), and most of those labels are proper names that translate
- *   verbatim — `TWA Flight Center` is spelled `TWA Flight Center` in fifteen languages. The
- *   instance layer is thin and its localization is mostly a no-op.
- *
- *   The DESIGNATOR is the label of the CLASS. `wd:Q849706` ("airport terminal") is labelled `Terminal`
- *   in German, `terminal aéroportuaire` in French, `ターミナルビル` in Japanese, `航站楼` in Chinese —
- *   and `skos:altLabel` adds the aliases (`Abfertigungsgebäude`, `Flughafenterminal`, `aerostazione`).
- *   Eight concept ids yield 877 label+alias rows across 174 languages, which is the vocabulary the
- *   corpus task asked for and the instance query does not contain. {@link SUBVENUE_CONCEPTS} is that
- *   list of ids; {@link buildDesignatorLabelQuery} is that query.
- *
- *   Instance labels are fetched too ({@link buildTerminalInstanceQuery}), for a different job: they
- *   are ATTESTED USAGE — evidence of how a designator combines with a modifier or an identifier in
- *   running text. 775 rows is small, and it is a validation set, not a vocabulary.
- *
- *   ## What a caller must NOT do with the output
- *
- *   A class label is a CONCEPT NAME, not a designator as written in an address. Q849706's Spanish
- *   label is `terminal aeroportuaria` and its French is `terminal d'aéroport`; nobody writes either on
- *   an envelope, they write `Terminal`. Q247739's Spanish is `puerta de embarque` where the addressed
- *   form is `Puerta`. So this fetch produces CANDIDATE SURFACES that need a head-noun/curation pass
- *   before any of them reaches `neural/venue-structure.ts`'s designator vocabulary — the lexicon
- *   builder marks every one `curated: false` and the burden of promotion is on a human. Wiring the raw
- *   pull straight into the span proposer would admit multi-word phrases that match nothing and, worse,
- *   admit `hall` in a language where it names an ordinary room.
- *
- *   ## Why `APIClient` here when the OurAirports sibling uses `downloadToFile`
- *
- *   This is the API-request side of `AGENTS.md`'s split: small JSON bodies, several calls per run, and
- *   a host that publishes a rate policy and enforces it with 429s. Pacing, bounded `Retry-After`-aware
- *   retry, response caching and `ResourceError` mapping all earn their keep, so it extends
- *   {@link APIClient}. `ourairports.ts` is four static file transfers off a CDN and correctly does not.
- *
- *   WDQS also REQUIRES a descriptive `User-Agent` naming the tool and a contact — an anonymous or
- *   library-default agent is blocked outright by the Wikimedia user-agent policy. See
- *   {@link WIKIDATA_USER_AGENT}.
- *
- *   Invoke via `mailwoman corpus fetch wikidata-subvenue --out-root <path>`.
- */
-import { APIClient, type ClockLike } from "@mailwoman/core/api";
-import type { BaseFetchOptions, FetchSummary } from "./download.ts";
-/**
- * The SPARQL endpoint. Public, no credential.
- */
-export declare const WDQS_ENDPOINT = "https://query.wikidata.org/sparql";
-/**
- * The `User-Agent` every request carries.
- *
- * NOT decoration. The Wikimedia user-agent policy blocks requests whose agent is absent, generic, or a library default,
- * and WDQS enforces it — an unidentified client gets a 403 that no amount of retrying fixes. The policy asks for a tool
- * name, a URL, and a contact address, all three of which are here.
- */
-export declare const WIKIDATA_USER_AGENT = "mailwoman/1.0 (https://github.com/sister-software/mailwoman; teffen@sister.software) corpus-subvenue-fetch";
-/**
- * One Wikidata concept whose labels are a designator's multilingual surface set.
- *
- * `designatorID` is the mailwoman-side vocabulary term, matching `neural/venue-structure.ts`'s
- * `VENUE_STRUCTURE_DESIGNATORS` wherever the two overlap. `qid` was resolved by `wbsearchentities` and hand-checked
- * against the entity's English description (recorded below) on 2026-08-04 — a QID picked by search alone is how you end
- * up pulling the labels of a Bronx neighbourhood called Concourse.
- *
- * `wing` is ABSENT and that is a finding, not an oversight: Wikidata has no clean concept for "wing of a building".
- * `wbsearchentities` for "wing" returns a surname, two English villages, a rugby position and a drone company. Since
- * `wing` is the single most valuable designator in the arc — `West Wing` is the one modifier case that already parses,
- * and `East Wing` is the one that does not — its localized surfaces have to come from somewhere else. See the wave-1
- * report.
- */
-export interface SubVenueConcept {
-    designatorID: string;
-    qid: string;
-    /**
-     * The entity's English description, recorded so a future reader can tell at a glance whether the QID still names what
-     * we think it names.
-     */
-    gloss: string;
-}
-/**
- * The concept table. Eight ids, each verified against its English description on 2026-08-04.
- */
-export declare const SUBVENUE_CONCEPTS: readonly SubVenueConcept[];
-/**
- * Build the class-label query: `rdfs:label` and `skos:altLabel` for every concept, in every language, tagged with which
- * of the two it came from so the lexicon can rank a label above an alias.
- *
- * `VALUES` rather than a property path over the whole class tree — the concept list is closed and hand-verified, and a
- * `wdt:P279*` walk from `building` would drag in every structure type on earth.
- */
-export declare function buildDesignatorLabelQuery(concepts?: readonly SubVenueConcept[]): string;
-/**
- * Build the instance-label query — every item that is an `instance of` (through any `subclass of` chain) an airport
- * terminal, with all of its labels. Measured at 246 items / 775 labels on 2026-08-04, well inside WDQS's 60-second
- * budget.
- *
- * A caveat worth knowing before trusting a row: Wikidata's P31 on these is not clean. `Q1322696` (Kigali International
- * Airport) is typed as an airport terminal, so the result set mixes AIRPORTS in with terminals. The consumer filters;
- * this module fetches what the query returns.
- */
-export declare function buildTerminalInstanceQuery(classQID?: string): string;
-/**
- * The SPARQL JSON results shape, narrowed to the two column types these queries produce.
- */
-export interface SPARQLResults {
-    results: {
-        bindings: Array<Record<string, {
-            type: string;
-            value: string;
-            "xml:lang"?: string;
-        }>>;
-    };
-}
-/**
- * Whether a decoded body is a SPARQL results envelope. Used as the cache's write validator so an HTML error page served
- * under a 200 is never persisted for the next run to destructure into `undefined`.
- */
-export declare function isSPARQLResults(value: unknown): value is SPARQLResults;
-export interface CreateWikidataClientOptions {
-    /**
-     * On-disk response-cache root. Defaults to a `http-cache` directory beside the fetch output.
-     */
-    cacheDir: string;
-    /**
-     * Time source powering the pacer and the retry backoff. Defaults to the system clock; tests inject a fake so no suite
-     * ever sleeps a real second.
-     */
-    clock?: ClockLike;
-    /**
-     * Axios overrides, merged over this client's defaults. THE TEST SEAM — pass an `adapter` and no live call is made.
-     * Overriding `headers` wholesale would drop the required `User-Agent`, so don't.
-     */
-    axios?: ConstructorParameters<typeof APIClient>[0]["axios"];
-}
-/**
- * A Wikidata Query Service client: paced, retrying, disk-cached, and correctly identified.
- */
-export declare class WikidataClient extends APIClient {
-    /**
-     * Run one SPARQL query and return its results envelope.
-     */
-    query(sparql: string): Promise<SPARQLResults>;
-}
-/**
- * Construct a {@link WikidataClient} with every default resolved.
- */
-export declare function createWikidataClient(options: CreateWikidataClientOptions): WikidataClient;
-export type FetchWikidataSubVenueOptions = BaseFetchOptions;
-/**
- * Run both queries and write their raw SPARQL JSON into `<outRoot>/wikidata-subvenue/`, with a `MANIFEST.json` carrying
- * the endpoint, the exact queries, the concept table, row counts and sha256s.
- *
- * The RAW envelope is written rather than a reshaped one on purpose: the lexicon build is a separate, pure step
- * (`sub-venue-lexicon.ts`) and keeping the fetch output byte-faithful to what WDQS served means a lexicon regeneration
- * never needs the network.
- */
-export declare function fetchWikidataSubVenue(options: FetchWikidataSubVenueOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=wikidata-subvenue.d.ts.map

package/out/src/tools/fetch/wikidata-subvenue.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"wikidata-subvenue.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/wikidata-subvenue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AAMH,OAAO,EAAE,SAAS,EAAE,KAAK,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAI/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAKnE;;GAEG;AACH,eAAO,MAAM,aAAa,sCAAsC,CAAA;AAEhE;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,+GAC6E,CAAA;AAkC7G;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,eAAe;IAC/B,YAAY,EAAE,MAAM,CAAA;IACpB,GAAG,EAAE,MAAM,CAAA;IACX;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAA;CACb;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,EAAE,SAAS,eAAe,EASvD,CAAA;AAOD;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,GAAE,SAAS,eAAe,EAAsB,GAAG,MAAM,CAU1G;AAED;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,GAAE,MAA2B,GAAG,MAAM,CAMxF;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,OAAO,EAAE;QACR,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC,CAAA;KACrF,CAAA;CACD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,KAAK,CAAC,EAAE,SAAS,CAAA;IACjB;;;OAGG;IACH,KAAK,CAAC,EAAE,qBAAqB,CAAC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;CAC3D;AAED;;GAEG;AACH,qBAAa,cAAe,SAAQ,SAAS;IAC5C;;OAEG;IACU,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;CAQ1D;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,2BAA2B,GAAG,cAAc,CAkCzF;AAED,MAAM,MAAM,4BAA4B,GAAG,gBAAgB,CAAA;AA0C3D;;;;;;;GAOG;AACH,wBAAsB,qBAAqB,CAC1C,OAAO,EAAE,4BAA4B,EACrC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA+CvB"}