@mailwoman/corpus 9.2.0 → 9.3.0

package/out/src/tools/fetch/ourairports.js

@@ -1,128 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the OurAirports CSV dumps — the VENUE side of the sub-venue corpus arc (#35).
- *
- *   Source : https://davidmegginson.github.io/ourairports-data/ (the project's own GitHub Pages
- *            mirror of the nightly export; `ourairports.com/data/` redirects here).
- *   License: PUBLIC DOMAIN. OurAirports places its data in the public domain and asks only for a
- *            courtesy credit — no attribution obligation rides on a derived shard, which makes this
- *            the one transport source in the arc with no licensing question at all. Tier A.
- *
- *   ## What it is good for, and what it is not
- *
- *   `airports.csv` is every airport on earth with ICAO/IATA codes, coordinates, `municipality`, and
- *   `iso_country` — 12.7 MB, ~83,000 rows as of 2026-08-04. That is the CONTAINING VENUE for a
- *   `<sub-venue>, <venue>, <street>, <locality>, <postcode>` corpus line, and it is better at that job
- *   than OSM: every row is named, the name is canonical, and `municipality` gives the locality without
- *   a spatial join.
- *
- *   It carries NO interior structure. There is no terminal, concourse, gate or pier table — the
- *   corpus task says as much ("Good for the venue side of each pair, weaker on interior structure")
- *   and a row-level read confirms it. Pair it with the OSM `aeroway` extractor
- *   (`@mailwoman/osm/sdk`'s `extractOSMSubVenues`), which is where the sub-venue half comes from.
- *
- *   ## Why `downloadToFile` and not `APIClient`
- *
- *   `AGENTS.md` routes HTTP through `APIClient`, and that rule is about API REQUESTS — small bodies,
- *   repeated calls, rate-limited hosts. This is four static file transfers against a GitHub Pages CDN
- *   with no rate limit and nothing to pace, run once per refresh. It uses the same `downloadToFile`
- *   every other module in this `fetch/` family uses, which is where the retry and timeout live.
- *   The Wikidata sibling (`wikidata-subvenue.ts`) IS an API client and is built on `APIClient`
- *   accordingly; the split between the two is the one `AGENTS.md` draws.
- *
- *   Invoke via `mailwoman corpus fetch ourairports --out-root <path>`.
- */
-import { mkdirSync } from "node:fs";
-import { join } from "node:path";
-import { APIClient } from "@mailwoman/core/api";
-import { sha256File } from "@mailwoman/core/utils";
-import { downloadToFile, writeManifest } from "./download.js";
-const SLUG = "ourairports";
-/**
- * The GitHub Pages mirror the project itself publishes. `ourairports.com/data/*.csv` 302s here, so pointing at the
- * mirror directly saves a redirect and is the URL the project's own README gives.
- */
-const BASE_URL = "https://davidmegginson.github.io/ourairports-data";
-/**
- * The files worth having, and why each one.
- *
- * `airports.csv` is the payload. The other three are small joins that turn its codes into text: `countries.csv` and
- * `regions.csv` expand `iso_country`/`iso_region` into names (a corpus line needs "Germany", not "DE"), and
- * `runways.csv` is the only file carrying per-airport sub-structure of any kind — runway designators, which are NOT
- * sub-venue designators (nobody addresses mail to a runway) but are worth having on disk as the negative class if the
- * shard ever needs one.
- */
-const FILES = ["airports.csv", "countries.csv", "regions.csv", "runways.csv"];
-/**
- * Read the upstream `Last-Modified` with a HEAD. Returns `null` on any failure — provenance metadata is nice to have
- * and must never fail a download that otherwise succeeded.
- */
-async function readLastModified(url) {
-    try {
-        const res = await new APIClient({ displayName: "ourairports", retry: true }).fetch({
-            url,
-            method: "head",
-            timeout: 30_000,
-        });
-        return res.headers["last-modified"] ?? null;
-    }
-    catch {
-        return null;
-    }
-}
-/**
- * Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `MANIFEST.json` carrying each file's
- * origin URL, sha256, byte count and upstream `Last-Modified`.
- */
-export async function fetchOurAirports(options, report) {
-    const destDir = join(options.outRoot, SLUG);
-    mkdirSync(destDir, { recursive: true });
-    const entries = [];
-    const failedCodes = [];
-    let fetched = 0;
-    let failed = 0;
-    for (const filename of FILES) {
-        const url = `${BASE_URL}/${filename}`;
-        const dest = join(destDir, filename);
-        report?.(`=== ${SLUG} / ${filename}`);
-        try {
-            const [{ bytes }, lastModified] = await Promise.all([
-                downloadToFile({
-                    url,
-                    dest,
-                    timeoutMs: 600_000,
-                    retries: 2,
-                    headers: { "Accept-Encoding": "gzip, br" },
-                    report,
-                }),
-                readLastModified(url),
-            ]);
-            entries.push({
-                filename,
-                source_url: url,
-                sha256: await sha256File(dest),
-                bytes,
-                last_modified: lastModified,
-            });
-            fetched++;
-        }
-        catch (error) {
-            report?.(`✗ ${filename}: ${error instanceof Error ? error.message : String(error)}`);
-            failedCodes.push(filename);
-            failed++;
-        }
-    }
-    const manifest = {
-        source: "OurAirports",
-        base_url: BASE_URL,
-        license: "public domain (courtesy credit requested)",
-        downloaded_at: new Date().toISOString(),
-        files: entries,
-    };
-    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
-    return { fetched, skipped: 0, failed, failedCodes };
-}
-//# sourceMappingURL=ourairports.js.map

package/out/src/tools/fetch/ourairports.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ourairports.js","sourceRoot":"","sources":["../../../../src/tools/fetch/ourairports.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE7D,MAAM,IAAI,GAAG,aAAa,CAAA;AAE1B;;;GAGG;AACH,MAAM,QAAQ,GAAG,mDAAmD,CAAA;AAEpE;;;;;;;;GAQG;AACH,MAAM,KAAK,GAAG,CAAC,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,aAAa,CAAU,CAAA;AAyBtF;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,GAAW;IAC1C,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,MAAM,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;YAClF,GAAG;YACH,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,MAAM;SACf,CAAC,CAAA;QAEF,OAAQ,GAAG,CAAC,OAAO,CAAC,eAAe,CAAwB,IAAI,IAAI,CAAA;IACpE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,OAAgC,EAChC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAA;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,CAAC;YACJ,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACnD,cAAc,CAAC;oBACd,GAAG;oBACH,IAAI;oBACJ,SAAS,EAAE,OAAO;oBAClB,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;oBAC1C,MAAM;iBACN,CAAC;gBACF,gBAAgB,CAAC,GAAG,CAAC;aACrB,CAAC,CAAA;YAEF,OAAO,CAAC,IAAI,CAAC;gBACZ,QAAQ;gBACR,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC;gBAC9B,KAAK;gBACL,aAAa,EAAE,YAAY;aAC3B,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,KAAK,QAAQ,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACpF,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAE1B,MAAM,EAAE,CAAA;QACT,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAwB;QACrC,MAAM,EAAE,aAAa;QACrB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,2CAA2C;QACpD,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/src/tools/fetch/ppd.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"ppd.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/ppd.ts"],"names":[],"mappings":"AA2BA,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;CACzB;AAID;;;;GAIG;AACH,wBAAsB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EACnD,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC3B,OAAO,CAAC,eAAe,CAAC,CAwD1B"}

package/out/src/tools/fetch/ppd.js

@@ -1,100 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   HM Land Registry Price Paid Data → OA-shaped GB tuples CSV for the `locale` shard recipe.
- *
- *   PPD is E&W-only, ALL-CAPS, and column-structured (no header row). We emit the exact OA header
- *   `readTuples` (`shard-recipes/locale.ts`) indexes by name, mapped for `districtAsLocality: true`:
- *   CITY = PPD locality (dependent locality; blank when it merely repeats the town — 1995-era rows
- *   pad locality=town), DISTRICT = PPD post town, REGION = county. SAON (flat/unit) rows and
- *   building-name PAONs are skipped in v1 and counted — `LocaleBaseTuple` has no unit field yet.
- *
- *   Snapshot provenance: `$MAILWOMAN_DATA_ROOT/ppd/<date>/pp-complete.csv` (md5 sibling).
- *   License: OGL v3 (attribution: HM Land Registry).
- */
-import { createReadStream, createWriteStream } from "node:fs";
-import { parseArgs } from "node:util";
-import { runIfScript } from "@mailwoman/core/scripting";
-import { dataRootPath } from "@mailwoman/core/utils";
-import { CSVSpliterator } from "spliterator";
-import { titleCaseGB } from "../gb-title-case.js";
-const HOUSE_NUMBER_PATTERN = /^\d+[A-Za-z]?(\s*-\s*\d+[A-Za-z]?)?$/;
-const quote = (value) => (value ? `"${value.replaceAll('"', '""')}"` : "");
-/**
- * Convert PPD rows (`id,price,date,postcode,type,new,tenure,PAON,SAON,street,locality,town,district,county,cat,status`
- * — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing. Accepts a plain array
- * of rows (tests) or a streamed async source (the real 31M-row extraction).
- */
-export async function extractPPDTuples(input, write) {
-    const stats = { kept: 0, skippedSAON: 0, skippedPAON: 0, skippedNoStreet: 0, skippedNoPostcode: 0 };
-    write("NUMBER,STREET,CITY,DISTRICT,REGION,POSTCODE");
-    for await (const cells of input) {
-        // HM Land Registry Price Paid Data column order (see the PPD FAQ for the full field list).
-        const postcode = cells[3];
-        const paon = cells[7];
-        const saon = cells[8];
-        const street = cells[9];
-        const locality = cells[10];
-        const town = cells[11];
-        const county = cells[13];
-        if (saon) {
-            stats.skippedSAON++;
-            continue;
-        }
-        if (!paon || !HOUSE_NUMBER_PATTERN.test(paon)) {
-            stats.skippedPAON++;
-            continue;
-        }
-        if (!street) {
-            stats.skippedNoStreet++;
-            continue;
-        }
-        if (!postcode) {
-            stats.skippedNoPostcode++;
-            continue;
-        }
-        const number = paon.replace(/\s*-\s*/, "-");
-        const city = locality && locality !== town ? titleCaseGB(locality) : "";
-        write([
-            number,
-            quote(titleCaseGB(street)),
-            quote(city),
-            quote(titleCaseGB(town ?? "")),
-            quote(titleCaseGB(county ?? "")),
-            postcode,
-        ].join(","));
-        stats.kept++;
-    }
-    return stats;
-}
-/**
- * Stream `inputPath` (PPD CSV) → `outputPath` (OA-shaped tuples CSV), returning the row-count stats.
- */
-async function runPPDExtract(inputPath, outputPath) {
-    // No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself (see readTuples in
-    // shard-recipes/locale.ts). `header: false` yields every row as data — PPD ships no header row.
-    const rows = CSVSpliterator.fromAsync(createReadStream(inputPath), {
-        mode: "array",
-        header: false,
-        enableQuoteHandling: true,
-    });
-    const out = createWriteStream(outputPath, { encoding: "utf8" });
-    const stats = await extractPPDTuples(rows, (line) => out.write(line + "\n"));
-    await new Promise((res) => {
-        out.end(res);
-    });
-    return stats;
-}
-runIfScript(import.meta, async () => {
-    const { values } = parseArgs({
-        options: {
-            input: { type: "string", default: dataRootPath("ppd", "2026-07-22", "pp-complete.csv") },
-            output: { type: "string", default: dataRootPath("ppd", "2026-07-22", "gb-tuples.csv") },
-        },
-    });
-    const stats = await runPPDExtract(values.input, values.output);
-    console.log(`[ppd] ${JSON.stringify(stats)}`);
-});
-//# sourceMappingURL=ppd.js.map

package/out/src/tools/fetch/ppd.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ppd.js","sourceRoot":"","sources":["../../../../src/tools/fetch/ppd.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AAEjD,MAAM,oBAAoB,GAAG,sCAAsC,CAAA;AAUnE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;AAE1F;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,KAAmD,EACnD,KAA6B;IAE7B,MAAM,KAAK,GAAoB,EAAE,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACpH,KAAK,CAAC,6CAA6C,CAAC,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QACjC,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACzB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QACtB,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAExB,IAAI,IAAI,EAAE,CAAC;YACV,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,KAAK,CAAC,eAAe,EAAE,CAAA;YAEvB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,KAAK,CAAC,iBAAiB,EAAE,CAAA;YAEzB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAEvE,KAAK,CACJ;YACC,MAAM;YACN,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC;YACX,KAAK,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ;SACR,CAAC,IAAI,CAAC,GAAG,CAAC,CACX,CAAA;QAED,KAAK,CAAC,IAAI,EAAE,CAAA;IACb,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAAC,SAAiB,EAAE,UAAkB;IACjE,gGAAgG;IAChG,gGAAgG;IAChG,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAW,gBAAgB,CAAC,SAAS,CAAC,EAAE;QAC5E,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,KAAK;QACb,mBAAmB,EAAE,IAAI;KACzB,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,iBAAiB,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC/D,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAA;IAE5E,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACb,CAAC,CAAC,CAAA;IAEF,OAAO,KAAK,CAAA;AACb,CAAC;AAED,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE;YACR,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,iBAAiB,CAAC,EAAE;YACxF,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,eAAe,CAAC,EAAE;SACvF;KACD,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,KAAM,EAAE,MAAM,CAAC,MAAO,CAAC,CAAA;IAEhE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC,CAAC,CAAA"}

package/out/src/tools/fetch/state-hi-schools.d.ts

@@ -1,25 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the Hawaii State DOE school directory and convert the XLSX workbook to a flat CSV the
- *   `state-hi-schools` adapter can consume.
- *
- *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
- *   (~38 public charter schools). Both sheets share the same header. This module concatenates them
- *   under one shared header so the adapter can stream a single CSV.
- *
- *   License: Hawaii state government open data (Tier A — state PD-equivalent).
- *
- *   Built-in `fetch` (gzip/brotli) replaces curl for the download; the XLSX → CSV step still rides
- *   `python3` + `openpyxl` via `node:child_process` — there is no clean node equivalent without
- *   adding a workbook-parsing dependency.
- *
- *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
- *   CSV exists and sha matches MANIFEST, skips download.
- */
-import type { BaseFetchOptions, FetchSummary } from "./download.ts";
-export type FetchStateHISchoolsOptions = BaseFetchOptions;
-export declare function fetchStateHISchools(options: FetchStateHISchoolsOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=state-hi-schools.d.ts.map

package/out/src/tools/fetch/state-hi-schools.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"state-hi-schools.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAUH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAanE,MAAM,MAAM,0BAA0B,GAAG,gBAAgB,CAAA;AAsEzD,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA2FvB"}

package/out/src/tools/fetch/state-hi-schools.js

@@ -1,162 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the Hawaii State DOE school directory and convert the XLSX workbook to a flat CSV the
- *   `state-hi-schools` adapter can consume.
- *
- *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
- *   (~38 public charter schools). Both sheets share the same header. This module concatenates them
- *   under one shared header so the adapter can stream a single CSV.
- *
- *   License: Hawaii state government open data (Tier A — state PD-equivalent).
- *
- *   Built-in `fetch` (gzip/brotli) replaces curl for the download; the XLSX → CSV step still rides
- *   `python3` + `openpyxl` via `node:child_process` — there is no clean node equivalent without
- *   adding a workbook-parsing dependency.
- *
- *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
- *   CSV exists and sha matches MANIFEST, skips download.
- */
-import { spawn, spawnSync } from "node:child_process";
-import { existsSync, mkdirSync, statSync } from "node:fs";
-import { unlink } from "node:fs/promises";
-import { join } from "node:path";
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/utils";
-import { sha256File } from "@mailwoman/core/utils";
-import { downloadToFile, readManifest, writeManifest } from "./download.js";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx";
-const SLUG = "state-hi-schools";
-const CSV_FILENAME = "HI_Public_Schools_List.csv";
-const XLSX_FILENAME = "HI_Public_Schools_List.xlsx";
-/**
- * The XLSX → CSV converter: concatenate every sheet under one shared header (the first sheet's). Runs as `python3 -c
- * <script> <xlsx-path> <csv-path>`, so `sys.argv[1]`/`sys.argv[2]` are the I/O paths. TODO: Get rid of this.
- */
-const PY_CONVERT = `
-import csv
-import sys
-from openpyxl import load_workbook
-
-xlsx_path, csv_path = sys.argv[1], sys.argv[2]
-wb = load_workbook(xlsx_path, data_only=True, read_only=True)
-
-with open(csv_path, "w", newline="", encoding="utf-8") as out:
-    writer = csv.writer(out)
-    shared_header = None
-    total_data_rows = 0
-    for sheet_name in wb.sheetnames:
-        ws = wb[sheet_name]
-        rows = ws.iter_rows(values_only=True)
-        try:
-            header = next(rows)
-        except StopIteration:
-            continue
-        norm_header = ["" if v is None else str(v).strip() for v in header]
-        if shared_header is None:
-            shared_header = norm_header
-            writer.writerow(shared_header)
-        elif norm_header != shared_header:
-            print(
-                f"  ! sheet '{sheet_name}' header diverges from shared header; concatenating anyway",
-                file=sys.stderr,
-            )
-        for row in rows:
-            if row is None:
-                continue
-            # Skip fully-empty rows (XLSX iter_rows can yield phantom trailing rows).
-            if all(v is None or (isinstance(v, str) and not v.strip()) for v in row):
-                continue
-            writer.writerow(["" if v is None else str(v).strip() for v in row])
-            total_data_rows += 1
-
-print(f"  converted {total_data_rows} data rows from {len(wb.sheetnames)} sheets", file=sys.stderr)
-`;
-/**
- * Run the openpyxl converter. Its stderr narration streams straight through to the process stderr (matching the old
- * `stdio: inherit` behavior) rather than routing through `report` — the python child owns those lines.
- */
-async function convertXLSXToCSV(xlsxPath, csvPath) {
-    const child = spawn("python3", ["-c", PY_CONVERT, xlsxPath, csvPath], {
-        stdio: ["ignore", "inherit", "inherit"],
-    });
-    await new Promise((resolve, reject) => {
-        child.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`python3 converter exited with ${code}`))));
-        child.on("error", reject);
-    });
-}
-export async function fetchStateHISchools(options, report) {
-    const destDir = join(options.outRoot, SLUG);
-    mkdirSync(destDir, { recursive: true });
-    const xlsxDest = join(destDir, XLSX_FILENAME);
-    const csvDest = join(destDir, CSV_FILENAME);
-    const manifestPath = join(destDir, "MANIFEST.json");
-    report?.(`=== ${SLUG}`);
-    // Idempotency: skip if CSV exists and sha matches recorded MANIFEST.
-    if (existsSync(csvDest)) {
-        const recorded = await readManifest(manifestPath);
-        if (recorded?.sha256 && recorded.filename === CSV_FILENAME) {
-            const actualSha = await sha256File(csvDest);
-            if (actualSha === recorded.sha256) {
-                report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`);
-                return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
-            }
-        }
-    }
-    // Preflight: openpyxl must be importable.
-    const preflight = spawnSync("python3", ["-c", "import openpyxl"], { stdio: "ignore" });
-    if (preflight.status !== 0) {
-        report?.(`  ✗ python3 with the \`openpyxl\` package is required to convert the HIDOE XLSX.\n` +
-            `    Debian/Ubuntu:  sudo apt-get install -y python3-openpyxl\n` +
-            `    macOS Homebrew: brew install python && pip3 install openpyxl`);
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    // Download XLSX.
-    report?.(`  Downloading ${SOURCE_URL} ...`);
-    try {
-        await downloadToFile({
-            url: SOURCE_URL,
-            dest: xlsxDest,
-            timeoutMs: 600_000,
-            headers: { "Accept-Encoding": "gzip, br" },
-            report,
-        });
-    }
-    catch (error) {
-        report?.(`  ✗ Download failed (${error.message})`);
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    const xlsxSize = statSync(xlsxDest).size;
-    report?.(`  Downloaded XLSX: ${ByteFormatter.formatIEC(xlsxSize)}`);
-    if (xlsxSize < BYTES_PER_KIB) {
-        report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`);
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    // Convert XLSX → CSV (concatenate both sheets under one shared header).
-    report?.(`  Converting XLSX → CSV (concatenating sheets) ...`);
-    await convertXLSXToCSV(xlsxDest, csvDest);
-    const csvSize = statSync(csvDest).size;
-    const csvSha = await sha256File(csvDest);
-    // Remove XLSX (CSV is the canonical artifact the adapter consumes).
-    await unlink(xlsxDest);
-    report?.(`  Removed XLSX (CSV kept)`);
-    // Write MANIFEST.
-    const manifest = {
-        source_url: SOURCE_URL,
-        downloaded_at: new Date().toISOString(),
-        filename: CSV_FILENAME,
-        sha256: csvSha,
-        bytes: csvSize,
-        notes: "Converted from XLSX (sheets HIDOE + PCS concatenated under shared header).",
-    };
-    await writeManifest(manifestPath, manifest);
-    report?.(`  ✓ ${ByteFormatter.formatIEC(csvSize)}  sha256=${csvSha}`);
-    report?.(`  MANIFEST written to ${manifestPath}`);
-    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
-}
-//# sourceMappingURL=state-hi-schools.js.map

package/out/src/tools/fetch/state-hi-schools.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"state-hi-schools.js","sourceRoot":"","sources":["../../../../src/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAA;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E;;;GAGG;AAEH,MAAM,UAAU,GAAG,iEAAiE,CAAA;AACpF,MAAM,IAAI,GAAG,kBAAkB,CAAA;AAC/B,MAAM,YAAY,GAAG,4BAA4B,CAAA;AACjD,MAAM,aAAa,GAAG,6BAA6B,CAAA;AAInD;;;GAGG;AACH,MAAM,UAAU,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsClB,CAAA;AAWD;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,QAAgB,EAAE,OAAe;IAChE,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE;QACrE,KAAK,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;KACvC,CAAC,CAAA;IAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAClH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,OAAmC,EACnC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAA;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,qEAAqE;IACrE,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAoB,YAAY,CAAC,CAAA;QAEpE,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC5D,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;YAE3C,IAAI,SAAS,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;gBAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;YAC9D,CAAC;QACF,CAAC;IACF,CAAC;IAED,0CAA0C;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAA;IAEtF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,EAAE,CACP,oFAAoF;YACnF,gEAAgE;YAChE,kEAAkE,CACnE,CAAA;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,iBAAiB;IACjB,MAAM,EAAE,CAAC,iBAAiB,UAAU,MAAM,CAAC,CAAA;IAE3C,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC;YACpB,GAAG,EAAE,UAAU;YACf,IAAI,EAAE,QAAQ;YACd,SAAS,EAAE,OAAO;YAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;YAC1C,MAAM;SACN,CAAC,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,wBAAyB,KAAe,CAAC,OAAO,GAAG,CAAC,CAAA;QAE7D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAA;IACxC,MAAM,EAAE,CAAC,sBAAsB,aAAa,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;IAEnE,IAAI,QAAQ,GAAG,aAAa,EAAE,CAAC;QAC9B,MAAM,EAAE,CAAC,2BAA2B,QAAQ,+BAA+B,CAAC,CAAA;QAE5E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,wEAAwE;IACxE,MAAM,EAAE,CAAC,oDAAoD,CAAC,CAAA;IAC9D,MAAM,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAEzC,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,oEAAoE;IACpE,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAA;IACtB,MAAM,EAAE,CAAC,2BAA2B,CAAC,CAAA;IAErC,kBAAkB;IAClB,MAAM,QAAQ,GAAa;QAC1B,UAAU,EAAE,UAAU;QACtB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,YAAY;QACtB,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,4EAA4E;KACnF,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,YAAY,MAAM,EAAE,CAAC,CAAA;IACrE,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/src/tools/fetch/state-sources.d.ts

@@ -1,26 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
- *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
- *
- *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
- *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
- *   before the adapter can consume it.
- *
- *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
- *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
- *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
- */
-import type { BaseFetchOptions, FetchSummary } from "./download.ts";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-export type FetchStateSourcesOptions = BaseFetchOptions;
-export declare function fetchStateSources(options: FetchStateSourcesOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=state-sources.d.ts.map

package/out/src/tools/fetch/state-sources.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"state-sources.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAGnE;;;GAGG;AAEH,MAAM,MAAM,wBAAwB,GAAG,gBAAgB,CAAA;AAsDvD,wBAAsB,iBAAiB,CACtC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAgEvB"}

package/out/src/tools/fetch/state-sources.js

@@ -1,111 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
- *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
- *
- *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
- *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
- *   before the adapter can consume it.
- *
- *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
- *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
- *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
- */
-import { mkdirSync } from "node:fs";
-import { join } from "node:path";
-import { BYTES_PER_KIB } from "@mailwoman/core/fs/utils";
-import { sha256File } from "@mailwoman/core/utils";
-import { downloadToFile, writeManifest } from "./download.js";
-const SOURCES = [
-    {
-        slug: "state-ny-notaries",
-        filename: "NY_Commissioned_Notaries.csv",
-        url: "https://data.ny.gov/api/views/rwbv-mz6z/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-tx-notaries",
-        filename: "TX_Notary_Public_Commissions.csv",
-        url: "https://data.texas.gov/api/views/gmd3-bnrd/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-de-notaries",
-        filename: "DE_Notaries_Commissioned.csv",
-        url: "https://data.delaware.gov/api/views/q8dr-mj6p/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-or-notaries",
-        filename: "OR_Active_Notaries.csv",
-        url: "https://data.oregon.gov/api/views/j2pk-zk6z/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-ia-contractors",
-        filename: "IA_Active_Construction_Contractor_Registrations.csv",
-        url: "https://data.iowa.gov/api/views/dpf3-iz94/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-wa-health-providers",
-        filename: "WA_Health_Care_Provider_Credential_Data.csv",
-        url: "https://data.wa.gov/api/views/qxh8-f4bd/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-hi-lobbyists",
-        filename: "HI_Lobbyist_Registration_Statements.csv",
-        url: "https://data.hawaii.gov/api/views/cm7c-skav/rows.csv?accessType=DOWNLOAD",
-    },
-];
-export async function fetchStateSources(options, report) {
-    mkdirSync(options.outRoot, { recursive: true });
-    let fetched = 0;
-    let failed = 0;
-    const failedCodes = [];
-    for (const { slug, filename, url } of SOURCES) {
-        const destDir = join(options.outRoot, slug);
-        mkdirSync(destDir, { recursive: true });
-        const dest = join(destDir, filename);
-        report?.(`=== ${slug} / ${filename}`);
-        let bytes;
-        try {
-            ;
-            ({ bytes } = await downloadToFile({
-                url,
-                dest,
-                timeoutMs: 600_000,
-                headers: { "Accept-Encoding": "gzip, br" },
-                report,
-            }));
-        }
-        catch (error) {
-            report?.(`  ✗ download failed for ${url}: ${error.message}`);
-            failed++;
-            failedCodes.push(slug);
-            continue;
-        }
-        if (bytes < BYTES_PER_KIB) {
-            report?.(`  ✗ response too small (${bytes} bytes) — probable 404 / error page`);
-            failed++;
-            failedCodes.push(slug);
-            continue;
-        }
-        const sha = await sha256File(dest);
-        const manifest = {
-            source_url: url,
-            downloaded_at: new Date().toISOString(),
-            filename,
-            sha256: sha,
-            bytes,
-        };
-        await writeManifest(join(destDir, "MANIFEST.json"), manifest);
-        report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
-        fetched++;
-    }
-    report?.(`=== summary ===`);
-    report?.(`fetched: ${fetched}`);
-    report?.(`failed:  ${failed}`);
-    return { fetched, skipped: 0, failed, failedCodes };
-}
-//# sourceMappingURL=state-sources.js.map

package/out/src/tools/fetch/state-sources.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"state-sources.js","sourceRoot":"","sources":["../../../../src/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAe7D,MAAM,OAAO,GAAsB;IAClC;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,kCAAkC;QAC5C,GAAG,EAAE,yEAAyE;KAC9E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,4EAA4E;KACjF;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,wBAAwB;QAClC,GAAG,EAAE,0EAA0E;KAC/E;IACD;QACC,IAAI,EAAE,sBAAsB;QAC5B,QAAQ,EAAE,qDAAqD;QAC/D,GAAG,EAAE,wEAAwE;KAC7E;IACD;QACC,IAAI,EAAE,2BAA2B;QACjC,QAAQ,EAAE,6CAA6C;QACvD,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,oBAAoB;QAC1B,QAAQ,EAAE,yCAAyC;QACnD,GAAG,EAAE,0EAA0E;KAC/E;CACD,CAAA;AAUD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,OAAiC,EACjC,MAA+B;IAE/B,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE/C,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,CAAC;YAAA,CAAC,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;gBAClC,GAAG;gBACH,IAAI;gBACJ,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;gBAC1C,MAAM;aACN,CAAC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,2BAA2B,GAAG,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAEvE,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,2BAA2B,KAAK,qCAAqC,CAAC,CAAA;YAE/E,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAElC,MAAM,QAAQ,GAAmB;YAChC,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ;YACR,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAA;QAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;QAE7D,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAErE,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,YAAY,OAAO,EAAE,CAAC,CAAA;IAC/B,MAAM,EAAE,CAAC,YAAY,MAAM,EAAE,CAAC,CAAA;IAE9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/src/tools/fetch/tiger-full.d.ts

@@ -1,42 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
- *
- *   TIGER ADDRFEAT 2024 source:
- *
- *   - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
- *   - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
- *
- *   Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
- *   `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
- *   already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
- *   this module is download + provenance only.)
- *
- *   Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
- *   county ZIP to disk (no curl subprocess).
- */
-import type { BaseFetchOptions, FetchSummary } from "./download.ts";
-export interface FetchTigerFullOptions extends BaseFetchOptions {
-    /**
-     * Space-separated list of 2-digit state FIPS codes to skip entirely. Default `"50"` — Vermont, already fetched in
-     * v0.1.1.
-     */
-    skipStateFips?: string;
-    /**
-     * Seconds to sleep between downloads. Default `0.2`.
-     */
-    rateSleep?: number;
-    /**
-     * Max concurrent download workers per state. Default `4`.
-     */
-    maxParallel?: number;
-    /**
-     * Print planned downloads without fetching. Default `false`.
-     */
-    dryRun?: boolean;
-}
-export declare function fetchTigerFull(options: FetchTigerFullOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=tiger-full.d.ts.map

package/out/src/tools/fetch/tiger-full.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"tiger-full.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAeH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAmBnE,MAAM,WAAW,qBAAsB,SAAQ,gBAAgB;IAC9D;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AA0GD,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA6KvB"}