@mailwoman/corpus 5.9.0 → 5.10.0

package/out/src/tools/fetch/openaddresses.js

@@ -0,0 +1,307 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch an OpenAddresses country collection from batch.openaddresses.io.
+ *
+ *   Source: https://batch.openaddresses.io
+ *   License: MIXED — OpenAddresses aggregates hundreds of upstream sources with per-source licenses
+ *   (CC-BY, CC0, PDDL, ODbL, CC-BY-SA, and proprietary attribution-only). The per-row LICENSE filter
+ *   in the openaddresses adapter is essential for proprietary-weights training: Tier-C rows (ODbL,
+ *   CC-BY-SA, CC-SA) are dropped at ingest by default. This module downloads the raw collection; the
+ *   adapter does the license gating.
+ *
+ *   Native `fetch` streams the download to disk (no curl + Python subprocess tax);
+ *   `node:child_process` keeps the genuine shell ops it still needs (`file` magic detection +
+ *   `gunzip` decompression, both nice/ionice-deprioritized).
+ *
+ *   ## Authentication note (2026-05-18)
+ *
+ *   The batch.openaddresses.io download endpoint now requires a registered account. Downloads are
+ *   still free at the "basic" tier (GeoJSON+LD output).
+ *
+ *   1. Register at https://batch.openaddresses.io/register
+ *   2. Log in and go to Profile → "Create Token"
+ *   3. Export the token: `export OA_BATCH_TOKEN=<your-token>`
+ *   4. Re-run the command.
+ *
+ *   The collection URL pattern (verified 2026-05-18):
+ *
+ *   - `POST /api/login {username, password}` → `{token}`
+ *   - `GET  /api/job/{job_id}/output/source.geojson.gz?token={token}`
+ *
+ *   Collections are downloaded as a combined GeoJSON.gz via:
+ *
+ *   - `GET  /api/collections/{collection_id}/download` (returns a redirect to S3)
+ *
+ *   Collection IDs discovered from `/api/collections`:
+ *
+ *   - `id=6  name="ca"  size=2044467556` (~1.9 GiB uncompressed, verified 2026-05-18)
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   # With token (preferred). Default country: ca. Supports any OA country code (us-west, fr, …)
+ *   OA_BATCH_TOKEN=<token> mailwoman corpus fetch openaddresses --country ca \
+ *     --out-root /mnt/playpen/mailwoman-data/corpus/sources
+ *
+ *   # Without token (will detect + print instructions, then report the failure):
+ *   mailwoman corpus fetch openaddresses --country ca
+ *   ```
+ */
+import { execFile, spawn } from "node:child_process";
+import { createReadStream, createWriteStream, existsSync, mkdirSync, renameSync, rmSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { setTimeout as sleep } from "node:timers/promises";
+import { promisify } from "node:util";
+import { $private } from "@mailwoman/core/env";
+import { sha256File } from "@mailwoman/core/utils";
+import { isTransientStatus, writeManifest } from "./download.js";
+const execFileAsync = promisify(execFile);
+const OA_BASE = "https://batch.openaddresses.io";
+/**
+ * Collection IDs known as of 2026-05-18 (discovered via `GET /api/collections`). OA assigns stable integer IDs to each
+ * country collection; re-check `GET /api/collections` if a new country is needed and the ID is unknown.
+ */
+const OA_COLLECTION_IDS = {
+    ca: 6,
+    "us-west": 4,
+    "us-south": 3,
+    "us-northeast": 2,
+    "us-midwest": 5,
+    global: 1,
+};
+/** Stream-count newlines, matching `wc -l` (memory-safe for the multi-GB collection). */
+async function countLines(path) {
+    let count = 0;
+    for await (const chunk of createReadStream(path)) {
+        for (let i = 0; i < chunk.length; i++) {
+            if (chunk[i] === 0x0a) {
+                count++;
+            }
+        }
+    }
+    return count;
+}
+function humanBytes(bytes) {
+    const units = ["B", "KiB", "MiB", "GiB", "TiB"];
+    let value = bytes;
+    let unit = 0;
+    while (value >= 1024 && unit < units.length - 1) {
+        value /= 1024;
+        unit++;
+    }
+    return `${value.toFixed(unit === 0 ? 0 : 1)}${units[unit]}`;
+}
+/**
+ * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
+ * redirects (the OA download endpoint 302s to a pre-signed S3 URL).
+ *
+ * NOTE(phase1): kept local instead of the shared `downloadToFile` — this one streams a multi-GB body to disk (the
+ * shared util buffers via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the caller needs for
+ * its two-URL fallback ladder.
+ */
+async function streamDownload(url, dest, opts) {
+    for (let attempt = 0; attempt <= opts.retries; attempt++) {
+        try {
+            const res = await fetch(url, {
+                headers: opts.headers ?? {},
+                redirect: "follow",
+                signal: AbortSignal.timeout(opts.timeoutMs),
+            });
+            if (res.ok && res.body) {
+                await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
+                return res.status;
+            }
+            if (attempt < opts.retries && isTransientStatus(res.status)) {
+                await sleep(opts.retryDelayMs);
+                continue;
+            }
+            return res.status;
+        }
+        catch {
+            if (attempt < opts.retries) {
+                await sleep(opts.retryDelayMs);
+                continue;
+            }
+            return 0;
+        }
+    }
+    return 0;
+}
+/** Decompress `src` → `dest` with the same deprioritized subprocess the old fetcher used. */
+async function gunzipToFile(src, dest) {
+    const child = spawn("nice", ["-n", "15", "ionice", "-c", "3", "gunzip", "-c", src], {
+        stdio: ["ignore", "pipe", "inherit"],
+    });
+    await pipeline(child.stdout, createWriteStream(dest));
+    await new Promise((resolve, reject) => {
+        child.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`gunzip exited with code ${code}`))));
+        child.on("error", reject);
+    });
+}
+export async function fetchOpenAddresses(options, report) {
+    const country = options.country ?? "ca";
+    const token = $private.OA_BATCH_TOKEN;
+    const destDir = join(options.outRoot, "openaddresses", country);
+    const manifestPath = join(destDir, "MANIFEST.json");
+    const outputFile = join(destDir, "collection.geojsonl");
+    const fail = (code) => ({ fetched: 0, skipped: 0, failed: 1, failedCodes: [code] });
+    report?.(`=== fetch openaddresses: country=${country}`);
+    report?.(`    dest: ${destDir}`);
+    mkdirSync(destDir, { recursive: true });
+    // -------------------------------------------------------------------------
+    // Authentication check
+    // -------------------------------------------------------------------------
+    if (!token) {
+        report?.(`
+ERROR: OA_BATCH_TOKEN is not set.
+
+As of 2026-05-18, batch.openaddresses.io requires a registered (free) account
+to download collection files.  Data remains openly licensed — the auth gate
+is there to prevent CDN abuse, not to restrict access.
+
+Steps to get a token:
+  1. Register at: https://batch.openaddresses.io/register
+  2. Verify your email and log in.
+  3. Go to Profile → "Create Token" → copy the token.
+  4. Export it in this shell:
+       export OA_BATCH_TOKEN=<your-token>
+  5. Re-run this command.
+
+The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
+(estimated), so budget ~20–45 minutes at typical cloud-to-host bandwidth.
+`);
+        return fail("OA_BATCH_TOKEN");
+    }
+    // -------------------------------------------------------------------------
+    // Determine collection ID
+    // -------------------------------------------------------------------------
+    let collectionID = OA_COLLECTION_IDS[country];
+    if (collectionID === undefined) {
+        report?.(`Unknown country code '${country}'. Fetching collection list to find ID...`);
+        const res = await fetch(`${OA_BASE}/api/collections`, {
+            headers: { Authorization: `Bearer ${token}`, "Accept-Encoding": "gzip, br" },
+            signal: AbortSignal.timeout(30_000),
+        });
+        if (!res.ok) {
+            report?.(`ERROR: GET /api/collections returned HTTP ${res.status}.`);
+            return fail(country);
+        }
+        const collections = (await res.json());
+        const match = collections.find((item) => item.name === country);
+        if (match?.id === undefined) {
+            report?.(`ERROR: Could not find a collection named '${country}' in GET /api/collections.`);
+            report?.(`Available collections:`);
+            for (const item of collections) {
+                const size = (item.size ?? 0).toLocaleString();
+                report?.(`  ${(item.name ?? "").padEnd(20)}  id=${item.id}  ${item.human ?? ""}  size=${size} bytes`);
+            }
+            return fail(country);
+        }
+        collectionID = match.id;
+        report?.(`  Found collection id=${collectionID} for '${country}'`);
+    }
+    // -------------------------------------------------------------------------
+    // Download via the collections download endpoint (302s to S3)
+    // -------------------------------------------------------------------------
+    report?.(`  Resolving download URL for collection id=${collectionID}...`);
+    report?.(`  Attempting authenticated download...`);
+    const tmpGz = join(destDir, "collection.geojsonl.gz.tmp");
+    const tmpRaw = join(destDir, "collection.geojsonl.tmp");
+    const sourceURL = `${OA_BASE}/api/collections/${collectionID}/download`;
+    let httpStatus = await streamDownload(sourceURL, tmpGz, {
+        headers: { Authorization: `Bearer ${token}` },
+        timeoutMs: 7_200_000,
+        retries: 3,
+        retryDelayMs: 30_000,
+    });
+    if (httpStatus !== 200) {
+        // Try the geojsonl.gz directly with token as query param (alternate URL shape).
+        httpStatus = await streamDownload(`${OA_BASE}/api/collections/${collectionID}/geojsonl.gz?token=${token}`, tmpGz, {
+            timeoutMs: 7_200_000,
+            retries: 3,
+            retryDelayMs: 30_000,
+        });
+    }
+    if (httpStatus !== 200) {
+        rmSync(tmpGz, { force: true });
+        report?.(`
+ERROR: Download returned HTTP ${httpStatus}.
+
+Likely causes:
+  1. OA_BATCH_TOKEN is invalid or expired — re-create it at Profile → Tokens.
+  2. The collection download endpoint URL has changed (this module was written
+     against the 2026-05-18 batch.openaddresses.io API; it may need updating).
+  3. Network error or CDN outage.
+
+Manual download (after logging in to batch.openaddresses.io):
+  - Navigate to https://batch.openaddresses.io/collection/${collectionID}
+  - Click "GeoJSON+LD" to download the collection.
+  - Save as: ${outputFile}
+
+URL tried: ${OA_BASE}/api/collections/${collectionID}/download
+`);
+        return fail(country);
+    }
+    // -------------------------------------------------------------------------
+    // Decompress if the downloaded file is gzipped
+    // -------------------------------------------------------------------------
+    const fileMagic = (await execFileAsync("file", ["--brief", tmpGz]).catch(() => ({ stdout: "" }))).stdout;
+    if (/gzip|compressed/i.test(fileMagic)) {
+        report?.(`  Decompressing gzip archive...`);
+        await gunzipToFile(tmpGz, tmpRaw);
+        rmSync(tmpGz, { force: true });
+        renameSync(tmpRaw, outputFile);
+    }
+    else if (/JSON|ASCII|UTF-8/i.test(fileMagic)) {
+        // Already line-delimited GeoJSON.
+        renameSync(tmpGz, outputFile);
+        rmSync(tmpRaw, { force: true });
+    }
+    else {
+        // Unknown type — keep as-is and let the operator inspect.
+        renameSync(tmpGz, outputFile);
+        report?.(`  WARNING: Downloaded file type is '${fileMagic.trim()}' — may need manual decompression.`);
+    }
+    // -------------------------------------------------------------------------
+    // Verify + write MANIFEST
+    // -------------------------------------------------------------------------
+    if (!existsSync(outputFile)) {
+        report?.(`ERROR: Output file not found at ${outputFile} after download.`);
+        return fail(country);
+    }
+    const size = statSync(outputFile).size;
+    if (size < 10240) {
+        report?.(`ERROR: File is suspiciously small (${size} bytes) — likely an error response.`);
+        return fail(country);
+    }
+    const sha = await sha256File(outputFile);
+    const rowCount = await countLines(outputFile);
+    const downloadedAt = new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
+    const manifest = {
+        source_url: sourceURL,
+        collection_id: collectionID,
+        country,
+        filename: "collection.geojsonl",
+        downloaded_at: downloadedAt,
+        sha256: sha,
+        bytes: size,
+        row_count: rowCount,
+        notes: "batch.openaddresses.io requires a free registered account for downloads. License is mixed per-row; use the openaddresses adapter with allowShareAlike=false (default) to filter Tier-C rows.",
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${humanBytes(size)}  rows=${rowCount}  sha256=${sha}`);
+    report?.(`  MANIFEST written to ${manifestPath}`);
+    report?.(`=== done`);
+    report?.(`Feed to the adapter:`);
+    report?.(`  mailwoman corpus run openaddresses \\`);
+    report?.(`    --input ${outputFile} \\`);
+    report?.(`    --country ${country.toUpperCase()} \\`);
+    report?.(`    --output ${options.outRoot}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=openaddresses.js.map

package/out/src/tools/fetch/openaddresses.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openaddresses.js","sourceRoot":"","sources":["../../../../src/tools/fetch/openaddresses.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAClH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAEhE,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAA;AAEzC,MAAM,OAAO,GAAG,gCAAgC,CAAA;AAEhD;;;GAGG;AACH,MAAM,iBAAiB,GAA2B;IACjD,EAAE,EAAE,CAAC;IACL,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,cAAc,EAAE,CAAC;IACjB,YAAY,EAAE,CAAC;IACf,MAAM,EAAE,CAAC;CACT,CAAA;AAcD,yFAAyF;AACzF,KAAK,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,IAAI,CAA0B,EAAE,CAAC;QAC3E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBACvB,KAAK,EAAE,CAAA;YACR,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAChC,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;IAC/C,IAAI,KAAK,GAAG,KAAK,CAAA;IACjB,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,KAAK,IAAI,IAAI,IAAI,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjD,KAAK,IAAI,IAAI,CAAA;QACb,IAAI,EAAE,CAAA;IACP,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAA;AAC5D,CAAC;AASD;;;;;;;GAOG;AACH,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,IAAwB;IAChF,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3B,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC3C,CAAC,CAAA;YAEF,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAA;gBAEnE,OAAO,GAAG,CAAC,MAAM,CAAA;YAClB,CAAC;YAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAC9B,SAAQ;YACT,CAAC;YAED,OAAO,GAAG,CAAC,MAAM,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAC9B,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,IAAY;IACpD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,EAAE;QACnF,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC;KACpC,CAAC,CAAA;IACF,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAA;IACtD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAC5G,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAkC,EAClC,MAA+B;IAE/B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAA;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAA;IAErC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,CAAC,CAAA;IAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IACnD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAA;IAEvD,MAAM,IAAI,GAAG,CAAC,IAAY,EAAgB,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEzG,MAAM,EAAE,CAAC,oCAAoC,OAAO,EAAE,CAAC,CAAA;IACvD,MAAM,EAAE,CAAC,aAAa,OAAO,EAAE,CAAC,CAAA;IAEhC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,4EAA4E;IAC5E,uBAAuB;IACvB,4EAA4E;IAC5E,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;;;;;;;;;;;;;;;;;CAiBV,CAAC,CAAA;QAEA,OAAO,IAAI,CAAC,gBAAgB,CAAC,CAAA;IAC9B,CAAC;IAED,4EAA4E;IAC5E,0BAA0B;IAC1B,4EAA4E;IAC5E,IAAI,YAAY,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,EAAE,CAAC,yBAAyB,OAAO,2CAA2C,CAAC,CAAA;QACrF,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,kBAAkB,EAAE;YACrD,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;YAC5E,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;SACnC,CAAC,CAAA;QAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,EAAE,CAAC,6CAA6C,GAAG,CAAC,MAAM,GAAG,CAAC,CAAA;YAEpE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;QACrB,CAAC;QAED,MAAM,WAAW,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAmB,CAAA;QACxD,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAA;QAE/D,IAAI,KAAK,EAAE,EAAE,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,EAAE,CAAC,6CAA6C,OAAO,4BAA4B,CAAC,CAAA;YAC1F,MAAM,EAAE,CAAC,wBAAwB,CAAC,CAAA;YAElC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;gBAChC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAA;gBAC9C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,IAAI,EAAE,UAAU,IAAI,QAAQ,CAAC,CAAA;YACtG,CAAC;YAED,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;QACrB,CAAC;QAED,YAAY,GAAG,KAAK,CAAC,EAAE,CAAA;QACvB,MAAM,EAAE,CAAC,yBAAyB,YAAY,SAAS,OAAO,GAAG,CAAC,CAAA;IACnE,CAAC;IAED,4EAA4E;IAC5E,8DAA8D;IAC9D,4EAA4E;IAC5E,MAAM,EAAE,CAAC,8CAA8C,YAAY,KAAK,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,wCAAwC,CAAC,CAAA;IAElD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,4BAA4B,CAAC,CAAA;IACzD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,yBAAyB,CAAC,CAAA;IACvD,MAAM,SAAS,GAAG,GAAG,OAAO,oBAAoB,YAAY,WAAW,CAAA;IAEvE,IAAI,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,KAAK,EAAE;QACvD,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE;QAC7C,SAAS,EAAE,SAAS;QACpB,OAAO,EAAE,CAAC;QACV,YAAY,EAAE,MAAM;KACpB,CAAC,CAAA;IAEF,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QACxB,gFAAgF;QAChF,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,OAAO,oBAAoB,YAAY,sBAAsB,KAAK,EAAE,EAAE,KAAK,EAAE;YACjH,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,CAAC;YACV,YAAY,EAAE,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9B,MAAM,EAAE,CAAC;gCACqB,UAAU;;;;;;;;;4DASkB,YAAY;;eAEzD,UAAU;;aAEZ,OAAO,oBAAoB,YAAY;CACnD,CAAC,CAAA;QAEA,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,4EAA4E;IAC5E,+CAA+C;IAC/C,4EAA4E;IAC5E,MAAM,SAAS,GAAG,CAAC,MAAM,aAAa,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IAExG,IAAI,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,MAAM,EAAE,CAAC,iCAAiC,CAAC,CAAA;QAC3C,MAAM,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACjC,MAAM,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9B,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC/B,CAAC;SAAM,IAAI,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAChD,kCAAkC;QAClC,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAChC,CAAC;SAAM,CAAC;QACP,0DAA0D;QAC1D,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;QAC7B,MAAM,EAAE,CAAC,uCAAuC,SAAS,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAA;IACtG,CAAC;IAED,4EAA4E;IAC5E,0BAA0B;IAC1B,4EAA4E;IAC5E,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7B,MAAM,EAAE,CAAC,mCAAmC,UAAU,kBAAkB,CAAC,CAAA;QAEzE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAA;IAEtC,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;QAClB,MAAM,EAAE,CAAC,sCAAsC,IAAI,qCAAqC,CAAC,CAAA;QAEzF,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAA;IACxC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAA;IAC7C,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IAEvE,MAAM,QAAQ,GAAG;QAChB,UAAU,EAAE,SAAS;QACrB,aAAa,EAAE,YAAY;QAC3B,OAAO;QACP,QAAQ,EAAE,qBAAqB;QAC/B,aAAa,EAAE,YAAY;QAC3B,MAAM,EAAE,GAAG;QACX,KAAK,EAAE,IAAI;QACX,SAAS,EAAE,QAAQ;QACnB,KAAK,EACJ,8LAA8L;KAC/L,CAAA;IACD,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,UAAU,CAAC,IAAI,CAAC,UAAU,QAAQ,YAAY,GAAG,EAAE,CAAC,CAAA;IACpE,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IACjD,MAAM,EAAE,CAAC,UAAU,CAAC,CAAA;IACpB,MAAM,EAAE,CAAC,sBAAsB,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,yCAAyC,CAAC,CAAA;IACnD,MAAM,EAAE,CAAC,eAAe,UAAU,KAAK,CAAC,CAAA;IACxC,MAAM,EAAE,CAAC,iBAAiB,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;IAE3C,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/src/tools/fetch/state-hi-schools.d.ts

@@ -0,0 +1,25 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the Hawaii State DOE school directory and convert the XLSX workbook to a flat CSV the
+ *   `state-hi-schools` adapter can consume.
+ *
+ *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
+ *   (~38 public charter schools). Both sheets share the same header. This module concatenates them
+ *   under one shared header so the adapter can stream a single CSV.
+ *
+ *   License: Hawaii state government open data (Tier A — state PD-equivalent).
+ *
+ *   Built-in `fetch` (gzip/brotli) replaces curl for the download; the XLSX → CSV step still rides
+ *   `python3` + `openpyxl` via `node:child_process` — there is no clean node equivalent without
+ *   adding a workbook-parsing dependency.
+ *
+ *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
+ *   CSV exists and sha matches MANIFEST, skips download.
+ */
+import type { BaseFetchOptions, FetchSummary } from "./download.ts";
+export type FetchStateHISchoolsOptions = BaseFetchOptions;
+export declare function fetchStateHISchools(options: FetchStateHISchoolsOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=state-hi-schools.d.ts.map

package/out/src/tools/fetch/state-hi-schools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"state-hi-schools.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AASH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAQnE,MAAM,MAAM,0BAA0B,GAAG,gBAAgB,CAAA;AAsFzD,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA0FvB"}

package/out/src/tools/fetch/state-hi-schools.js

@@ -0,0 +1,171 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the Hawaii State DOE school directory and convert the XLSX workbook to a flat CSV the
+ *   `state-hi-schools` adapter can consume.
+ *
+ *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
+ *   (~38 public charter schools). Both sheets share the same header. This module concatenates them
+ *   under one shared header so the adapter can stream a single CSV.
+ *
+ *   License: Hawaii state government open data (Tier A — state PD-equivalent).
+ *
+ *   Built-in `fetch` (gzip/brotli) replaces curl for the download; the XLSX → CSV step still rides
+ *   `python3` + `openpyxl` via `node:child_process` — there is no clean node equivalent without
+ *   adding a workbook-parsing dependency.
+ *
+ *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
+ *   CSV exists and sha matches MANIFEST, skips download.
+ */
+import { spawn, spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, statSync } from "node:fs";
+import { unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { sha256File } from "@mailwoman/core/utils";
+import { downloadToFile, readManifest, writeManifest } from "./download.js";
+const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx";
+const SLUG = "state-hi-schools";
+const CSV_FILENAME = "HI_Public_Schools_List.csv";
+const XLSX_FILENAME = "HI_Public_Schools_List.xlsx";
+/**
+ * The XLSX → CSV converter: concatenate every sheet under one shared header (the first sheet's). Runs as `python3 -c
+ * <script> <xlsx-path> <csv-path>`, so `sys.argv[1]`/`sys.argv[2]` are the I/O paths. TODO: Get rid of this.
+ */
+const PY_CONVERT = `
+import csv
+import sys
+from openpyxl import load_workbook
+
+xlsx_path, csv_path = sys.argv[1], sys.argv[2]
+wb = load_workbook(xlsx_path, data_only=True, read_only=True)
+
+with open(csv_path, "w", newline="", encoding="utf-8") as out:
+    writer = csv.writer(out)
+    shared_header = None
+    total_data_rows = 0
+    for sheet_name in wb.sheetnames:
+        ws = wb[sheet_name]
+        rows = ws.iter_rows(values_only=True)
+        try:
+            header = next(rows)
+        except StopIteration:
+            continue
+        norm_header = ["" if v is None else str(v).strip() for v in header]
+        if shared_header is None:
+            shared_header = norm_header
+            writer.writerow(shared_header)
+        elif norm_header != shared_header:
+            print(
+                f"  ! sheet '{sheet_name}' header diverges from shared header; concatenating anyway",
+                file=sys.stderr,
+            )
+        for row in rows:
+            if row is None:
+                continue
+            # Skip fully-empty rows (XLSX iter_rows can yield phantom trailing rows).
+            if all(v is None or (isinstance(v, str) and not v.strip()) for v in row):
+                continue
+            writer.writerow(["" if v is None else str(v).strip() for v in row])
+            total_data_rows += 1
+
+print(f"  converted {total_data_rows} data rows from {len(wb.sheetnames)} sheets", file=sys.stderr)
+`;
+/** Mimic `numfmt --to=iec` for a friendly byte-size log line. */
+function iec(bytes) {
+    if (bytes < 1024)
+        return String(bytes);
+    const units = ["K", "M", "G", "T", "P"];
+    let value = bytes / 1024;
+    let i = 0;
+    while (value >= 1024 && i < units.length - 1) {
+        value /= 1024;
+        i++;
+    }
+    const rounded = value < 10 ? value.toFixed(1) : Math.round(value).toString();
+    return `${rounded}${units[i] ?? ""}`;
+}
+/**
+ * Run the openpyxl converter. Its stderr narration streams straight through to the process stderr (matching the old
+ * `stdio: inherit` behavior) rather than routing through `report` — the python child owns those lines.
+ */
+async function convertXLSXToCSV(xlsxPath, csvPath) {
+    const child = spawn("python3", ["-c", PY_CONVERT, xlsxPath, csvPath], {
+        stdio: ["ignore", "inherit", "inherit"],
+    });
+    await new Promise((resolve, reject) => {
+        child.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`python3 converter exited with ${code}`))));
+        child.on("error", reject);
+    });
+}
+export async function fetchStateHISchools(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    mkdirSync(destDir, { recursive: true });
+    const xlsxDest = join(destDir, XLSX_FILENAME);
+    const csvDest = join(destDir, CSV_FILENAME);
+    const manifestPath = join(destDir, "MANIFEST.json");
+    report?.(`=== ${SLUG}`);
+    // Idempotency: skip if CSV exists and sha matches recorded MANIFEST.
+    if (existsSync(csvDest)) {
+        const recorded = await readManifest(manifestPath);
+        if (recorded?.sha256 && recorded.filename === CSV_FILENAME) {
+            const actualSha = await sha256File(csvDest);
+            if (actualSha === recorded.sha256) {
+                report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`);
+                return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
+            }
+        }
+    }
+    // Preflight: openpyxl must be importable.
+    const preflight = spawnSync("python3", ["-c", "import openpyxl"], { stdio: "ignore" });
+    if (preflight.status !== 0) {
+        report?.(`  ✗ python3 with the \`openpyxl\` package is required to convert the HIDOE XLSX.\n` +
+            `    Debian/Ubuntu:  sudo apt-get install -y python3-openpyxl\n` +
+            `    macOS Homebrew: brew install python && pip3 install openpyxl`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    // Download XLSX.
+    report?.(`  Downloading ${SOURCE_URL} ...`);
+    try {
+        await downloadToFile({
+            url: SOURCE_URL,
+            dest: xlsxDest,
+            timeoutMs: 600_000,
+            headers: { "Accept-Encoding": "gzip, br" },
+            report,
+        });
+    }
+    catch (err) {
+        report?.(`  ✗ Download failed (${err.message})`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    const xlsxSize = statSync(xlsxDest).size;
+    report?.(`  Downloaded XLSX: ${iec(xlsxSize)}`);
+    if (xlsxSize < 1024) {
+        report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    // Convert XLSX → CSV (concatenate both sheets under one shared header).
+    report?.(`  Converting XLSX → CSV (concatenating sheets) ...`);
+    await convertXLSXToCSV(xlsxDest, csvDest);
+    const csvSize = statSync(csvDest).size;
+    const csvSha = await sha256File(csvDest);
+    // Remove XLSX (CSV is the canonical artifact the adapter consumes).
+    await unlink(xlsxDest);
+    report?.(`  Removed XLSX (CSV kept)`);
+    // Write MANIFEST.
+    const manifest = {
+        source_url: SOURCE_URL,
+        downloaded_at: new Date().toISOString(),
+        filename: CSV_FILENAME,
+        sha256: csvSha,
+        bytes: csvSize,
+        notes: "Converted from XLSX (sheets HIDOE + PCS concatenated under shared header).",
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${iec(csvSize)}  sha256=${csvSha}`);
+    report?.(`  MANIFEST written to ${manifestPath}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=state-hi-schools.js.map

package/out/src/tools/fetch/state-hi-schools.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"state-hi-schools.js","sourceRoot":"","sources":["../../../../src/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,UAAU,GAAG,iEAAiE,CAAA;AACpF,MAAM,IAAI,GAAG,kBAAkB,CAAA;AAC/B,MAAM,YAAY,GAAG,4BAA4B,CAAA;AACjD,MAAM,aAAa,GAAG,6BAA6B,CAAA;AAInD;;;GAGG;AACH,MAAM,UAAU,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsClB,CAAA;AAWD,iEAAiE;AACjE,SAAS,GAAG,CAAC,KAAa;IACzB,IAAI,KAAK,GAAG,IAAI;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAA;IACtC,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IACvC,IAAI,KAAK,GAAG,KAAK,GAAG,IAAI,CAAA;IACxB,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,KAAK,IAAI,IAAI,CAAA;QACb,CAAC,EAAE,CAAA;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAA;IAE5E,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAA;AACrC,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,QAAgB,EAAE,OAAe;IAChE,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE;QACrE,KAAK,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;KACvC,CAAC,CAAA;IACF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAClH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,OAAmC,EACnC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAA;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,qEAAqE;IACrE,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAoB,YAAY,CAAC,CAAA;QAEpE,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC5D,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;YAE3C,IAAI,SAAS,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;gBAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;YAC9D,CAAC;QACF,CAAC;IACF,CAAC;IAED,0CAA0C;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAA;IAEtF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,EAAE,CACP,oFAAoF;YACnF,gEAAgE;YAChE,kEAAkE,CACnE,CAAA;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,iBAAiB;IACjB,MAAM,EAAE,CAAC,iBAAiB,UAAU,MAAM,CAAC,CAAA;IAE3C,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC;YACpB,GAAG,EAAE,UAAU;YACf,IAAI,EAAE,QAAQ;YACd,SAAS,EAAE,OAAO;YAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;YAC1C,MAAM;SACN,CAAC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,wBAAyB,GAAa,CAAC,OAAO,GAAG,CAAC,CAAA;QAE3D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAA;IACxC,MAAM,EAAE,CAAC,sBAAsB,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;IAE/C,IAAI,QAAQ,GAAG,IAAI,EAAE,CAAC;QACrB,MAAM,EAAE,CAAC,2BAA2B,QAAQ,+BAA+B,CAAC,CAAA;QAE5E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,wEAAwE;IACxE,MAAM,EAAE,CAAC,oDAAoD,CAAC,CAAA;IAC9D,MAAM,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAEzC,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,oEAAoE;IACpE,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAA;IACtB,MAAM,EAAE,CAAC,2BAA2B,CAAC,CAAA;IAErC,kBAAkB;IAClB,MAAM,QAAQ,GAAa;QAC1B,UAAU,EAAE,UAAU;QACtB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,YAAY;QACtB,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,4EAA4E;KACnF,CAAA;IACD,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC,YAAY,MAAM,EAAE,CAAC,CAAA;IACjD,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/src/tools/fetch/state-sources.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
+ *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
+ *
+ *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
+ *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
+ *   before the adapter can consume it.
+ *
+ *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
+ *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
+ *
+ *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
+ *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
+ */
+import type { BaseFetchOptions, FetchSummary } from "./download.ts";
+export type FetchStateSourcesOptions = BaseFetchOptions;
+export declare function fetchStateSources(options: FetchStateSourcesOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=state-sources.d.ts.map

package/out/src/tools/fetch/state-sources.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"state-sources.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAGnE,MAAM,MAAM,wBAAwB,GAAG,gBAAgB,CAAA;AAsDvD,wBAAsB,iBAAiB,CACtC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAyDvB"}