@mailwoman/corpus 9.2.0 → 9.3.0

package/out/tools/fetch/ppd.js

@@ -0,0 +1,85 @@
+import { dataRootPath } from "@mailwoman/core/data-root";
+import { openReadStream, openWriteStream } from "@mailwoman/core/fs/streams";
+import { runIfScript } from "@mailwoman/core/scripting";
+import { parseArguments } from "@mailwoman/core/scripting/arguments";
+import { resolvePath } from "path-ts";
+import { CSVSpliterator } from "spliterator";
+import { titleCaseGB } from "#tools/gb-title-case";
+const HOUSE_NUMBER_PATTERN = /^\d+[A-Za-z]?(\s*-\s*\d+[A-Za-z]?)?$/;
+const quote = (value) => (value ? `"${value.replaceAll('"', '""')}"` : "");
+/**
+ * Convert PPD rows (`id,price,date,postcode,type,new,tenure,PAON,SAON,street,locality,town,district,county,cat,status`
+ * — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing. Accepts a plain array
+ * of rows (tests) or a streamed async source (the real 31M-row extraction).
+ */
+export async function extractPPDTuples(input, write) {
+    const stats = { kept: 0, skippedSAON: 0, skippedPAON: 0, skippedNoStreet: 0, skippedNoPostcode: 0 };
+    write("NUMBER,STREET,CITY,DISTRICT,REGION,POSTCODE");
+    for await (const cells of input) {
+        // HM Land Registry Price Paid Data column order (see the PPD FAQ for the full field list).
+        const postcode = cells[3];
+        const paon = cells[7];
+        const saon = cells[8];
+        const street = cells[9];
+        const locality = cells[10];
+        const town = cells[11];
+        const county = cells[13];
+        if (saon) {
+            stats.skippedSAON++;
+            continue;
+        }
+        if (!paon || !HOUSE_NUMBER_PATTERN.test(paon)) {
+            stats.skippedPAON++;
+            continue;
+        }
+        if (!street) {
+            stats.skippedNoStreet++;
+            continue;
+        }
+        if (!postcode) {
+            stats.skippedNoPostcode++;
+            continue;
+        }
+        const number = paon.replace(/\s*-\s*/, "-");
+        const city = locality && locality !== town ? titleCaseGB(locality) : "";
+        write([
+            number,
+            quote(titleCaseGB(street)),
+            quote(city),
+            quote(titleCaseGB(town ?? "")),
+            quote(titleCaseGB(county ?? "")),
+            postcode,
+        ].join(","));
+        stats.kept++;
+    }
+    return stats;
+}
+/**
+ * Stream `inputPath` (PPD CSV) → `outputPath` (OA-shaped tuples CSV), returning the row-count stats.
+ */
+async function runPPDExtract(inputPath, outputPath) {
+    // No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself (see readTuples in
+    // recipes/locale.ts). `header: false` yields every row as data — PPD ships no header row.
+    const rows = CSVSpliterator.fromAsync(openReadStream(inputPath), {
+        mode: "array",
+        header: false,
+        enableQuoteHandling: true,
+    });
+    const out = openWriteStream(outputPath, { encoding: "utf8" });
+    const stats = await extractPPDTuples(rows, (line) => out.write(line + "\n"));
+    await new Promise((res) => {
+        out.end(res);
+    });
+    return stats;
+}
+runIfScript(import.meta, async () => {
+    const { values } = parseArguments({
+        options: {
+            input: { type: "string", default: resolvePath(dataRootPath("ppd", "2026-07-22", "pp-complete.csv")) },
+            output: { type: "string", default: resolvePath(dataRootPath("ppd", "2026-07-22", "gb-tuples.csv")) },
+        },
+    });
+    const stats = await runPPDExtract(values.input, values.output);
+    console.log(`[ppd] ${JSON.stringify(stats)}`);
+});
+//# sourceMappingURL=ppd.js.map

package/out/tools/fetch/ppd.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ppd.js","sourceRoot":"","sources":["../../../lib/tools/fetch/ppd.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAA;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAElD,MAAM,oBAAoB,GAAG,sCAAsC,CAAA;AAUnE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;AAE1F;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,KAAmD,EACnD,KAA6B;IAE7B,MAAM,KAAK,GAAoB,EAAE,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACpH,KAAK,CAAC,6CAA6C,CAAC,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QACjC,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACzB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QACtB,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAExB,IAAI,IAAI,EAAE,CAAC;YACV,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,KAAK,CAAC,eAAe,EAAE,CAAA;YAEvB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,KAAK,CAAC,iBAAiB,EAAE,CAAA;YAEzB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAEvE,KAAK,CACJ;YACC,MAAM;YACN,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC;YACX,KAAK,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ;SACR,CAAC,IAAI,CAAC,GAAG,CAAC,CACX,CAAA;QAED,KAAK,CAAC,IAAI,EAAE,CAAA;IACb,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAAC,SAAiB,EAAE,UAAkB;IACjE,gGAAgG;IAChG,0FAA0F;IAC1F,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAW,cAAc,CAAC,SAAS,CAAC,EAAE;QAC1E,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,KAAK;QACb,mBAAmB,EAAE,IAAI;KACzB,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,eAAe,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC7D,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAA;IAE5E,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACb,CAAC,CAAC,CAAA;IAEF,OAAO,KAAK,CAAA;AACb,CAAC;AAED,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,EAAE,GAAG,cAAc,CAAC;QACjC,OAAO,EAAE;YACR,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC,EAAE;YACrG,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC,EAAE;SACpG;KACD,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,KAAM,EAAE,MAAM,CAAC,MAAO,CAAC,CAAA;IAEhE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC,CAAC,CAAA"}

package/out/tools/fetch/state-hi-schools.d.ts

@@ -0,0 +1,26 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the Hawaii State DOE school directory as its original XLSX workbook. The
+ *   `state-hi-schools` adapter reads both worksheets directly.
+ *
+ *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
+ *   (~38 public charter schools). Both sheets are validated against the adapter's required columns
+ *   before the workbook is recorded as fetched.
+ *
+ *   License: Hawaii state government open data (Tier A — state PD-equivalent).
+ *
+ *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
+ *   workbook exists and its sha matches MANIFEST, skips download.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export interface FetchStateHISchoolsOptions extends BaseFetchOptions {
+    /**
+     * Workbook URL. Defaults to the Hawaii DOE source; overridable for an isolated fetch test.
+     */
+    sourceURL?: string;
+}
+export declare function fetchStateHISchools(options: FetchStateHISchoolsOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=state-hi-schools.d.ts.map

package/out/tools/fetch/state-hi-schools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"state-hi-schools.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAUH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAO3E,MAAM,WAAW,0BAA2B,SAAQ,gBAAgB;IACnE;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAqCD,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAkFvB"}

package/out/tools/fetch/state-hi-schools.js

@@ -0,0 +1,112 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the Hawaii State DOE school directory as its original XLSX workbook. The
+ *   `state-hi-schools` adapter reads both worksheets directly.
+ *
+ *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
+ *   (~38 public charter schools). Both sheets are validated against the adapter's required columns
+ *   before the workbook is recorded as fetched.
+ *
+ *   License: Hawaii state government open data (Tier A — state PD-equivalent).
+ *
+ *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
+ *   workbook exists and its sha matches MANIFEST, skips download.
+ */
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { statPath, pathExists } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { XLSXSpliterator } from "spliterator";
+import { STATE_HI_SCHOOL_REQUIRED_COLUMNS, STATE_HI_SCHOOL_SHEETS } from "#adapters/state-hi-schools/workbook";
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download";
+const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx";
+const SLUG = "state-hi-schools";
+const XLSX_FILENAME = "HI_Public_Schools_List.xlsx";
+async function validateWorkbook(path) {
+    for (const sheet of STATE_HI_SCHOOL_SHEETS) {
+        const rows = await XLSXSpliterator.fromAsync(path, {
+            sheet,
+            header: false,
+            take: 2,
+        }).toArray();
+        const header = rows[0];
+        if (!header || rows.length < 2) {
+            throw new Error(`sheet ${sheet} must contain a header and at least one school row`);
+        }
+        // Exact casing is part of the adapter contract: object mode preserves the workbook's header names, so accepting
+        // `Code` here while the reader asks for `record.code` would validate a workbook whose every row is later skipped.
+        const columns = new Set(header.map((cell) => String(cell ?? "").trim()));
+        const missing = STATE_HI_SCHOOL_REQUIRED_COLUMNS.filter((column) => !columns.has(column));
+        if (missing.length) {
+            throw new Error(`sheet ${sheet} is missing required columns: ${missing.join(", ")}`);
+        }
+    }
+}
+export async function fetchStateHISchools(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const xlsxDest = join(destDir, XLSX_FILENAME);
+    const manifestPath = join(destDir, "MANIFEST.json");
+    const sourceURL = options.sourceURL ?? SOURCE_URL;
+    report?.(`=== ${SLUG}`);
+    // Idempotency: skip if the source workbook exists and sha matches the recorded manifest.
+    if (await pathExists(xlsxDest)) {
+        const recorded = await readManifest(manifestPath);
+        if (recorded?.sha256 && recorded.filename === XLSX_FILENAME) {
+            const actualSha = await sha256File(xlsxDest);
+            if (actualSha === recorded.sha256) {
+                report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`);
+                return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
+            }
+        }
+    }
+    // Download XLSX.
+    report?.(`  Downloading ${sourceURL} ...`);
+    try {
+        await downloadToFile({
+            url: sourceURL,
+            dest: xlsxDest,
+            timeoutMs: 600_000,
+            headers: { "Accept-Encoding": "gzip, br" },
+            report,
+        });
+    }
+    catch (error) {
+        report?.(`  ✗ Download failed (${error.message})`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    const xlsxSize = (await statPath(xlsxDest)).size;
+    report?.(`  Downloaded XLSX: ${ByteFormatter.formatIEC(xlsxSize)}`);
+    if (xlsxSize < BYTES_PER_KIB) {
+        report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`);
+        await removePathIfPresent(xlsxDest);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    try {
+        await validateWorkbook(xlsxDest);
+    }
+    catch (error) {
+        report?.(`  ✗ Workbook validation failed (${error.message})`);
+        await removePathIfPresent(xlsxDest);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    const xlsxSha = await sha256File(xlsxDest);
+    // Write MANIFEST.
+    const manifest = {
+        source_url: sourceURL,
+        downloaded_at: new Date().toISOString(),
+        filename: XLSX_FILENAME,
+        sha256: xlsxSha,
+        bytes: xlsxSize,
+        notes: "Original XLSX; the adapter reads the HIDOE and PCS sheets directly.",
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${STATE_HI_SCHOOL_SHEETS.join(" + ")} validated · ${ByteFormatter.formatIEC(xlsxSize)}  sha256=${xlsxSha}`);
+    report?.(`  MANIFEST written to ${manifestPath}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=state-hi-schools.js.map

package/out/tools/fetch/state-hi-schools.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"state-hi-schools.js","sourceRoot":"","sources":["../../../lib/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC5E,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAsB,MAAM,aAAa,CAAA;AAEjE,OAAO,EAAE,gCAAgC,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAA;AAE9G,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEnF,MAAM,UAAU,GAAG,iEAAiE,CAAA;AACpF,MAAM,IAAI,GAAG,kBAAkB,CAAA;AAC/B,MAAM,aAAa,GAAG,6BAA6B,CAAA;AAkBnD,KAAK,UAAU,gBAAgB,CAAC,IAAY;IAC3C,KAAK,MAAM,KAAK,IAAI,sBAAsB,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,SAAS,CAAkB,IAAI,EAAE;YACnE,KAAK;YACL,MAAM,EAAE,KAAK;YACb,IAAI,EAAE,CAAC;SACP,CAAC,CAAC,OAAO,EAAE,CAAA;QAEZ,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QAEtB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,oDAAoD,CAAC,CAAA;QACpF,CAAC;QAED,gHAAgH;QAChH,kHAAkH;QAClH,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;QAExE,MAAM,OAAO,GAAG,gCAAgC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;QAEzF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,iCAAiC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACrF,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,OAAmC,EACnC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAA;IAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IACnD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,UAAU,CAAA;IAEjD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,yFAAyF;IACzF,IAAI,MAAM,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAoB,YAAY,CAAC,CAAA;QAEpE,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;YAC7D,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAA;YAE5C,IAAI,SAAS,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;gBAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;YAC9D,CAAC;QACF,CAAC;IACF,CAAC;IAED,iBAAiB;IACjB,MAAM,EAAE,CAAC,iBAAiB,SAAS,MAAM,CAAC,CAAA;IAE1C,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC;YACpB,GAAG,EAAE,SAAS;YACd,IAAI,EAAE,QAAQ;YACd,SAAS,EAAE,OAAO;YAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;YAC1C,MAAM;SACN,CAAC,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,wBAAyB,KAAe,CAAC,OAAO,GAAG,CAAC,CAAA;QAE7D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAA;IAChD,MAAM,EAAE,CAAC,sBAAsB,aAAa,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;IAEnE,IAAI,QAAQ,GAAG,aAAa,EAAE,CAAC;QAC9B,MAAM,EAAE,CAAC,2BAA2B,QAAQ,+BAA+B,CAAC,CAAA;QAC5E,MAAM,mBAAmB,CAAC,QAAQ,CAAC,CAAA;QAEnC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,mCAAoC,KAAe,CAAC,OAAO,GAAG,CAAC,CAAA;QACxE,MAAM,mBAAmB,CAAC,QAAQ,CAAC,CAAA;QAEnC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAA;IAE1C,kBAAkB;IAClB,MAAM,QAAQ,GAAa;QAC1B,UAAU,EAAE,SAAS;QACrB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,aAAa;QACvB,MAAM,EAAE,OAAO;QACf,KAAK,EAAE,QAAQ;QACf,KAAK,EAAE,qEAAqE;KAC5E,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CACP,OAAO,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,aAAa,CAAC,SAAS,CAAC,QAAQ,CAAC,YAAY,OAAO,EAAE,CAC/G,CAAA;IAED,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/tools/fetch/state-sources.d.ts

@@ -0,0 +1,26 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
+ *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
+ *
+ *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
+ *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
+ *   before the adapter can consume it.
+ *
+ *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
+ *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
+ *
+ *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
+ *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+export type FetchStateSourcesOptions = BaseFetchOptions;
+export declare function fetchStateSources(options: FetchStateSourcesOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=state-sources.d.ts.map

package/out/tools/fetch/state-sources.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"state-sources.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAkB,MAAM,uBAAuB,CAAA;AAG3F;;;GAGG;AAEH,MAAM,MAAM,wBAAwB,GAAG,gBAAgB,CAAA;AA8CvD,wBAAsB,iBAAiB,CACtC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAgEvB"}

package/out/tools/fetch/state-sources.js

@@ -0,0 +1,111 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
+ *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
+ *
+ *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
+ *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
+ *   before the adapter can consume it.
+ *
+ *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
+ *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
+ *
+ *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
+ *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
+ */
+import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters";
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { downloadToFile, writeManifest } from "#tools/fetch/download";
+const SOURCES = [
+    {
+        slug: "state-ny-notaries",
+        filename: "NY_Commissioned_Notaries.csv",
+        url: "https://data.ny.gov/api/views/rwbv-mz6z/rows.csv?accessType=DOWNLOAD",
+    },
+    {
+        slug: "state-tx-notaries",
+        filename: "TX_Notary_Public_Commissions.csv",
+        url: "https://data.texas.gov/api/views/gmd3-bnrd/rows.csv?accessType=DOWNLOAD",
+    },
+    {
+        slug: "state-de-notaries",
+        filename: "DE_Notaries_Commissioned.csv",
+        url: "https://data.delaware.gov/api/views/q8dr-mj6p/rows.csv?accessType=DOWNLOAD",
+    },
+    {
+        slug: "state-or-notaries",
+        filename: "OR_Active_Notaries.csv",
+        url: "https://data.oregon.gov/api/views/j2pk-zk6z/rows.csv?accessType=DOWNLOAD",
+    },
+    {
+        slug: "state-ia-contractors",
+        filename: "IA_Active_Construction_Contractor_Registrations.csv",
+        url: "https://data.iowa.gov/api/views/dpf3-iz94/rows.csv?accessType=DOWNLOAD",
+    },
+    {
+        slug: "state-wa-health-providers",
+        filename: "WA_Health_Care_Provider_Credential_Data.csv",
+        url: "https://data.wa.gov/api/views/qxh8-f4bd/rows.csv?accessType=DOWNLOAD",
+    },
+    {
+        slug: "state-hi-lobbyists",
+        filename: "HI_Lobbyist_Registration_Statements.csv",
+        url: "https://data.hawaii.gov/api/views/cm7c-skav/rows.csv?accessType=DOWNLOAD",
+    },
+];
+export async function fetchStateSources(options, report) {
+    await makeDirectories(options.outRoot);
+    let fetched = 0;
+    let failed = 0;
+    const failedCodes = [];
+    for (const { slug, filename, url } of SOURCES) {
+        const destDir = join(options.outRoot, slug);
+        await makeDirectories(destDir);
+        const dest = join(destDir, filename);
+        report?.(`=== ${slug} / ${filename}`);
+        let bytes;
+        try {
+            ;
+            ({ bytes } = await downloadToFile({
+                url,
+                dest,
+                timeoutMs: 600_000,
+                headers: { "Accept-Encoding": "gzip, br" },
+                report,
+            }));
+        }
+        catch (error) {
+            report?.(`  ✗ download failed for ${url}: ${error.message}`);
+            failed++;
+            failedCodes.push(slug);
+            continue;
+        }
+        if (bytes < BYTES_PER_KIB) {
+            report?.(`  ✗ response too small (${bytes} bytes) — probable 404 / error page`);
+            failed++;
+            failedCodes.push(slug);
+            continue;
+        }
+        const sha = await sha256File(dest);
+        const manifest = {
+            source_url: url,
+            downloaded_at: new Date().toISOString(),
+            filename,
+            sha256: sha,
+            bytes,
+        };
+        await writeManifest(join(destDir, "MANIFEST.json"), manifest);
+        report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
+        fetched++;
+    }
+    report?.(`=== summary ===`);
+    report?.(`fetched: ${fetched}`);
+    report?.(`failed:  ${failed}`);
+    return { fetched, skipped: 0, failed, failedCodes };
+}
+//# sourceMappingURL=state-sources.js.map

package/out/tools/fetch/state-sources.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"state-sources.js","sourceRoot":"","sources":["../../../lib/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAerE,MAAM,OAAO,GAAsB;IAClC;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,kCAAkC;QAC5C,GAAG,EAAE,yEAAyE;KAC9E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,4EAA4E;KACjF;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,wBAAwB;QAClC,GAAG,EAAE,0EAA0E;KAC/E;IACD;QACC,IAAI,EAAE,sBAAsB;QAC5B,QAAQ,EAAE,qDAAqD;QAC/D,GAAG,EAAE,wEAAwE;KAC7E;IACD;QACC,IAAI,EAAE,2BAA2B;QACjC,QAAQ,EAAE,6CAA6C;QACvD,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,oBAAoB;QAC1B,QAAQ,EAAE,yCAAyC;QACnD,GAAG,EAAE,0EAA0E;KAC/E;CACD,CAAA;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,OAAiC,EACjC,MAA+B;IAE/B,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEtC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,CAAC;YAAA,CAAC,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;gBAClC,GAAG;gBACH,IAAI;gBACJ,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;gBAC1C,MAAM;aACN,CAAC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,2BAA2B,GAAG,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAEvE,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,2BAA2B,KAAK,qCAAqC,CAAC,CAAA;YAE/E,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAElC,MAAM,QAAQ,GAAmB;YAChC,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ;YACR,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAA;QAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;QAE7D,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAErE,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,YAAY,OAAO,EAAE,CAAC,CAAA;IAC/B,MAAM,EAAE,CAAC,YAAY,MAAM,EAAE,CAAC,CAAA;IAE9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/tools/fetch/tiger-full.d.ts

@@ -0,0 +1,22 @@
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export interface FetchTigerFullOptions extends BaseFetchOptions {
+    /**
+     * Space-separated list of 2-digit state FIPS codes to skip entirely. Default `"50"` — Vermont, already fetched in
+     * v0.1.1.
+     */
+    skipStateFips?: string;
+    /**
+     * Seconds to sleep between downloads. Default `0.2`.
+     */
+    rateSleep?: number;
+    /**
+     * Max concurrent download workers per state. Default `4`.
+     */
+    maxParallel?: number;
+    /**
+     * Print planned downloads without fetching. Default `false`.
+     */
+    dryRun?: boolean;
+}
+export declare function fetchTigerFull(options: FetchTigerFullOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=tiger-full.d.ts.map

package/out/tools/fetch/tiger-full.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tiger-full.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AA+BA,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAmB3E,MAAM,WAAW,qBAAsB,SAAQ,gBAAgB;IAC9D;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AA+DD,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA6KvB"}

package/out/tools/fetch/tiger-full.js

@@ -0,0 +1,217 @@
+import { APIClient, pluckResponseData } from "@mailwoman/core/api";
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
+ *
+ *   TIGER ADDRFEAT 2024 source:
+ *
+ *   - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
+ *   - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
+ *
+ *   Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
+ *   `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
+ *   already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
+ *   this module is download + provenance only.)
+ *
+ *   Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
+ *   county ZIP to disk (no curl subprocess).
+ */
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { statPath, pathExists } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { isoSeconds } from "@mailwoman/core/utils";
+import { sleep } from "@mailwoman/core/utils/sleep";
+import { basename, join } from "path-ts";
+import { readManifest, streamDownload, writeManifest } from "#tools/fetch/download";
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+/**
+ * Lowest 2xx status; anything below is informational.
+ */
+const HTTP_OK = 200;
+/**
+ * Lowest 3xx status; at or above it the response is a redirect or an error, not a body.
+ */
+const HTTP_REDIRECT = 300;
+const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT";
+/**
+ * Read a per-state MANIFEST.json into a filename → entry map.
+ */
+async function readCountyManifest(manifestPath) {
+    const map = new Map();
+    const parsed = await readManifest(manifestPath);
+    for (const c of parsed?.counties ?? []) {
+        if (c.filename) {
+            map.set(c.filename, { filename: c.filename, sha256: c.sha256, bytes: c.bytes });
+        }
+    }
+    return map;
+}
+/**
+ * Check whether a file already matches a recorded sha256 and byte count.
+ */
+async function fileMatchesSha(path, expectedSha, expectedBytes) {
+    if (!(await pathExists(path)))
+        return false;
+    if ((await statPath(path)).size !== expectedBytes)
+        return false;
+    return (await sha256File(path)) === expectedSha;
+}
+/**
+ * Download one county ZIP (size sanity check + sha256).
+ */
+async function downloadCounty(url, dest) {
+    const filename = basename(dest);
+    const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 });
+    if (status < HTTP_OK || status >= HTTP_REDIRECT) {
+        return { ok: false, filename, reason: `HTTP ${status}` };
+    }
+    const bytes = (await statPath(dest)).size;
+    if (bytes < BYTES_PER_KIB) {
+        await removePathIfPresent(dest);
+        return { ok: false, filename, reason: `too small (${bytes} bytes)` };
+    }
+    const sha256 = await sha256File(dest);
+    return { ok: true, filename, sha256, bytes };
+}
+export async function fetchTigerFull(options, report) {
+    const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length > 0);
+    const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000);
+    const maxParallel = options.maxParallel ?? 4;
+    const dryRun = options.dryRun ?? false;
+    const addrfeatDir = join(options.outRoot, "tiger", "addrfeat");
+    await makeDirectories(addrfeatDir);
+    // MARK: Step 1 — discover the county file list
+    report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`);
+    // `responseType: "text"` — an Apache directory index, scraped below. The per-county ARCHIVE downloads
+    // stay on raw `fetch` (they stream to disk; see `downloadOne`).
+    const listingRes = await new APIClient({
+        displayName: "tiger-listing",
+        retry: true,
+        axios: { headers: { "Accept-Encoding": "gzip, br" } },
+    })
+        .fetch({ url: `${TIGER_BASE_URL}/`, responseType: "text", timeout: 60_000 })
+        .then(pluckResponseData);
+    const html = listingRes;
+    const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted();
+    const totalCounties = allZips.length;
+    report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`);
+    // Build a map: state_fips -> list of filenames.
+    // tl_2024_SSCCC_addrfeat.zip — SS = 2-digit state FIPS (chars 8-9), CCC = county FIPS.
+    const stateFiles = new Map();
+    for (const fname of allZips) {
+        const stateFips = fname.slice(8, 10);
+        const list = stateFiles.get(stateFips) ?? [];
+        list.push(fname);
+        stateFiles.set(stateFips, list);
+    }
+    report?.(`  Spans ${stateFiles.size} state/territory FIPS codes.`);
+    // MARK: Step 2 — download the county ZIPs
+    let totalFetched = 0;
+    let totalSkipped = 0;
+    let totalSkippedState = 0;
+    let totalFailed = 0;
+    let totalBytesFetched = 0;
+    const failedCodes = [];
+    // Process states in sorted FIPS order for predictable output.
+    const sortedStates = [...stateFiles.keys()].toSorted();
+    for (const stateFips of sortedStates) {
+        const countyFiles = stateFiles.get(stateFips) ?? [];
+        // --- Skip entire state if requested ------------------------------------------
+        if (skipStateFips.includes(stateFips)) {
+            report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`);
+            totalSkippedState += countyFiles.length;
+            continue;
+        }
+        const stateDir = join(addrfeatDir, `state-${stateFips}`);
+        await makeDirectories(stateDir);
+        const manifestPath = join(stateDir, "MANIFEST.json");
+        // Load existing manifest for O(1) verified-skip lookup.
+        const manifest = await readCountyManifest(manifestPath);
+        report?.(`--- State ${stateFips} — ${countyFiles.length} counties`);
+        // Build a list of URLs+dests that need fetching.
+        const pending = [];
+        for (const fname of countyFiles) {
+            const dest = join(stateDir, fname);
+            const url = `${TIGER_BASE_URL}/${fname}`;
+            const known = manifest.get(fname);
+            // Skip if already verified via MANIFEST.
+            if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
+                report?.(`  skip (verified) ${fname}`);
+                totalSkipped++;
+                continue;
+            }
+            if (dryRun) {
+                report?.(`  would fetch: ${url}`);
+                totalFetched++;
+                continue;
+            }
+            pending.push({ url, dest });
+        }
+        if (dryRun)
+            continue;
+        if (!pending.length)
+            continue;
+        // --- Download pending files with bounded parallelism + rate-limit spacing ---
+        const results = Array.from({ length: pending.length });
+        let cursor = 0;
+        const workers = Array.from({ length: Math.min(maxParallel, pending.length) }, async () => {
+            while (true) {
+                const i = cursor++;
+                if (i >= pending.length)
+                    return;
+                const item = pending[i];
+                // Rate-limit: polite spacing before each fetch.
+                await sleep(rateSleepMs);
+                results[i] = await downloadCounty(item.url, item.dest);
+            }
+        });
+        await Promise.all(workers);
+        // Collect results from this state.
+        for (const result of results) {
+            if (result.ok) {
+                report?.(`  ok ${result.filename}  ${ByteFormatter.formatIEC(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`);
+                manifest.set(result.filename, { filename: result.filename, sha256: result.sha256, bytes: result.bytes });
+                totalFetched++;
+                totalBytesFetched += result.bytes;
+            }
+            else {
+                report?.(`  FAIL ${result.filename} -- ${result.reason}`);
+                totalFailed++;
+                failedCodes.push(result.filename);
+            }
+        }
+        // Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
+        const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename));
+        const manifestDoc = {
+            state_fips: stateFips,
+            updated_at: isoSeconds(),
+            tiger_base_url: TIGER_BASE_URL,
+            counties,
+        };
+        await writeManifest(manifestPath, manifestDoc);
+    }
+    // MARK: Summary
+    report?.(`=== Summary ===`);
+    report?.(`  Total counties in index   : ${totalCounties}`);
+    report?.(`  State(s) fully skipped    : ${totalSkippedState} (--skip-state-fips "${skipStateFips.join(" ")}")`);
+    report?.(`  Counties already present  : ${totalSkipped}`);
+    report?.(`  Counties fetched this run : ${totalFetched}`);
+    report?.(`  Counties failed           : ${totalFailed}`);
+    if (totalBytesFetched > 0) {
+        report?.(`  Bytes fetched this run    : ${ByteFormatter.formatIEC(totalBytesFetched)} (${totalBytesFetched})`);
+    }
+    if (totalFailed > 0) {
+        report?.(`WARNING: ${totalFailed} download(s) failed. Re-run to retry.`);
+    }
+    return { fetched: totalFetched, skipped: totalSkipped + totalSkippedState, failed: totalFailed, failedCodes };
+}
+//# sourceMappingURL=tiger-full.js.map