@mailwoman/corpus 9.1.0 → 9.3.0

package/out/src/tools/fetch/state-hi-schools.d.ts

@@ -1,25 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the Hawaii State DOE school directory and convert the XLSX workbook to a flat CSV the
- *   `state-hi-schools` adapter can consume.
- *
- *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
- *   (~38 public charter schools). Both sheets share the same header. This module concatenates them
- *   under one shared header so the adapter can stream a single CSV.
- *
- *   License: Hawaii state government open data (Tier A — state PD-equivalent).
- *
- *   Built-in `fetch` (gzip/brotli) replaces curl for the download; the XLSX → CSV step still rides
- *   `python3` + `openpyxl` via `node:child_process` — there is no clean node equivalent without
- *   adding a workbook-parsing dependency.
- *
- *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
- *   CSV exists and sha matches MANIFEST, skips download.
- */
-import type { BaseFetchOptions, FetchSummary } from "./download.ts";
-export type FetchStateHISchoolsOptions = BaseFetchOptions;
-export declare function fetchStateHISchools(options: FetchStateHISchoolsOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=state-hi-schools.d.ts.map

package/out/src/tools/fetch/state-hi-schools.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"state-hi-schools.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AASH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAcnE,MAAM,MAAM,0BAA0B,GAAG,gBAAgB,CAAA;AA0FzD,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA2FvB"}

package/out/src/tools/fetch/state-hi-schools.js

@@ -1,178 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the Hawaii State DOE school directory and convert the XLSX workbook to a flat CSV the
- *   `state-hi-schools` adapter can consume.
- *
- *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
- *   (~38 public charter schools). Both sheets share the same header. This module concatenates them
- *   under one shared header so the adapter can stream a single CSV.
- *
- *   License: Hawaii state government open data (Tier A — state PD-equivalent).
- *
- *   Built-in `fetch` (gzip/brotli) replaces curl for the download; the XLSX → CSV step still rides
- *   `python3` + `openpyxl` via `node:child_process` — there is no clean node equivalent without
- *   adding a workbook-parsing dependency.
- *
- *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
- *   CSV exists and sha matches MANIFEST, skips download.
- */
-import { spawn, spawnSync } from "node:child_process";
-import { existsSync, mkdirSync, statSync } from "node:fs";
-import { unlink } from "node:fs/promises";
-import { join } from "node:path";
-import { sha256File } from "@mailwoman/core/utils";
-import { downloadToFile, readManifest, writeManifest } from "./download.js";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-const BYTES_PER_KIB = 1024;
-const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx";
-const SLUG = "state-hi-schools";
-const CSV_FILENAME = "HI_Public_Schools_List.csv";
-const XLSX_FILENAME = "HI_Public_Schools_List.xlsx";
-/**
- * The XLSX → CSV converter: concatenate every sheet under one shared header (the first sheet's). Runs as `python3 -c
- * <script> <xlsx-path> <csv-path>`, so `sys.argv[1]`/`sys.argv[2]` are the I/O paths. TODO: Get rid of this.
- */
-const PY_CONVERT = `
-import csv
-import sys
-from openpyxl import load_workbook
-
-xlsx_path, csv_path = sys.argv[1], sys.argv[2]
-wb = load_workbook(xlsx_path, data_only=True, read_only=True)
-
-with open(csv_path, "w", newline="", encoding="utf-8") as out:
-    writer = csv.writer(out)
-    shared_header = None
-    total_data_rows = 0
-    for sheet_name in wb.sheetnames:
-        ws = wb[sheet_name]
-        rows = ws.iter_rows(values_only=True)
-        try:
-            header = next(rows)
-        except StopIteration:
-            continue
-        norm_header = ["" if v is None else str(v).strip() for v in header]
-        if shared_header is None:
-            shared_header = norm_header
-            writer.writerow(shared_header)
-        elif norm_header != shared_header:
-            print(
-                f"  ! sheet '{sheet_name}' header diverges from shared header; concatenating anyway",
-                file=sys.stderr,
-            )
-        for row in rows:
-            if row is None:
-                continue
-            # Skip fully-empty rows (XLSX iter_rows can yield phantom trailing rows).
-            if all(v is None or (isinstance(v, str) and not v.strip()) for v in row):
-                continue
-            writer.writerow(["" if v is None else str(v).strip() for v in row])
-            total_data_rows += 1
-
-print(f"  converted {total_data_rows} data rows from {len(wb.sheetnames)} sheets", file=sys.stderr)
-`;
-/**
- * Mimic `numfmt --to=iec` for a friendly byte-size log line.
- */
-function iec(bytes) {
-    if (bytes < BYTES_PER_KIB)
-        return String(bytes);
-    const units = ["K", "M", "G", "T", "P"];
-    let value = bytes / 1024;
-    let i = 0;
-    while (value >= BYTES_PER_KIB && i < units.length - 1) {
-        value /= 1024;
-        i++;
-    }
-    const rounded = value < 10 ? value.toFixed(1) : Math.round(value).toString();
-    return `${rounded}${units[i] ?? ""}`;
-}
-/**
- * Run the openpyxl converter. Its stderr narration streams straight through to the process stderr (matching the old
- * `stdio: inherit` behavior) rather than routing through `report` — the python child owns those lines.
- */
-async function convertXLSXToCSV(xlsxPath, csvPath) {
-    const child = spawn("python3", ["-c", PY_CONVERT, xlsxPath, csvPath], {
-        stdio: ["ignore", "inherit", "inherit"],
-    });
-    await new Promise((resolve, reject) => {
-        child.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`python3 converter exited with ${code}`))));
-        child.on("error", reject);
-    });
-}
-export async function fetchStateHISchools(options, report) {
-    const destDir = join(options.outRoot, SLUG);
-    mkdirSync(destDir, { recursive: true });
-    const xlsxDest = join(destDir, XLSX_FILENAME);
-    const csvDest = join(destDir, CSV_FILENAME);
-    const manifestPath = join(destDir, "MANIFEST.json");
-    report?.(`=== ${SLUG}`);
-    // Idempotency: skip if CSV exists and sha matches recorded MANIFEST.
-    if (existsSync(csvDest)) {
-        const recorded = await readManifest(manifestPath);
-        if (recorded?.sha256 && recorded.filename === CSV_FILENAME) {
-            const actualSha = await sha256File(csvDest);
-            if (actualSha === recorded.sha256) {
-                report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`);
-                return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
-            }
-        }
-    }
-    // Preflight: openpyxl must be importable.
-    const preflight = spawnSync("python3", ["-c", "import openpyxl"], { stdio: "ignore" });
-    if (preflight.status !== 0) {
-        report?.(`  ✗ python3 with the \`openpyxl\` package is required to convert the HIDOE XLSX.\n` +
-            `    Debian/Ubuntu:  sudo apt-get install -y python3-openpyxl\n` +
-            `    macOS Homebrew: brew install python && pip3 install openpyxl`);
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    // Download XLSX.
-    report?.(`  Downloading ${SOURCE_URL} ...`);
-    try {
-        await downloadToFile({
-            url: SOURCE_URL,
-            dest: xlsxDest,
-            timeoutMs: 600_000,
-            headers: { "Accept-Encoding": "gzip, br" },
-            report,
-        });
-    }
-    catch (error) {
-        report?.(`  ✗ Download failed (${error.message})`);
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    const xlsxSize = statSync(xlsxDest).size;
-    report?.(`  Downloaded XLSX: ${iec(xlsxSize)}`);
-    if (xlsxSize < BYTES_PER_KIB) {
-        report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`);
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    // Convert XLSX → CSV (concatenate both sheets under one shared header).
-    report?.(`  Converting XLSX → CSV (concatenating sheets) ...`);
-    await convertXLSXToCSV(xlsxDest, csvDest);
-    const csvSize = statSync(csvDest).size;
-    const csvSha = await sha256File(csvDest);
-    // Remove XLSX (CSV is the canonical artifact the adapter consumes).
-    await unlink(xlsxDest);
-    report?.(`  Removed XLSX (CSV kept)`);
-    // Write MANIFEST.
-    const manifest = {
-        source_url: SOURCE_URL,
-        downloaded_at: new Date().toISOString(),
-        filename: CSV_FILENAME,
-        sha256: csvSha,
-        bytes: csvSize,
-        notes: "Converted from XLSX (sheets HIDOE + PCS concatenated under shared header).",
-    };
-    await writeManifest(manifestPath, manifest);
-    report?.(`  ✓ ${iec(csvSize)}  sha256=${csvSha}`);
-    report?.(`  MANIFEST written to ${manifestPath}`);
-    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
-}
-//# sourceMappingURL=state-hi-schools.js.map

package/out/src/tools/fetch/state-hi-schools.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"state-hi-schools.js","sourceRoot":"","sources":["../../../../src/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E;;;GAGG;AACH,MAAM,aAAa,GAAG,IAAI,CAAA;AAE1B,MAAM,UAAU,GAAG,iEAAiE,CAAA;AACpF,MAAM,IAAI,GAAG,kBAAkB,CAAA;AAC/B,MAAM,YAAY,GAAG,4BAA4B,CAAA;AACjD,MAAM,aAAa,GAAG,6BAA6B,CAAA;AAInD;;;GAGG;AACH,MAAM,UAAU,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsClB,CAAA;AAWD;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa;IACzB,IAAI,KAAK,GAAG,aAAa;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAA;IAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IACvC,IAAI,KAAK,GAAG,KAAK,GAAG,IAAI,CAAA;IACxB,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,KAAK,IAAI,aAAa,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvD,KAAK,IAAI,IAAI,CAAA;QAEb,CAAC,EAAE,CAAA;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAA;IAE5E,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAA;AACrC,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,QAAgB,EAAE,OAAe;IAChE,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE;QACrE,KAAK,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;KACvC,CAAC,CAAA;IAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAClH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,OAAmC,EACnC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAA;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,qEAAqE;IACrE,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAoB,YAAY,CAAC,CAAA;QAEpE,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC5D,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;YAE3C,IAAI,SAAS,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;gBAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;YAC9D,CAAC;QACF,CAAC;IACF,CAAC;IAED,0CAA0C;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAA;IAEtF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,EAAE,CACP,oFAAoF;YACnF,gEAAgE;YAChE,kEAAkE,CACnE,CAAA;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,iBAAiB;IACjB,MAAM,EAAE,CAAC,iBAAiB,UAAU,MAAM,CAAC,CAAA;IAE3C,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC;YACpB,GAAG,EAAE,UAAU;YACf,IAAI,EAAE,QAAQ;YACd,SAAS,EAAE,OAAO;YAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;YAC1C,MAAM;SACN,CAAC,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,wBAAyB,KAAe,CAAC,OAAO,GAAG,CAAC,CAAA;QAE7D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAA;IACxC,MAAM,EAAE,CAAC,sBAAsB,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;IAE/C,IAAI,QAAQ,GAAG,aAAa,EAAE,CAAC;QAC9B,MAAM,EAAE,CAAC,2BAA2B,QAAQ,+BAA+B,CAAC,CAAA;QAE5E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,wEAAwE;IACxE,MAAM,EAAE,CAAC,oDAAoD,CAAC,CAAA;IAC9D,MAAM,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAEzC,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,oEAAoE;IACpE,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAA;IACtB,MAAM,EAAE,CAAC,2BAA2B,CAAC,CAAA;IAErC,kBAAkB;IAClB,MAAM,QAAQ,GAAa;QAC1B,UAAU,EAAE,UAAU;QACtB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,YAAY;QACtB,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,4EAA4E;KACnF,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC,YAAY,MAAM,EAAE,CAAC,CAAA;IACjD,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/src/tools/fetch/state-sources.d.ts

@@ -1,22 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
- *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
- *
- *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
- *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
- *   before the adapter can consume it.
- *
- *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
- *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
- *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
- */
-import type { BaseFetchOptions, FetchSummary } from "./download.ts";
-export type FetchStateSourcesOptions = BaseFetchOptions;
-export declare function fetchStateSources(options: FetchStateSourcesOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=state-sources.d.ts.map

package/out/src/tools/fetch/state-sources.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"state-sources.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AASnE,MAAM,MAAM,wBAAwB,GAAG,gBAAgB,CAAA;AAsDvD,wBAAsB,iBAAiB,CACtC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAgEvB"}

package/out/src/tools/fetch/state-sources.js

@@ -1,115 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
- *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
- *
- *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
- *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
- *   before the adapter can consume it.
- *
- *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
- *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
- *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
- */
-import { mkdirSync } from "node:fs";
-import { join } from "node:path";
-import { sha256File } from "@mailwoman/core/utils";
-import { downloadToFile, writeManifest } from "./download.js";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-const BYTES_PER_KIB = 1024;
-const SOURCES = [
-    {
-        slug: "state-ny-notaries",
-        filename: "NY_Commissioned_Notaries.csv",
-        url: "https://data.ny.gov/api/views/rwbv-mz6z/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-tx-notaries",
-        filename: "TX_Notary_Public_Commissions.csv",
-        url: "https://data.texas.gov/api/views/gmd3-bnrd/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-de-notaries",
-        filename: "DE_Notaries_Commissioned.csv",
-        url: "https://data.delaware.gov/api/views/q8dr-mj6p/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-or-notaries",
-        filename: "OR_Active_Notaries.csv",
-        url: "https://data.oregon.gov/api/views/j2pk-zk6z/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-ia-contractors",
-        filename: "IA_Active_Construction_Contractor_Registrations.csv",
-        url: "https://data.iowa.gov/api/views/dpf3-iz94/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-wa-health-providers",
-        filename: "WA_Health_Care_Provider_Credential_Data.csv",
-        url: "https://data.wa.gov/api/views/qxh8-f4bd/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-hi-lobbyists",
-        filename: "HI_Lobbyist_Registration_Statements.csv",
-        url: "https://data.hawaii.gov/api/views/cm7c-skav/rows.csv?accessType=DOWNLOAD",
-    },
-];
-export async function fetchStateSources(options, report) {
-    mkdirSync(options.outRoot, { recursive: true });
-    let fetched = 0;
-    let failed = 0;
-    const failedCodes = [];
-    for (const { slug, filename, url } of SOURCES) {
-        const destDir = join(options.outRoot, slug);
-        mkdirSync(destDir, { recursive: true });
-        const dest = join(destDir, filename);
-        report?.(`=== ${slug} / ${filename}`);
-        let bytes;
-        try {
-            ;
-            ({ bytes } = await downloadToFile({
-                url,
-                dest,
-                timeoutMs: 600_000,
-                headers: { "Accept-Encoding": "gzip, br" },
-                report,
-            }));
-        }
-        catch (error) {
-            report?.(`  ✗ download failed for ${url}: ${error.message}`);
-            failed++;
-            failedCodes.push(slug);
-            continue;
-        }
-        if (bytes < BYTES_PER_KIB) {
-            report?.(`  ✗ response too small (${bytes} bytes) — probable 404 / error page`);
-            failed++;
-            failedCodes.push(slug);
-            continue;
-        }
-        const sha = await sha256File(dest);
-        const manifest = {
-            source_url: url,
-            downloaded_at: new Date().toISOString(),
-            filename,
-            sha256: sha,
-            bytes,
-        };
-        await writeManifest(join(destDir, "MANIFEST.json"), manifest);
-        report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
-        fetched++;
-    }
-    report?.(`=== summary ===`);
-    report?.(`fetched: ${fetched}`);
-    report?.(`failed:  ${failed}`);
-    return { fetched, skipped: 0, failed, failedCodes };
-}
-//# sourceMappingURL=state-sources.js.map

package/out/src/tools/fetch/state-sources.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"state-sources.js","sourceRoot":"","sources":["../../../../src/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE7D;;;GAGG;AACH,MAAM,aAAa,GAAG,IAAI,CAAA;AAU1B,MAAM,OAAO,GAAsB;IAClC;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,kCAAkC;QAC5C,GAAG,EAAE,yEAAyE;KAC9E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,4EAA4E;KACjF;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,wBAAwB;QAClC,GAAG,EAAE,0EAA0E;KAC/E;IACD;QACC,IAAI,EAAE,sBAAsB;QAC5B,QAAQ,EAAE,qDAAqD;QAC/D,GAAG,EAAE,wEAAwE;KAC7E;IACD;QACC,IAAI,EAAE,2BAA2B;QACjC,QAAQ,EAAE,6CAA6C;QACvD,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,oBAAoB;QAC1B,QAAQ,EAAE,yCAAyC;QACnD,GAAG,EAAE,0EAA0E;KAC/E;CACD,CAAA;AAUD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,OAAiC,EACjC,MAA+B;IAE/B,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE/C,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,CAAC;YAAA,CAAC,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;gBAClC,GAAG;gBACH,IAAI;gBACJ,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;gBAC1C,MAAM;aACN,CAAC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,2BAA2B,GAAG,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAEvE,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,2BAA2B,KAAK,qCAAqC,CAAC,CAAA;YAE/E,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAElC,MAAM,QAAQ,GAAmB;YAChC,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ;YACR,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAA;QAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;QAE7D,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAErE,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,YAAY,OAAO,EAAE,CAAC,CAAA;IAC/B,MAAM,EAAE,CAAC,YAAY,MAAM,EAAE,CAAC,CAAA;IAE9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/src/tools/fetch/tiger-full.d.ts

@@ -1,42 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
- *
- *   TIGER ADDRFEAT 2024 source:
- *
- *   - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
- *   - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
- *
- *   Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
- *   `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
- *   already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
- *   this module is download + provenance only.)
- *
- *   Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
- *   county ZIP to disk (no curl subprocess).
- */
-import type { BaseFetchOptions, FetchSummary } from "./download.ts";
-export interface FetchTigerFullOptions extends BaseFetchOptions {
-    /**
-     * Space-separated list of 2-digit state FIPS codes to skip entirely. Default `"50"` — Vermont, already fetched in
-     * v0.1.1.
-     */
-    skipStateFips?: string;
-    /**
-     * Seconds to sleep between downloads. Default `0.2`.
-     */
-    rateSleep?: number;
-    /**
-     * Max concurrent download workers per state. Default `4`.
-     */
-    maxParallel?: number;
-    /**
-     * Print planned downloads without fetching. Default `false`.
-     */
-    dryRun?: boolean;
-}
-export declare function fetchTigerFull(options: FetchTigerFullOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=tiger-full.d.ts.map

package/out/src/tools/fetch/tiger-full.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"tiger-full.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAaH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAqBnE,MAAM,WAAW,qBAAsB,SAAQ,gBAAgB;IAC9D;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AAwHD,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAsKvB"}

package/out/src/tools/fetch/tiger-full.js

@@ -1,254 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
- *
- *   TIGER ADDRFEAT 2024 source:
- *
- *   - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
- *   - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
- *
- *   Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
- *   `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
- *   already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
- *   this module is download + provenance only.)
- *
- *   Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
- *   county ZIP to disk (no curl subprocess).
- */
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-import { createWriteStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
-import { basename, join } from "node:path";
-import { Readable } from "node:stream";
-import { pipeline } from "node:stream/promises";
-import { setTimeout as sleep } from "node:timers/promises";
-import { sha256File } from "@mailwoman/core/utils";
-import { isTransientStatus, readManifest, writeManifest } from "./download.js";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-/**
- * Lowest 2xx status; anything below is informational.
- */
-const HTTP_OK = 200;
-/**
- * Lowest 3xx status; at or above it the response is a redirect or an error, not a body.
- */
-const HTTP_REDIRECT = 300;
-const BYTES_PER_KIB = 1024;
-const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT";
-function humanBytes(bytes) {
-    const units = ["B", "KiB", "MiB", "GiB", "TiB"];
-    let value = bytes;
-    let unit = 0;
-    while (value >= BYTES_PER_KIB && unit < units.length - 1) {
-        value /= 1024;
-        unit++;
-    }
-    return `${value.toFixed(unit === 0 ? 0 : 1)}${units[unit]}`;
-}
-/**
- * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries).
- *
- * NOTE(phase1): kept local instead of the shared `downloadToFile` — this one streams each county ZIP to disk (the
- * shared util buffers via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-county result
- * collector consumes.
- */
-async function streamDownload(url, dest, opts) {
-    for (let attempt = 0; attempt <= opts.retries; attempt++) {
-        try {
-            const res = await fetch(url, { redirect: "follow", signal: AbortSignal.timeout(opts.timeoutMs) });
-            if (res.ok && res.body) {
-                await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
-                return res.status;
-            }
-            if (attempt < opts.retries && isTransientStatus(res.status)) {
-                await sleep(opts.retryDelayMs);
-                continue;
-            }
-            return res.status;
-        }
-        catch {
-            if (attempt < opts.retries) {
-                await sleep(opts.retryDelayMs);
-                continue;
-            }
-            return 0;
-        }
-    }
-    return 0;
-}
-/**
- * Read a per-state MANIFEST.json into a filename → entry map.
- */
-async function readCountyManifest(manifestPath) {
-    const map = new Map();
-    const parsed = await readManifest(manifestPath);
-    for (const c of parsed?.counties ?? []) {
-        if (c.filename) {
-            map.set(c.filename, { filename: c.filename, sha256: c.sha256, bytes: c.bytes });
-        }
-    }
-    return map;
-}
-/**
- * Check whether a file already matches a recorded sha256 and byte count.
- */
-async function fileMatchesSha(path, expectedSha, expectedBytes) {
-    if (!existsSync(path))
-        return false;
-    if (statSync(path).size !== expectedBytes)
-        return false;
-    return (await sha256File(path)) === expectedSha;
-}
-/**
- * Download one county ZIP (size sanity check + sha256).
- */
-async function downloadCounty(url, dest) {
-    const filename = basename(dest);
-    const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 });
-    if (status < HTTP_OK || status >= HTTP_REDIRECT) {
-        return { ok: false, filename, reason: `HTTP ${status}` };
-    }
-    const bytes = statSync(dest).size;
-    if (bytes < BYTES_PER_KIB) {
-        rmSync(dest, { force: true });
-        return { ok: false, filename, reason: `too small (${bytes} bytes)` };
-    }
-    const sha256 = await sha256File(dest);
-    return { ok: true, filename, sha256, bytes };
-}
-export async function fetchTigerFull(options, report) {
-    const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter(Boolean);
-    const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000);
-    const maxParallel = options.maxParallel ?? 4;
-    const dryRun = options.dryRun ?? false;
-    const addrfeatDir = join(options.outRoot, "tiger", "addrfeat");
-    mkdirSync(addrfeatDir, { recursive: true });
-    // MARK: Step 1 — discover the county file list
-    report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`);
-    const listingRes = await fetch(`${TIGER_BASE_URL}/`, {
-        headers: { "Accept-Encoding": "gzip, br" },
-        signal: AbortSignal.timeout(60_000),
-    });
-    if (!listingRes.ok)
-        throw new Error(`Failed to fetch TIGER directory listing: HTTP ${listingRes.status}`);
-    const html = await listingRes.text();
-    const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted();
-    const totalCounties = allZips.length;
-    report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`);
-    // Build a map: state_fips -> list of filenames.
-    // tl_2024_SSCCC_addrfeat.zip — SS = 2-digit state FIPS (chars 8-9), CCC = county FIPS.
-    const stateFiles = new Map();
-    for (const fname of allZips) {
-        const stateFips = fname.slice(8, 10);
-        const list = stateFiles.get(stateFips) ?? [];
-        list.push(fname);
-        stateFiles.set(stateFips, list);
-    }
-    report?.(`  Spans ${stateFiles.size} state/territory FIPS codes.`);
-    // MARK: Step 2 — download the county ZIPs
-    let totalFetched = 0;
-    let totalSkipped = 0;
-    let totalSkippedState = 0;
-    let totalFailed = 0;
-    let totalBytesFetched = 0;
-    const failedCodes = [];
-    // Process states in sorted FIPS order for predictable output.
-    const sortedStates = [...stateFiles.keys()].toSorted();
-    for (const stateFips of sortedStates) {
-        const countyFiles = stateFiles.get(stateFips) ?? [];
-        // --- Skip entire state if requested ------------------------------------------
-        if (skipStateFips.includes(stateFips)) {
-            report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`);
-            totalSkippedState += countyFiles.length;
-            continue;
-        }
-        const stateDir = join(addrfeatDir, `state-${stateFips}`);
-        mkdirSync(stateDir, { recursive: true });
-        const manifestPath = join(stateDir, "MANIFEST.json");
-        // Load existing manifest for O(1) verified-skip lookup.
-        const manifest = await readCountyManifest(manifestPath);
-        report?.(`--- State ${stateFips} — ${countyFiles.length} counties`);
-        // Build a list of URLs+dests that need fetching.
-        const pending = [];
-        for (const fname of countyFiles) {
-            const dest = join(stateDir, fname);
-            const url = `${TIGER_BASE_URL}/${fname}`;
-            const known = manifest.get(fname);
-            // Skip if already verified via MANIFEST.
-            if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
-                report?.(`  skip (verified) ${fname}`);
-                totalSkipped++;
-                continue;
-            }
-            if (dryRun) {
-                report?.(`  would fetch: ${url}`);
-                totalFetched++;
-                continue;
-            }
-            pending.push({ url, dest });
-        }
-        if (dryRun)
-            continue;
-        if (!pending.length)
-            continue;
-        // --- Download pending files with bounded parallelism + rate-limit spacing ---
-        const results = new Array(pending.length);
-        let cursor = 0;
-        const workers = Array.from({ length: Math.min(maxParallel, pending.length) }, async () => {
-            while (true) {
-                const i = cursor++;
-                if (i >= pending.length)
-                    return;
-                const item = pending[i];
-                // Rate-limit: polite spacing before each fetch.
-                await sleep(rateSleepMs);
-                results[i] = await downloadCounty(item.url, item.dest);
-            }
-        });
-        await Promise.all(workers);
-        // Collect results from this state.
-        for (const result of results) {
-            if (result.ok) {
-                report?.(`  ok ${result.filename}  ${humanBytes(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`);
-                manifest.set(result.filename, { filename: result.filename, sha256: result.sha256, bytes: result.bytes });
-                totalFetched++;
-                totalBytesFetched += result.bytes;
-            }
-            else {
-                report?.(`  FAIL ${result.filename} -- ${result.reason}`);
-                totalFailed++;
-                failedCodes.push(result.filename);
-            }
-        }
-        // Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
-        const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename));
-        const manifestDoc = {
-            state_fips: stateFips,
-            updated_at: new Date().toISOString().replace(/\.\d{3}Z$/, "Z"),
-            tiger_base_url: TIGER_BASE_URL,
-            counties,
-        };
-        await writeManifest(manifestPath, manifestDoc);
-    }
-    // MARK: Summary
-    report?.(`=== Summary ===`);
-    report?.(`  Total counties in index   : ${totalCounties}`);
-    report?.(`  State(s) fully skipped    : ${totalSkippedState} (--skip-state-fips "${skipStateFips.join(" ")}")`);
-    report?.(`  Counties already present  : ${totalSkipped}`);
-    report?.(`  Counties fetched this run : ${totalFetched}`);
-    report?.(`  Counties failed           : ${totalFailed}`);
-    if (totalBytesFetched > 0) {
-        report?.(`  Bytes fetched this run    : ${humanBytes(totalBytesFetched)} (${totalBytesFetched})`);
-    }
-    if (totalFailed > 0) {
-        report?.(`WARNING: ${totalFailed} download(s) failed. Re-run to retry.`);
-    }
-    return { fetched: totalFetched, skipped: totalSkipped + totalSkippedState, failed: totalFailed, failedCodes };
-}
-//# sourceMappingURL=tiger-full.js.map