@mailwoman/corpus 9.3.0 → 10.0.0

package/out/tools/fetch/state-sources.js

@@ -1,111 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
- *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
- *
- *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
- *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
- *   before the adapter can consume it.
- *
- *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
- *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
- *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
- */
-import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters";
-import { makeDirectories } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { join } from "path-ts";
-import { downloadToFile, writeManifest } from "#tools/fetch/download";
-const SOURCES = [
-    {
-        slug: "state-ny-notaries",
-        filename: "NY_Commissioned_Notaries.csv",
-        url: "https://data.ny.gov/api/views/rwbv-mz6z/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-tx-notaries",
-        filename: "TX_Notary_Public_Commissions.csv",
-        url: "https://data.texas.gov/api/views/gmd3-bnrd/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-de-notaries",
-        filename: "DE_Notaries_Commissioned.csv",
-        url: "https://data.delaware.gov/api/views/q8dr-mj6p/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-or-notaries",
-        filename: "OR_Active_Notaries.csv",
-        url: "https://data.oregon.gov/api/views/j2pk-zk6z/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-ia-contractors",
-        filename: "IA_Active_Construction_Contractor_Registrations.csv",
-        url: "https://data.iowa.gov/api/views/dpf3-iz94/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-wa-health-providers",
-        filename: "WA_Health_Care_Provider_Credential_Data.csv",
-        url: "https://data.wa.gov/api/views/qxh8-f4bd/rows.csv?accessType=DOWNLOAD",
-    },
-    {
-        slug: "state-hi-lobbyists",
-        filename: "HI_Lobbyist_Registration_Statements.csv",
-        url: "https://data.hawaii.gov/api/views/cm7c-skav/rows.csv?accessType=DOWNLOAD",
-    },
-];
-export async function fetchStateSources(options, report) {
-    await makeDirectories(options.outRoot);
-    let fetched = 0;
-    let failed = 0;
-    const failedCodes = [];
-    for (const { slug, filename, url } of SOURCES) {
-        const destDir = join(options.outRoot, slug);
-        await makeDirectories(destDir);
-        const dest = join(destDir, filename);
-        report?.(`=== ${slug} / ${filename}`);
-        let bytes;
-        try {
-            ;
-            ({ bytes } = await downloadToFile({
-                url,
-                dest,
-                timeoutMs: 600_000,
-                headers: { "Accept-Encoding": "gzip, br" },
-                report,
-            }));
-        }
-        catch (error) {
-            report?.(`  ✗ download failed for ${url}: ${error.message}`);
-            failed++;
-            failedCodes.push(slug);
-            continue;
-        }
-        if (bytes < BYTES_PER_KIB) {
-            report?.(`  ✗ response too small (${bytes} bytes) — probable 404 / error page`);
-            failed++;
-            failedCodes.push(slug);
-            continue;
-        }
-        const sha = await sha256File(dest);
-        const manifest = {
-            source_url: url,
-            downloaded_at: new Date().toISOString(),
-            filename,
-            sha256: sha,
-            bytes,
-        };
-        await writeManifest(join(destDir, "MANIFEST.json"), manifest);
-        report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
-        fetched++;
-    }
-    report?.(`=== summary ===`);
-    report?.(`fetched: ${fetched}`);
-    report?.(`failed:  ${failed}`);
-    return { fetched, skipped: 0, failed, failedCodes };
-}
-//# sourceMappingURL=state-sources.js.map

package/out/tools/fetch/state-sources.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"state-sources.js","sourceRoot":"","sources":["../../../lib/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAerE,MAAM,OAAO,GAAsB;IAClC;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,kCAAkC;QAC5C,GAAG,EAAE,yEAAyE;KAC9E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,4EAA4E;KACjF;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,wBAAwB;QAClC,GAAG,EAAE,0EAA0E;KAC/E;IACD;QACC,IAAI,EAAE,sBAAsB;QAC5B,QAAQ,EAAE,qDAAqD;QAC/D,GAAG,EAAE,wEAAwE;KAC7E;IACD;QACC,IAAI,EAAE,2BAA2B;QACjC,QAAQ,EAAE,6CAA6C;QACvD,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,oBAAoB;QAC1B,QAAQ,EAAE,yCAAyC;QACnD,GAAG,EAAE,0EAA0E;KAC/E;CACD,CAAA;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,OAAiC,EACjC,MAA+B;IAE/B,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEtC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,CAAC;YAAA,CAAC,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;gBAClC,GAAG;gBACH,IAAI;gBACJ,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;gBAC1C,MAAM;aACN,CAAC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,2BAA2B,GAAG,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAEvE,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,2BAA2B,KAAK,qCAAqC,CAAC,CAAA;YAE/E,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAElC,MAAM,QAAQ,GAAmB;YAChC,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ;YACR,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAA;QAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;QAE7D,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAErE,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,YAAY,OAAO,EAAE,CAAC,CAAA;IAC/B,MAAM,EAAE,CAAC,YAAY,MAAM,EAAE,CAAC,CAAA;IAE9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/tools/fetch/tiger-full.d.ts

@@ -1,22 +0,0 @@
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
-export interface FetchTigerFullOptions extends BaseFetchOptions {
-    /**
-     * Space-separated list of 2-digit state FIPS codes to skip entirely. Default `"50"` — Vermont, already fetched in
-     * v0.1.1.
-     */
-    skipStateFips?: string;
-    /**
-     * Seconds to sleep between downloads. Default `0.2`.
-     */
-    rateSleep?: number;
-    /**
-     * Max concurrent download workers per state. Default `4`.
-     */
-    maxParallel?: number;
-    /**
-     * Print planned downloads without fetching. Default `false`.
-     */
-    dryRun?: boolean;
-}
-export declare function fetchTigerFull(options: FetchTigerFullOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=tiger-full.d.ts.map

package/out/tools/fetch/tiger-full.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"tiger-full.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AA+BA,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAmB3E,MAAM,WAAW,qBAAsB,SAAQ,gBAAgB;IAC9D;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AA+DD,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA6KvB"}

package/out/tools/fetch/tiger-full.js

@@ -1,217 +0,0 @@
-import { APIClient, pluckResponseData } from "@mailwoman/core/api";
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
- *
- *   TIGER ADDRFEAT 2024 source:
- *
- *   - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
- *   - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
- *
- *   Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
- *   `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
- *   already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
- *   this module is download + provenance only.)
- *
- *   Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
- *   county ZIP to disk (no curl subprocess).
- */
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters";
-import { statPath, pathExists } from "@mailwoman/core/fs/readers";
-import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { isoSeconds } from "@mailwoman/core/utils";
-import { sleep } from "@mailwoman/core/utils/sleep";
-import { basename, join } from "path-ts";
-import { readManifest, streamDownload, writeManifest } from "#tools/fetch/download";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-/**
- * Lowest 2xx status; anything below is informational.
- */
-const HTTP_OK = 200;
-/**
- * Lowest 3xx status; at or above it the response is a redirect or an error, not a body.
- */
-const HTTP_REDIRECT = 300;
-const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT";
-/**
- * Read a per-state MANIFEST.json into a filename → entry map.
- */
-async function readCountyManifest(manifestPath) {
-    const map = new Map();
-    const parsed = await readManifest(manifestPath);
-    for (const c of parsed?.counties ?? []) {
-        if (c.filename) {
-            map.set(c.filename, { filename: c.filename, sha256: c.sha256, bytes: c.bytes });
-        }
-    }
-    return map;
-}
-/**
- * Check whether a file already matches a recorded sha256 and byte count.
- */
-async function fileMatchesSha(path, expectedSha, expectedBytes) {
-    if (!(await pathExists(path)))
-        return false;
-    if ((await statPath(path)).size !== expectedBytes)
-        return false;
-    return (await sha256File(path)) === expectedSha;
-}
-/**
- * Download one county ZIP (size sanity check + sha256).
- */
-async function downloadCounty(url, dest) {
-    const filename = basename(dest);
-    const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 });
-    if (status < HTTP_OK || status >= HTTP_REDIRECT) {
-        return { ok: false, filename, reason: `HTTP ${status}` };
-    }
-    const bytes = (await statPath(dest)).size;
-    if (bytes < BYTES_PER_KIB) {
-        await removePathIfPresent(dest);
-        return { ok: false, filename, reason: `too small (${bytes} bytes)` };
-    }
-    const sha256 = await sha256File(dest);
-    return { ok: true, filename, sha256, bytes };
-}
-export async function fetchTigerFull(options, report) {
-    const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length > 0);
-    const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000);
-    const maxParallel = options.maxParallel ?? 4;
-    const dryRun = options.dryRun ?? false;
-    const addrfeatDir = join(options.outRoot, "tiger", "addrfeat");
-    await makeDirectories(addrfeatDir);
-    // MARK: Step 1 — discover the county file list
-    report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`);
-    // `responseType: "text"` — an Apache directory index, scraped below. The per-county ARCHIVE downloads
-    // stay on raw `fetch` (they stream to disk; see `downloadOne`).
-    const listingRes = await new APIClient({
-        displayName: "tiger-listing",
-        retry: true,
-        axios: { headers: { "Accept-Encoding": "gzip, br" } },
-    })
-        .fetch({ url: `${TIGER_BASE_URL}/`, responseType: "text", timeout: 60_000 })
-        .then(pluckResponseData);
-    const html = listingRes;
-    const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted();
-    const totalCounties = allZips.length;
-    report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`);
-    // Build a map: state_fips -> list of filenames.
-    // tl_2024_SSCCC_addrfeat.zip — SS = 2-digit state FIPS (chars 8-9), CCC = county FIPS.
-    const stateFiles = new Map();
-    for (const fname of allZips) {
-        const stateFips = fname.slice(8, 10);
-        const list = stateFiles.get(stateFips) ?? [];
-        list.push(fname);
-        stateFiles.set(stateFips, list);
-    }
-    report?.(`  Spans ${stateFiles.size} state/territory FIPS codes.`);
-    // MARK: Step 2 — download the county ZIPs
-    let totalFetched = 0;
-    let totalSkipped = 0;
-    let totalSkippedState = 0;
-    let totalFailed = 0;
-    let totalBytesFetched = 0;
-    const failedCodes = [];
-    // Process states in sorted FIPS order for predictable output.
-    const sortedStates = [...stateFiles.keys()].toSorted();
-    for (const stateFips of sortedStates) {
-        const countyFiles = stateFiles.get(stateFips) ?? [];
-        // --- Skip entire state if requested ------------------------------------------
-        if (skipStateFips.includes(stateFips)) {
-            report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`);
-            totalSkippedState += countyFiles.length;
-            continue;
-        }
-        const stateDir = join(addrfeatDir, `state-${stateFips}`);
-        await makeDirectories(stateDir);
-        const manifestPath = join(stateDir, "MANIFEST.json");
-        // Load existing manifest for O(1) verified-skip lookup.
-        const manifest = await readCountyManifest(manifestPath);
-        report?.(`--- State ${stateFips} — ${countyFiles.length} counties`);
-        // Build a list of URLs+dests that need fetching.
-        const pending = [];
-        for (const fname of countyFiles) {
-            const dest = join(stateDir, fname);
-            const url = `${TIGER_BASE_URL}/${fname}`;
-            const known = manifest.get(fname);
-            // Skip if already verified via MANIFEST.
-            if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
-                report?.(`  skip (verified) ${fname}`);
-                totalSkipped++;
-                continue;
-            }
-            if (dryRun) {
-                report?.(`  would fetch: ${url}`);
-                totalFetched++;
-                continue;
-            }
-            pending.push({ url, dest });
-        }
-        if (dryRun)
-            continue;
-        if (!pending.length)
-            continue;
-        // --- Download pending files with bounded parallelism + rate-limit spacing ---
-        const results = Array.from({ length: pending.length });
-        let cursor = 0;
-        const workers = Array.from({ length: Math.min(maxParallel, pending.length) }, async () => {
-            while (true) {
-                const i = cursor++;
-                if (i >= pending.length)
-                    return;
-                const item = pending[i];
-                // Rate-limit: polite spacing before each fetch.
-                await sleep(rateSleepMs);
-                results[i] = await downloadCounty(item.url, item.dest);
-            }
-        });
-        await Promise.all(workers);
-        // Collect results from this state.
-        for (const result of results) {
-            if (result.ok) {
-                report?.(`  ok ${result.filename}  ${ByteFormatter.formatIEC(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`);
-                manifest.set(result.filename, { filename: result.filename, sha256: result.sha256, bytes: result.bytes });
-                totalFetched++;
-                totalBytesFetched += result.bytes;
-            }
-            else {
-                report?.(`  FAIL ${result.filename} -- ${result.reason}`);
-                totalFailed++;
-                failedCodes.push(result.filename);
-            }
-        }
-        // Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
-        const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename));
-        const manifestDoc = {
-            state_fips: stateFips,
-            updated_at: isoSeconds(),
-            tiger_base_url: TIGER_BASE_URL,
-            counties,
-        };
-        await writeManifest(manifestPath, manifestDoc);
-    }
-    // MARK: Summary
-    report?.(`=== Summary ===`);
-    report?.(`  Total counties in index   : ${totalCounties}`);
-    report?.(`  State(s) fully skipped    : ${totalSkippedState} (--skip-state-fips "${skipStateFips.join(" ")}")`);
-    report?.(`  Counties already present  : ${totalSkipped}`);
-    report?.(`  Counties fetched this run : ${totalFetched}`);
-    report?.(`  Counties failed           : ${totalFailed}`);
-    if (totalBytesFetched > 0) {
-        report?.(`  Bytes fetched this run    : ${ByteFormatter.formatIEC(totalBytesFetched)} (${totalBytesFetched})`);
-    }
-    if (totalFailed > 0) {
-        report?.(`WARNING: ${totalFailed} download(s) failed. Re-run to retry.`);
-    }
-    return { fetched: totalFetched, skipped: totalSkipped + totalSkippedState, failed: totalFailed, failedCodes };
-}
-//# sourceMappingURL=tiger-full.js.map

package/out/tools/fetch/tiger-full.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"tiger-full.js","sourceRoot":"","sources":["../../../lib/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE;;;;;;;;;;;;;;;;;;;GAmBG;AACH;kGACkG;AAClG,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC5E,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AACnD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAGxC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEnF;;;GAGG;AACH;;GAEG;AACH,MAAM,OAAO,GAAG,GAAG,CAAA;AAEnB;;GAEG;AACH,MAAM,aAAa,GAAG,GAAG,CAAA;AAEzB,MAAM,cAAc,GAAG,sDAAsD,CAAA;AA4B7E;;GAEG;AACH,KAAK,UAAU,kBAAkB,CAAC,YAAoB;IACrD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAuB,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,YAAY,CAA+B,YAAY,CAAC,CAAA;IAE7E,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,QAAQ,IAAI,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YAChB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;QAChF,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,WAAmB,EAAE,aAAqB;IACrF,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAA;IAE3C,IAAI,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa;QAAE,OAAO,KAAK,CAAA;IAE/D,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,WAAW,CAAA;AAChD,CAAC;AAMD;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY;IACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAC/B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtG,IAAI,MAAM,GAAG,OAAO,IAAI,MAAM,IAAI,aAAa,EAAE,CAAC;QACjD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,MAAM,EAAE,EAAE,CAAA;IACzD,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;IAEzC,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;QAC3B,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAE/B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,KAAK,SAAS,EAAE,CAAA;IACrE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAErC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,aAAa,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACtG,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;IACjE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IAEtC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,CAAA;IAC9D,MAAM,eAAe,CAAC,WAAW,CAAC,CAAA;IAElC,+CAA+C;IAE/C,MAAM,EAAE,CAAC,uDAAuD,CAAC,CAAA;IAEjE,sGAAsG;IACtG,gEAAgE;IAChE,MAAM,UAAU,GAAG,MAAM,IAAI,SAAS,CAAC;QACtC,WAAW,EAAE,eAAe;QAC5B,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;KACrD,CAAC;SACA,KAAK,CAAS,EAAE,GAAG,EAAE,GAAG,cAAc,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SACnF,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,MAAM,IAAI,GAAG,UAAU,CAAA;IACvB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;IACtF,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAA;IACpC,MAAM,EAAE,CAAC,WAAW,aAAa,gDAAgD,CAAC,CAAA;IAElF,gDAAgD;IAChD,uFAAuF;IACvF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE9C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACpC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAC5C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChB,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAChC,CAAC;IAED,MAAM,EAAE,CAAC,WAAW,UAAU,CAAC,IAAI,8BAA8B,CAAC,CAAA;IAElE,0CAA0C;IAE1C,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,8DAA8D;IAC9D,MAAM,YAAY,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEtD,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAEnD,gFAAgF;QAChF,IAAI,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,MAAM,EAAE,CAAC,aAAa,SAAS,qCAAqC,WAAW,CAAC,MAAM,YAAY,CAAC,CAAA;YACnG,iBAAiB,IAAI,WAAW,CAAC,MAAM,CAAA;YAEvC,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,SAAS,SAAS,EAAE,CAAC,CAAA;QACxD,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;QAC/B,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAA;QAEpD,wDAAwD;QACxD,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,CAAA;QAEvD,MAAM,EAAE,CAAC,aAAa,SAAS,MAAM,WAAW,CAAC,MAAM,WAAW,CAAC,CAAA;QAEnE,iDAAiD;QACjD,MAAM,OAAO,GAAyC,EAAE,CAAA;QAExD,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YAClC,MAAM,GAAG,GAAG,GAAG,cAAc,IAAI,KAAK,EAAE,CAAA;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAEjC,yCAAyC;YACzC,IAAI,KAAK,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACtE,MAAM,EAAE,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAA;gBAEtC,YAAY,EAAE,CAAA;gBAEd,SAAQ;YACT,CAAC;YAED,IAAI,MAAM,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAA;gBAEjC,YAAY,EAAE,CAAA;gBAEd,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;QAC5B,CAAC;QAED,IAAI,MAAM;YAAE,SAAQ;QAEpB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,SAAQ;QAE7B,+EAA+E;QAC/E,MAAM,OAAO,GAAmB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QACtE,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;YACxF,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;gBAElB,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM;oBAAE,OAAM;gBAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;gBACxB,gDAAgD;gBAChD,MAAM,KAAK,CAAC,WAAW,CAAC,CAAA;gBACxB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACvD,CAAC;QACF,CAAC,CAAC,CAAA;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAE1B,mCAAmC;QACnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,EAAE,CACP,QAAQ,MAAM,CAAC,QAAQ,KAAK,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAC5G,CAAA;gBAED,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;gBAExG,YAAY,EAAE,CAAA;gBACd,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAA;YAClC,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,UAAU,MAAM,CAAC,QAAQ,OAAO,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;gBAEzD,WAAW,EAAE,CAAA;gBACb,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YAClC,CAAC;QACF,CAAC;QAED,yFAAyF;QACzF,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;QAEhG,MAAM,WAAW,GAAG;YACnB,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,UAAU,EAAE;YACxB,cAAc,EAAE,cAAc;YAC9B,QAAQ;SACR,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;IAC/C,CAAC;IAED,gBAAgB;IAEhB,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,iCAAiC,aAAa,EAAE,CAAC,CAAA;IAC1D,MAAM,EAAE,CAAC,iCAAiC,iBAAiB,wBAAwB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,WAAW,EAAE,CAAC,CAAA;IAExD,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,CAAC,iCAAiC,aAAa,CAAC,SAAS,CAAC,iBAAiB,CAAC,KAAK,iBAAiB,GAAG,CAAC,CAAA;IAC/G,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,EAAE,CAAC,YAAY,WAAW,uCAAuC,CAAC,CAAA;IACzE,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,GAAG,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAA;AAC9G,CAAC"}

package/out/tools/golden-expand.d.ts

@@ -1,90 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Generate golden-set candidate entries by LLM-driven surface-form synthesis from a verified-label
- *   seed pulled out of the corpus's labeled test split.
- *
- *   ## Why this approach
- *
- *   The Phase 2 golden set has 74 entries; session-notes.md called for ≥500/locale. Manual curation
- *   doesn't scale. Pure-LLM generation (invent raw + labels from scratch) is too noisy — labels
- *   would be unverified.
- *
- *   This module takes the middle path:
- *
- *   1. **Seeds come from corpus-v0.2.0 test slice** — already through the alignment pipeline, so labels
- *        are pipeline-verified.
- *   2. **LLM only varies the surface form** — case, abbreviations, reordering, dropped components. The
- *        component VALUES (locality string, postcode digits, etc.) are preserved verbatim.
- *   3. **Programmatic validator drops hallucinations** — every component value must appear as a
- *        substring (case-insensitive, whitespace-normalized) of the variant's raw. Failures dropped
- *        silently; cost is wasted tokens, never bad-labeled golden entries.
- *
- *   ## Usage
- *
- *   ```sh
- *   DEEPSEEK_API_KEY=sk-... \
- *   mailwoman corpus golden expand \
- *   --count 1000 \
- *   --variants 5 \
- *   --output data/eval/golden/candidates/expand-$(date +%Y%m%d-%H%M%S).jsonl
- *   ```
- *
- *   ## Env
- *
- *   - `DEEPSEEK_API_KEY` — required for provider `deepseek`
- *   - `ANTHROPIC_API_KEY` — required for provider `anthropic`
- *
- *   ## What this module does NOT do
- *
- *   - Does not commit anything or modify the versioned golden dir. Candidates land in
- *       `data/eval/golden/candidates/` for operator review (skim, prune, then run
- *       `mailwoman corpus golden promote`).
- *   - Does not score the LLM's quality — that's an eyeball job after pilot lands.
- *   - Does not retry hallucinated candidates. Cost of wasted tokens is trivial (~$0.0006/each).
- */
-export interface ExpandGoldenOptions {
-    /**
-     * Corpus test slice path(s), comma-separated. Default: the v0.2.0 test slice under the data root.
-     */
-    corpus?: string;
-    /**
-     * Total seeds to process. Default `100` (pilot).
-     */
-    count?: number;
-    /**
-     * Variants requested per seed. Default `5`.
-     */
-    variants?: number;
-    /**
-     * JSONL output path. Default `data/eval/golden/candidates/expand-<ts>.jsonl`.
-     */
-    output?: string;
-    /**
-     * LLM provider. Default `deepseek`.
-     */
-    provider?: "deepseek" | "anthropic";
-    /**
-     * Model id. Default depends on provider.
-     */
-    model?: string;
-    /**
-     * Parallel LLM calls. Default `4`.
-     */
-    concurrency?: number;
-    /**
-     * Comma-separated source allow-list.
-     */
-    includeSources?: string;
-}
-export interface ExpandGoldenSummary {
-    seedsProcessed: number;
-    kept: number;
-    dropped: number;
-    errored: number;
-    outputPath: string;
-}
-export declare function expandGolden(options?: ExpandGoldenOptions, report?: (line: string) => void): Promise<ExpandGoldenSummary>;
-//# sourceMappingURL=golden-expand.d.ts.map

package/out/tools/golden-expand.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"golden-expand.d.ts","sourceRoot":"","sources":["../../lib/tools/golden-expand.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAuDH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,QAAQ,CAAC,EAAE,UAAU,GAAG,WAAW,CAAA;IACnC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;CAClB;AA0UD,wBAAsB,YAAY,CACjC,OAAO,GAAE,mBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,mBAAmB,CAAC,CAkG9B"}