@mailwoman/corpus 10.0.0 → 10.2.0

package/out/us/tools/fetch/tiger-full.js

@@ -4,47 +4,26 @@ import { APIClient, pluckResponseData } from "@mailwoman/core/api";
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
+ * Fetch the full TIGER 2024 addrfeat dataset for all US counties from
+ * `https://www2.census.gov/geo/tiger/TIGER2024/addrfeat/`, where files use the pattern
+ * `tl_2024_<statefips><countyfips>_addrfeat.zip`.
  *
- *   TIGER ADDRFEAT 2024 source:
- *
- *   - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
- *   - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
- *
- *   Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
- *   `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
- *   already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
- *   this module is download + provenance only.)
- *
- *   Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
- *   county ZIP to disk (no curl subprocess).
+ * Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
+ * `manifest.json` recording filename, sha256 and bytes so re-runs skip already-verified files.
+ * Extraction and ogr2ogr ingestion happen later, in the `tiger` adapter.
  */
 /* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+   procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
 import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters";
-import { statPath, pathExists } from "@mailwoman/core/fs/readers";
+import { statPath, pathExists } from "@mailwoman/core/fs/readers/stat";
 import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
 import { sha256File } from "@mailwoman/core/hash";
 import { isoSeconds } from "@mailwoman/core/utils";
 import { sleep } from "@mailwoman/core/utils/sleep";
-import { basename, join } from "path-ts";
-import { readManifest, streamDownload, writeManifest } from "#tools/fetch/download/index";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-/**
- * Lowest 2xx status; anything below is informational.
- */
+import { readManifest, streamDownload, writeManifest } from "#tools/fetch/download";
 const HTTP_OK = 200;
-/**
- * Lowest 3xx status; at or above it the response is a redirect or an error, not a body.
- */
 const HTTP_REDIRECT = 300;
 const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT";
-/**
- * Read a per-state MANIFEST.json into a filename → entry map.
- */
 async function readCountyManifest(manifestPath) {
     const map = new Map();
     const parsed = await readManifest(manifestPath);
@@ -55,9 +34,6 @@ async function readCountyManifest(manifestPath) {
     }
     return map;
 }
-/**
- * Check whether a file already matches a recorded sha256 and byte count.
- */
 async function fileMatchesSha(path, expectedSha, expectedBytes) {
     if (!(await pathExists(path)))
         return false;
@@ -65,11 +41,8 @@ async function fileMatchesSha(path, expectedSha, expectedBytes) {
         return false;
     return (await sha256File(path)) === expectedSha;
 }
-/**
- * Download one county ZIP (size sanity check + sha256).
- */
 async function downloadCounty(url, dest) {
-    const filename = basename(dest);
+    const filename = dest.basename();
     const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 });
     if (status < HTTP_OK || status >= HTTP_REDIRECT) {
         return { ok: false, filename, reason: `HTTP ${status}` };
@@ -83,16 +56,14 @@ async function downloadCounty(url, dest) {
     return { ok: true, filename, sha256, bytes };
 }
 export async function fetchTigerFull(options, report) {
-    const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length > 0);
+    const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length);
     const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000);
     const maxParallel = options.maxParallel ?? 4;
     const dryRun = options.dryRun ?? false;
-    const addrfeatDir = join(options.outRoot, "tiger", "addrfeat");
+    const addrfeatDir = options.outRoot("tiger", "addrfeat");
     await makeDirectories(addrfeatDir);
     // MARK: Step 1 — discover the county file list
     report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`);
-    // `responseType: "text"` — an Apache directory index, scraped below. The per-county ARCHIVE downloads
-    // stay on raw `fetch` (they stream to disk; see `downloadOne`).
     const listingRes = await new APIClient({
         displayName: "tiger-listing",
         retry: true,
@@ -104,8 +75,7 @@ export async function fetchTigerFull(options, report) {
     const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted();
     const totalCounties = allZips.length;
     report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`);
-    // Build a map: state_fips -> list of filenames.
-    // tl_2024_SSCCC_addrfeat.zip — SS = 2-digit state FIPS (chars 8-9), CCC = county FIPS.
+    // `tl_2024_SSCCC_addrfeat.zip`: SS is the 2-digit state FIPS at chars 8-9.
     const stateFiles = new Map();
     for (const fname of allZips) {
         const stateFips = fname.slice(8, 10);
@@ -121,29 +91,24 @@ export async function fetchTigerFull(options, report) {
     let totalFailed = 0;
     let totalBytesFetched = 0;
     const failedCodes = [];
-    // Process states in sorted FIPS order for predictable output.
     const sortedStates = [...stateFiles.keys()].toSorted();
     for (const stateFips of sortedStates) {
         const countyFiles = stateFiles.get(stateFips) ?? [];
-        // Respect an explicit request to skip this state before scheduling its counties.
         if (skipStateFips.includes(stateFips)) {
             report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`);
             totalSkippedState += countyFiles.length;
             continue;
         }
-        const stateDir = join(addrfeatDir, `state-${stateFips}`);
+        const stateDir = addrfeatDir(`state-${stateFips}`);
         await makeDirectories(stateDir);
-        const manifestPath = join(stateDir, "MANIFEST.json");
-        // Load existing manifest for O(1) verified-skip lookup.
+        const manifestPath = stateDir("MANIFEST.json");
         const manifest = await readCountyManifest(manifestPath);
         report?.(`--- State ${stateFips} — ${countyFiles.length} counties`);
-        // Build a list of URLs+dests that need fetching.
         const pending = [];
         for (const fname of countyFiles) {
-            const dest = join(stateDir, fname);
+            const dest = stateDir(fname);
             const url = `${TIGER_BASE_URL}/${fname}`;
             const known = manifest.get(fname);
-            // Skip if already verified via MANIFEST.
             if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
                 report?.(`  skip (verified) ${fname}`);
                 totalSkipped++;
@@ -160,7 +125,6 @@ export async function fetchTigerFull(options, report) {
             continue;
         if (!pending.length)
             continue;
-        // Download pending counties with bounded parallelism and rate-limit spacing.
         const results = Array.from({ length: pending.length });
         let cursor = 0;
         const workers = Array.from({ length: Math.min(maxParallel, pending.length) }, async () => {
@@ -169,13 +133,11 @@ export async function fetchTigerFull(options, report) {
                 if (i >= pending.length)
                     return;
                 const item = pending[i];
-                // Rate-limit: polite spacing before each fetch.
                 await sleep(rateSleepMs);
                 results[i] = await downloadCounty(item.url, item.dest);
             }
         });
         await Promise.all(workers);
-        // Collect results from this state.
         for (const result of results) {
             if (result.ok) {
                 report?.(`  ok ${result.filename}  ${ByteFormatter.formatIEC(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`);
@@ -189,7 +151,6 @@ export async function fetchTigerFull(options, report) {
                 failedCodes.push(result.filename);
             }
         }
-        // Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
         const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename));
         const manifestDoc = {
             state_fips: stateFips,

package/out/us/tools/fetch/tiger-full.js.map

@@ -1 +1 @@
-{"version":3,"file":"tiger-full.js","sourceRoot":"","sources":["../../../../lib/us/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE;;;;;;;;;;;;;;;;;;;GAmBG;AACH;kGACkG;AAClG,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC5E,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AACnD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAGxC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAEzF;;;GAGG;AACH;;GAEG;AACH,MAAM,OAAO,GAAG,GAAG,CAAA;AAEnB;;GAEG;AACH,MAAM,aAAa,GAAG,GAAG,CAAA;AAEzB,MAAM,cAAc,GAAG,sDAAsD,CAAA;AA4B7E;;GAEG;AACH,KAAK,UAAU,kBAAkB,CAAC,YAAoB;IACrD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAuB,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,YAAY,CAA+B,YAAY,CAAC,CAAA;IAE7E,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,QAAQ,IAAI,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YAChB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;QAChF,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,WAAmB,EAAE,aAAqB;IACrF,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAA;IAE3C,IAAI,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa;QAAE,OAAO,KAAK,CAAA;IAE/D,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,WAAW,CAAA;AAChD,CAAC;AAMD;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY;IACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAC/B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtG,IAAI,MAAM,GAAG,OAAO,IAAI,MAAM,IAAI,aAAa,EAAE,CAAC;QACjD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,MAAM,EAAE,EAAE,CAAA;IACzD,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;IAEzC,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;QAC3B,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAE/B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,KAAK,SAAS,EAAE,CAAA;IACrE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAErC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,aAAa,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACtG,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;IACjE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IAEtC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,CAAA;IAC9D,MAAM,eAAe,CAAC,WAAW,CAAC,CAAA;IAElC,+CAA+C;IAE/C,MAAM,EAAE,CAAC,uDAAuD,CAAC,CAAA;IAEjE,sGAAsG;IACtG,gEAAgE;IAChE,MAAM,UAAU,GAAG,MAAM,IAAI,SAAS,CAAC;QACtC,WAAW,EAAE,eAAe;QAC5B,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;KACrD,CAAC;SACA,KAAK,CAAS,EAAE,GAAG,EAAE,GAAG,cAAc,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SACnF,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,MAAM,IAAI,GAAG,UAAU,CAAA;IACvB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;IACtF,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAA;IACpC,MAAM,EAAE,CAAC,WAAW,aAAa,gDAAgD,CAAC,CAAA;IAElF,gDAAgD;IAChD,uFAAuF;IACvF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE9C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACpC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAC5C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChB,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAChC,CAAC;IAED,MAAM,EAAE,CAAC,WAAW,UAAU,CAAC,IAAI,8BAA8B,CAAC,CAAA;IAElE,0CAA0C;IAE1C,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,8DAA8D;IAC9D,MAAM,YAAY,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEtD,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAEnD,iFAAiF;QACjF,IAAI,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,MAAM,EAAE,CAAC,aAAa,SAAS,qCAAqC,WAAW,CAAC,MAAM,YAAY,CAAC,CAAA;YACnG,iBAAiB,IAAI,WAAW,CAAC,MAAM,CAAA;YAEvC,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,SAAS,SAAS,EAAE,CAAC,CAAA;QACxD,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;QAC/B,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAA;QAEpD,wDAAwD;QACxD,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,CAAA;QAEvD,MAAM,EAAE,CAAC,aAAa,SAAS,MAAM,WAAW,CAAC,MAAM,WAAW,CAAC,CAAA;QAEnE,iDAAiD;QACjD,MAAM,OAAO,GAAyC,EAAE,CAAA;QAExD,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YAClC,MAAM,GAAG,GAAG,GAAG,cAAc,IAAI,KAAK,EAAE,CAAA;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAEjC,yCAAyC;YACzC,IAAI,KAAK,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACtE,MAAM,EAAE,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAA;gBAEtC,YAAY,EAAE,CAAA;gBAEd,SAAQ;YACT,CAAC;YAED,IAAI,MAAM,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAA;gBAEjC,YAAY,EAAE,CAAA;gBAEd,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;QAC5B,CAAC;QAED,IAAI,MAAM;YAAE,SAAQ;QAEpB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,SAAQ;QAE7B,6EAA6E;QAC7E,MAAM,OAAO,GAAmB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QACtE,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;YACxF,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;gBAElB,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM;oBAAE,OAAM;gBAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;gBACxB,gDAAgD;gBAChD,MAAM,KAAK,CAAC,WAAW,CAAC,CAAA;gBACxB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACvD,CAAC;QACF,CAAC,CAAC,CAAA;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAE1B,mCAAmC;QACnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,EAAE,CACP,QAAQ,MAAM,CAAC,QAAQ,KAAK,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAC5G,CAAA;gBAED,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;gBAExG,YAAY,EAAE,CAAA;gBACd,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAA;YAClC,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,UAAU,MAAM,CAAC,QAAQ,OAAO,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;gBAEzD,WAAW,EAAE,CAAA;gBACb,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YAClC,CAAC;QACF,CAAC;QAED,yFAAyF;QACzF,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;QAEhG,MAAM,WAAW,GAAG;YACnB,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,UAAU,EAAE;YACxB,cAAc,EAAE,cAAc;YAC9B,QAAQ;SACR,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;IAC/C,CAAC;IAED,gBAAgB;IAEhB,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,iCAAiC,aAAa,EAAE,CAAC,CAAA;IAC1D,MAAM,EAAE,CAAC,iCAAiC,iBAAiB,wBAAwB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,WAAW,EAAE,CAAC,CAAA;IAExD,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,CAAC,iCAAiC,aAAa,CAAC,SAAS,CAAC,iBAAiB,CAAC,KAAK,iBAAiB,GAAG,CAAC,CAAA;IAC/G,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,EAAE,CAAC,YAAY,WAAW,uCAAuC,CAAC,CAAA;IACzE,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,GAAG,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAA;AAC9G,CAAC"}
+{"version":3,"file":"tiger-full.js","sourceRoot":"","sources":["../../../../lib/us/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE;;;;;;;;;;;;GAYG;AACH;4GAC4G;AAC5G,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC5E,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AACtE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AAInD,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEnF,MAAM,OAAO,GAAG,GAAG,CAAA;AACnB,MAAM,aAAa,GAAG,GAAG,CAAA;AAEzB,MAAM,cAAc,GAAG,sDAAsD,CAAA;AAe7E,KAAK,UAAU,kBAAkB,CAAC,YAAyB;IAC1D,MAAM,GAAG,GAAG,IAAI,GAAG,EAAuB,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,YAAY,CAA+B,YAAY,CAAC,CAAA;IAE7E,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,QAAQ,IAAI,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YAChB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;QAChF,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAiB,EAAE,WAAmB,EAAE,aAAqB;IAC1F,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAA;IAE3C,IAAI,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa;QAAE,OAAO,KAAK,CAAA;IAE/D,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,WAAW,CAAA;AAChD,CAAC;AAMD,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAiB;IAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAA;IAChC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtG,IAAI,MAAM,GAAG,OAAO,IAAI,MAAM,IAAI,aAAa,EAAE,CAAC;QACjD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,MAAM,EAAE,EAAE,CAAA;IACzD,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;IAEzC,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;QAC3B,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAE/B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,KAAK,SAAS,EAAE,CAAA;IACrE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAErC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,aAAa,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAClG,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;IACjE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IAEtC,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAA;IACxD,MAAM,eAAe,CAAC,WAAW,CAAC,CAAA;IAElC,+CAA+C;IAE/C,MAAM,EAAE,CAAC,uDAAuD,CAAC,CAAA;IAEjE,MAAM,UAAU,GAAG,MAAM,IAAI,SAAS,CAAC;QACtC,WAAW,EAAE,eAAe;QAC5B,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;KACrD,CAAC;SACA,KAAK,CAAS,EAAE,GAAG,EAAE,GAAG,cAAc,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SACnF,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,MAAM,IAAI,GAAG,UAAU,CAAA;IACvB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;IACtF,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAA;IACpC,MAAM,EAAE,CAAC,WAAW,aAAa,gDAAgD,CAAC,CAAA;IAElF,2EAA2E;IAC3E,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE9C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACpC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAC5C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChB,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAChC,CAAC;IAED,MAAM,EAAE,CAAC,WAAW,UAAU,CAAC,IAAI,8BAA8B,CAAC,CAAA;IAElE,0CAA0C;IAE1C,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,MAAM,YAAY,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEtD,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAEnD,IAAI,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,MAAM,EAAE,CAAC,aAAa,SAAS,qCAAqC,WAAW,CAAC,MAAM,YAAY,CAAC,CAAA;YACnG,iBAAiB,IAAI,WAAW,CAAC,MAAM,CAAA;YAEvC,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,SAAS,EAAE,CAAC,CAAA;QAClD,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;QAC/B,MAAM,YAAY,GAAG,QAAQ,CAAC,eAAe,CAAC,CAAA;QAE9C,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,CAAA;QAEvD,MAAM,EAAE,CAAC,aAAa,SAAS,MAAM,WAAW,CAAC,MAAM,WAAW,CAAC,CAAA;QAEnE,MAAM,OAAO,GAA8C,EAAE,CAAA;QAE7D,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;YAC5B,MAAM,GAAG,GAAG,GAAG,cAAc,IAAI,KAAK,EAAE,CAAA;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAEjC,IAAI,KAAK,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACtE,MAAM,EAAE,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAA;gBAEtC,YAAY,EAAE,CAAA;gBAEd,SAAQ;YACT,CAAC;YAED,IAAI,MAAM,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAA;gBAEjC,YAAY,EAAE,CAAA;gBAEd,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;QAC5B,CAAC;QAED,IAAI,MAAM;YAAE,SAAQ;QAEpB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,SAAQ;QAE7B,MAAM,OAAO,GAAmB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QACtE,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;YACxF,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;gBAElB,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM;oBAAE,OAAM;gBAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;gBACxB,MAAM,KAAK,CAAC,WAAW,CAAC,CAAA;gBACxB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACvD,CAAC;QACF,CAAC,CAAC,CAAA;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAE1B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,EAAE,CACP,QAAQ,MAAM,CAAC,QAAQ,KAAK,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAC5G,CAAA;gBAED,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;gBAExG,YAAY,EAAE,CAAA;gBACd,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAA;YAClC,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,UAAU,MAAM,CAAC,QAAQ,OAAO,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;gBAEzD,WAAW,EAAE,CAAA;gBACb,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YAClC,CAAC;QACF,CAAC;QAED,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;QAEhG,MAAM,WAAW,GAAG;YACnB,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,UAAU,EAAE;YACxB,cAAc,EAAE,cAAc;YAC9B,QAAQ;SACR,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;IAC/C,CAAC;IAED,gBAAgB;IAEhB,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,iCAAiC,aAAa,EAAE,CAAC,CAAA;IAC1D,MAAM,EAAE,CAAC,iCAAiC,iBAAiB,wBAAwB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,WAAW,EAAE,CAAC,CAAA;IAExD,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,CAAC,iCAAiC,aAAa,CAAC,SAAS,CAAC,iBAAiB,CAAC,KAAK,iBAAiB,GAAG,CAAC,CAAA;IAC/G,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,EAAE,CAAC,YAAY,WAAW,uCAAuC,CAAC,CAAA;IACzE,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,GAAG,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAA;AAC9G,CAAC"}

package/out/utils/align.d.ts

@@ -19,17 +19,16 @@
  *        character range. First token in a component span → `B-<tag>`; subsequent tokens →
  *        `I-<tag>`; no overlap → `O`.
  *   5. Emit the located char spans verbatim as `span_starts[]` / `span_ends[]` / `span_tags[]` (the
- *        v0.5.0 char-offset format, #519). The token quantization in step 4 is the part the v0.5.0
- *        rebuild deletes; during the transition both representations ride on every labeled row.
+ *        v0.5.0 char-offset format).
  *
  *   Structural invariants the function preserves (the span ones loudly — a violation throws rather
- *   than quarantines, because it indicates a bug here, not bad source data):
+ *   than quarantines, because it indicates a bug here rather than bad source data):
  *
  *   - `tokens.length === labels.length` always.
  *   - Each component contributes at most one contiguous BIO run (no `B-tag … O … I-tag` gaps). This is
  *       enforced by greedy first-match span assignment + ordered token iteration.
  *   - The span triple is sorted ascending by start and non-overlapping.
- *   - `raw` is NFC-normalized (asserted per row; a non-NFC raw makes char offsets ambiguous downstream
+ *   - `raw` is NFC-normalized (asserted per row. a non-NFC raw makes char offsets ambiguous downstream
  *       — NFD `é` occupies two code units where NFC `é` occupies one — and silently so).
  */
 import type { ComponentTag } from "@mailwoman/codex/component";
@@ -40,20 +39,29 @@ import { type Tokenizer } from "#utils/tokenize";
  */
 export interface AlignOptions {
     /**
-     * Tokenizer to use. Defaults to `whitespaceTokenizer()`.
+     * Tokenizer to use.
+     *
+     * @defaultValue `whitespaceTokenizer()`
      */
     tokenizer?: Tokenizer;
     /**
-     * Max Levenshtein edit distance to accept when a verbatim substring match fails. Set `0` to require verbatim matches
-     * only. Default `2`.
+     * Max Levenshtein edit distance to accept when a verbatim substring match fails.
+     *
+     * Set `0` to require verbatim matches only.
+     *
+     * Distance is computed against same-length windows in `raw`, so the threshold
+     * scales naturally with the component value length.
      *
-     * Distance is computed against same-length windows in `raw`, so the threshold scales naturally with the component
-     * value length.
+     * @defaultValue `2`
      */
     maxEditDistance?: number;
     /**
-     * Case-insensitive comparison for substring search. Default `true`. The retained span in `raw` is the original case;
-     * only matching is case-insensitive.
+     * Case-insensitive comparison for substring search.
+     *
+     * The retained span in `raw` is the original case.
+     * Only matching is case-insensitive.
+     *
+     * @defaultValue `true`
      */
     caseInsensitive?: boolean;
 }
@@ -68,8 +76,7 @@ export type AlignmentResult = {
     row: QuarantinedRow;
 };
 /**
- * One located char-offset label span over a row's `raw` ([start, end) in UTF-16 code units). The element type behind
- * the parallel `span_starts[]`/`span_ends[]`/`span_tags[]` triple on `LabeledRow` (#519).
+ * One located char-offset label span over a row's `raw` ([start, end) in UTF-16 code units). The element type behind the parallel `span_starts[]`/`span_ends[]`/`span_tags[]` triple on `LabeledRow`.
  */
 export interface ComponentSpan {
     tag: ComponentTag;
@@ -81,13 +88,13 @@ export interface ComponentSpan {
  */
 export declare function alignRow(row: CanonicalRow, opts?: AlignOptions): AlignmentResult;
 /**
- * Enforce the #519 span-triple invariants — in-bounds, sorted ascending by start, non-overlapping — loudly.
+ * Enforce the span-triple invariants: in-bounds, sorted ascending by start, non-overlapping.
+ *
+ * For `alignRow`: `claimed`-span bookkeeping in `locateSpan` already makes overlap impossible
+ * and the caller sorts, so a violation here is a bug in this file rather than bad source data.
+ * Throw rather than quarantine, so the corruption cannot ride into a corpus.
  *
- * For `alignRow`: `claimed`-span bookkeeping in `locateSpan` already makes overlap impossible and the caller sorts, so
- * a violation here is a bug in this file, not bad source data: throw (naming the row) rather than quarantine, so the
- * corruption can't ride into a corpus. Exported for every OTHER span producer (`composeAdversarialRow`'s offset
- * arithmetic, future synthesis paths) — any code that emits the triple without going through `alignRow` must pass its
- * output through this.
+ * Any code that emits the triple without going through `alignRow` must pass its output through this.
  */
 export declare function assertSpanInvariants(spans: readonly ComponentSpan[], row: Pick<CanonicalRow, "raw" | "source" | "source_id">): void;
 //# sourceMappingURL=align.d.ts.map

package/out/utils/align.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"align.d.ts","sourceRoot":"","sources":["../../lib/utils/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAY,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAGxE,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAA;AACtE,OAAO,EAAuC,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAErF;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,CAAA;IAErB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IAExB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,UAAU,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,GAAG,EAAE,cAAc,CAAA;CAAE,CAAA;AAEjH;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC7B,GAAG,EAAE,YAAY,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACX;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,IAAI,GAAE,YAAiB,GAAG,eAAe,CAsFpF;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CACnC,KAAK,EAAE,SAAS,aAAa,EAAE,EAC/B,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,GAAG,QAAQ,GAAG,WAAW,CAAC,GACrD,IAAI,CA4BN"}
+{"version":3,"file":"align.d.ts","sourceRoot":"","sources":["../../lib/utils/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,KAAK,EAAY,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAIxE,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAA;AACtE,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAErE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B;;;;OAIG;IACH,SAAS,CAAC,EAAE,SAAS,CAAA;IAErB;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IAExB;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,OAAO,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,UAAU,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,GAAG,EAAE,cAAc,CAAA;CAAE,CAAA;AAEjH;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,GAAG,EAAE,YAAY,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACX;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,IAAI,GAAE,YAAiB,GAAG,eAAe,CA2EpF;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CACnC,KAAK,EAAE,SAAS,aAAa,EAAE,EAC/B,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,GAAG,QAAQ,GAAG,WAAW,CAAC,GACrD,IAAI,CA4BN"}

package/out/utils/align.js

@@ -19,17 +19,16 @@
  *        character range. First token in a component span → `B-<tag>`; subsequent tokens →
  *        `I-<tag>`; no overlap → `O`.
  *   5. Emit the located char spans verbatim as `span_starts[]` / `span_ends[]` / `span_tags[]` (the
- *        v0.5.0 char-offset format, #519). The token quantization in step 4 is the part the v0.5.0
- *        rebuild deletes; during the transition both representations ride on every labeled row.
+ *        v0.5.0 char-offset format).
  *
  *   Structural invariants the function preserves (the span ones loudly — a violation throws rather
- *   than quarantines, because it indicates a bug here, not bad source data):
+ *   than quarantines, because it indicates a bug here rather than bad source data):
  *
  *   - `tokens.length === labels.length` always.
  *   - Each component contributes at most one contiguous BIO run (no `B-tag … O … I-tag` gaps). This is
  *       enforced by greedy first-match span assignment + ordered token iteration.
  *   - The span triple is sorted ascending by start and non-overlapping.
- *   - `raw` is NFC-normalized (asserted per row; a non-NFC raw makes char offsets ambiguous downstream
+ *   - `raw` is NFC-normalized (asserted per row. a non-NFC raw makes char offsets ambiguous downstream
  *       — NFD `é` occupies two code units where NFC `é` occupies one — and silently so).
  */
 import { distance as levenshteinDistance } from "fastest-levenshtein";
@@ -44,10 +43,6 @@ export function alignRow(row, opts = {}) {
     if (!row.raw) {
         return { kind: "quarantined", row: { row, reason: "raw-empty" } };
     }
-    // #519 NFC handling — relaxed from a hard throw to normalization (2026-06-12, DeepSeek-validated):
-    // one non-NFC row (e.g. a non-Latin name variant like "দক্ষিণ কোরিয়া") must not crash a multi-hour
-    // build. Normalize `raw` AND every component value to NFC, compute spans over the NFC raw, and
-    // store the NFC raw — preserving the #519 single-normalization-form principle while keeping the row.
     const raw = row.raw.normalize("NFC");
     const components = { ...row.components };
     for (const key of Object.keys(components)) {
@@ -59,10 +54,9 @@ export function alignRow(row, opts = {}) {
     const componentSpans = [];
     const claimed = [];
     const haystack = caseInsensitive ? raw.toLowerCase() : raw;
-    // Longest value first: a short component must not claim a word that a longer, more specific
-    // component owns ("Alaska Regional Dr, Alaska" — region "Alaska" stealing the street's first
-    // word quarantined the street; pilot2's residual class). Emit order is unaffected — spans are
-    // re-sorted by start below.
+    // Longest value first: a short component must not claim a word that a longer,
+    // more specific component owns.
+    // Emit order is unaffected, since spans are re-sorted by start below.
     const entries = Object.entries(components).toSorted((a, b) => (b[1]?.length ?? 0) - (a[1]?.length ?? 0));
     for (const [tag, value] of entries) {
         if (!value)
@@ -75,12 +69,8 @@ export function alignRow(row, opts = {}) {
                 row: { row, reason: `component-not-found:${tag}` },
             };
         }
-        // Defensive bounds quarantine (2026-06-12): locateSpan's fuzzy/boundary logic can over-run the
-        // raw by a code unit on some non-Latin / combining-mark strings (e.g. Bengali name variants),
-        // yielding a span past the end. An out-of-bounds offset can never be a valid char span —
-        // quarantine the row rather than crash assertSpanInvariants and take down a multi-hour build.
-        // (If this class proves large in the quarantine report, locateSpan's boundary logic needs a
-        // combining-mark fix to KEEP these non-Latin rows.)
+        // An out-of-bounds offset can never be a valid char span, so quarantine the row
+        // rather than crash assertSpanInvariants and take down a multi-hour build.
         if (span.start < 0 || span.end > raw.length || span.start >= span.end) {
             return {
                 kind: "quarantined",
@@ -100,8 +90,6 @@ export function alignRow(row, opts = {}) {
         components,
         tokens: tokens.map((t) => t.text),
         labels,
-        // The v0.5.0 char-offset triple (#519): the located spans, emitted verbatim. The token
-        // quantization above is what the rebuild deletes; both ride during the transition.
         span_starts: componentSpans.map((s) => s.start),
         span_ends: componentSpans.map((s) => s.end),
         span_tags: componentSpans.map((s) => s.tag),
@@ -109,13 +97,13 @@ export function alignRow(row, opts = {}) {
     return { kind: "labeled", row: labeled };
 }
 /**
- * Enforce the #519 span-triple invariants — in-bounds, sorted ascending by start, non-overlapping — loudly.
+ * Enforce the span-triple invariants: in-bounds, sorted ascending by start, non-overlapping.
  *
- * For `alignRow`: `claimed`-span bookkeeping in `locateSpan` already makes overlap impossible and the caller sorts, so
- * a violation here is a bug in this file, not bad source data: throw (naming the row) rather than quarantine, so the
- * corruption can't ride into a corpus. Exported for every OTHER span producer (`composeAdversarialRow`'s offset
- * arithmetic, future synthesis paths) — any code that emits the triple without going through `alignRow` must pass its
- * output through this.
+ * For `alignRow`: `claimed`-span bookkeeping in `locateSpan` already makes overlap impossible
+ * and the caller sorts, so a violation here is a bug in this file rather than bad source data.
+ * Throw rather than quarantine, so the corruption cannot ride into a corpus.
+ *
+ * Any code that emits the triple without going through `alignRow` must pass its output through this.
  */
 export function assertSpanInvariants(spans, row) {
     for (let i = 0; i < spans.length; i++) {
@@ -138,23 +126,24 @@ export function assertSpanInvariants(spans, row) {
     }
 }
 /**
- * Locate `needle` in `haystack` (both already normalized for case if requested), preferring verbatim substring match.
- * Falls back to a fuzzy window scan when verbatim fails and `maxEditDistance > 0`. Already-claimed spans are skipped so
- * two components don't grab overlapping ranges.
+ * Locate `needle` in `haystack` (both already normalized for case if requested),
+ * preferring verbatim substring match.
+ *
+ * Falls back to a fuzzy window scan when verbatim fails and `maxEditDistance > 0`.
+ * Already-claimed spans are skipped so two components don't grab overlapping ranges.
  *
- * Returns the span in the original `raw` (not the lower-cased `haystack`).
+ * @returns The span in the original `raw` (not the lower-cased `haystack`).
  */
 function locateSpan(args) {
     const { haystack, needle, claimed, maxEditDistance } = args;
-    if (!needle.length)
-        return undefined;
-    // Pass 1: verbatim substring. Word-boundary-aligned matches are PREFERRED over intra-word ones
-    // — leftmost-substring alone let a short value claim the inside of an earlier word (region "AK"
-    // matched inside "Umak"/"Lake", scrambling every later span; caught by the v0.5.0 pilot build).
+    if (!needle)
+        return null;
+    // Pass 1: verbatim substring.
+    // Word-boundary-aligned matches are preferred over intra-word ones, so a short
+    // value cannot claim the inside of an earlier word.
     // Intra-word matches stay allowed as the fallback because they are essential for affix
-    // supervision (street_suffix "straße" inside "Hauptstraße" has no boundary-aligned occurrence —
-    // sub-word spans are the point of the char-offset format).
-    let intraWord;
+    // supervision (street_suffix "straße" inside "Hauptstraße" has no boundary-aligned occurrence).
+    let intraWord = null;
     let from = 0;
     while (true) {
         const idx = haystack.indexOf(needle, from);
@@ -171,21 +160,20 @@ function locateSpan(args) {
     if (intraWord)
         return intraWord;
     if (maxEditDistance <= 0)
-        return undefined;
-    // Pass 2: fuzzy sliding-window. Walk over candidate windows of length `needle.length`
-    // across haystack, compute Levenshtein, pick the leftmost window under the threshold.
+        return null;
+    // Pass 2: fuzzy sliding-window.
     const len = needle.length;
     for (let i = 0; i + len <= haystack.length; i++) {
         if (overlapsClaimed(i, i + len, claimed))
             continue;
         const window = haystack.slice(i, i + len);
         if (window === needle)
-            return { start: i, end: i + len }; // covered by pass 1, but cheap
+            return { start: i, end: i + len };
         const d = levenshteinDistance(window, needle);
         if (d <= maxEditDistance)
             return { start: i, end: i + len };
     }
-    return undefined;
+    return null;
 }
 const WORD_CHAR = /[\p{L}\p{N}]/u;
 /**
@@ -204,14 +192,11 @@ function overlapsClaimed(start, end, claimed) {
     return false;
 }
 /**
- * Assign BIO labels to tokens given the component spans. Components MUST be sorted by start offset. For each token,
- * find the first component span that contains the token's start offset; if the token is the first one inside that span
- * emit `B-<tag>`, else `I-<tag>`.
+ * Assign BIO labels to tokens from component spans sorted by start offset.
  */
 function labelTokens(tokens, spans) {
     const out = [];
     const seenSpan = new Set();
-    // index into `spans`
     for (const token of tokens) {
         let assigned = "O";
         for (let i = 0; i < spans.length; i++) {

package/out/utils/align.js.map

@@ -1 +1 @@
-{"version":3,"file":"align.js","sourceRoot":"","sources":["../../lib/utils/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAGH,OAAO,EAAE,QAAQ,IAAI,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AAGrE,OAAO,EAAE,mBAAmB,EAAkC,MAAM,iBAAiB,CAAA;AA0CrF;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAiB,EAAE,IAAI,GAAiB,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IACzD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;IACjD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAA;IAClE,CAAC;IAED,mGAAmG;IACnG,oGAAoG;IACpG,+FAA+F;IAC/F,qGAAqG;IACrG,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAExC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,GAAG,UAAU,CAAC,GAA8B,CAAC,CAAA;QAEpD,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,CAAC,GAA8B,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QAChE,CAAC;IACF,CAAC;IAED,MAAM,cAAc,GAAoB,EAAE,CAAA;IAC1C,MAAM,OAAO,GAA4B,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;IAE1D,4FAA4F;IAC5F,6FAA6F;IAC7F,8FAA8F;IAC9F,4BAA4B;IAC5B,MAAM,OAAO,GAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAA+C,CAAC,QAAQ,CACjG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CACnD,CAAA;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;QAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;QAE5E,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,uBAAuB,GAAG,EAAE,EAAE;aAClD,CAAA;QACF,CAAC;QAED,+FAA+F;QAC/F,8FAA8F;QAC9F,yFAAyF;QACzF,8FAA8F;QAC9F,4FAA4F;QAC5F,oDAAoD;QACpD,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACvE,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,sBAAsB,GAAG,EAAE,EAAE;aACjD,CAAA;QACF,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAChD,oBAAoB,CAAC,cAAc,EAAE,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAElD,MAAM,OAAO,GAAe;QAC3B,GAAG,GAAG;QACN,GAAG;QACH,UAAU;QACV,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjC,MAAM;QACN,uFAAuF;QACvF,mFAAmF;QACnF,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KAC3C,CAAA;IAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CACnC,KAA+B,EAC/B,GAAuD;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QAEnB,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CACd,wCAAwC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAClF,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,wBAAwB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CACvE,CAAA;QACF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC;YAAE,SAAQ;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAE1B,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACd,sCAAsC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAChF,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;QAED,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACd,mCAAmC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAC7E,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED;;;;;;GAMG;AACH,SAAS,UAAU,CAAC,IAMnB;IACA,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CAAA;IAE3D,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,SAAS,CAAA;IAEpC,+FAA+F;IAC/F,gGAAgG;IAChG,gGAAgG;IAChG,uFAAuF;IACvF,gGAAgG;IAChG,2DAA2D;IAC3D,IAAI,SAAqD,CAAA;IACzD,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAE1C,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAK;QACrB,MAAM,GAAG,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;QAE/B,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACzC,IAAI,iBAAiB,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC;gBAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;YACrE,SAAS,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;QAClC,CAAC;QAED,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;IACf,CAAC;IAED,IAAI,SAAS;QAAE,OAAO,SAAS,CAAA;IAE/B,IAAI,eAAe,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IAE1C,sFAAsF;IACtF,sFAAsF;IACtF,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,OAAO,CAAC;YAAE,SAAQ;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAA;QAEzC,IAAI,MAAM,KAAK,MAAM;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA,CAAC,+BAA+B;QACxF,MAAM,CAAC,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAE7C,IAAI,CAAC,IAAI,eAAe;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA;IAC5D,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,SAAS,GAAG,eAAe,CAAA;AAEjC;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAgB,EAAE,KAAa,EAAE,GAAW;IACtE,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,CAAA;IACnE,MAAM,KAAK,GAAG,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAE,CAAC,CAAA;IAExE,OAAO,MAAM,IAAI,KAAK,CAAA;AACvB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,GAAW,EAAE,OAAgC;IACpF,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG;YAAE,OAAO,IAAI,CAAA;IACtC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,MAA4B,EAAE,KAA+B;IACjF,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAElC,qBAAqB;IAErB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,QAAQ,GAAa,GAAG,CAAA;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YAEnB,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;gBAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtB,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;oBACnC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBAChB,CAAC;qBAAM,CAAC;oBACP,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;gBACpC,CAAC;gBAED,MAAK;YACN,CAAC;QACF,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACnB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
+{"version":3,"file":"align.js","sourceRoot":"","sources":["../../lib/utils/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAIH,OAAO,EAAE,QAAQ,IAAI,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AAGrE,OAAO,EAAE,mBAAmB,EAAkB,MAAM,iBAAiB,CAAA;AAkDrE;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAiB,EAAE,IAAI,GAAiB,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IACzD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;IACjD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAExC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,GAAG,UAAU,CAAC,GAA8B,CAAC,CAAA;QAEpD,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,CAAC,GAA8B,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QAChE,CAAC;IACF,CAAC;IAED,MAAM,cAAc,GAAoB,EAAE,CAAA;IAC1C,MAAM,OAAO,GAA4B,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;IAE1D,8EAA8E;IAC9E,gCAAgC;IAChC,sEAAsE;IACtE,MAAM,OAAO,GAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAA+C,CAAC,QAAQ,CACjG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CACnD,CAAA;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;QAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;QAE5E,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,uBAAuB,GAAG,EAAE,EAAE;aAClD,CAAA;QACF,CAAC;QAED,gFAAgF;QAChF,2EAA2E;QAC3E,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACvE,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,sBAAsB,GAAG,EAAE,EAAE;aACjD,CAAA;QACF,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAChD,oBAAoB,CAAC,cAAc,EAAE,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAElD,MAAM,OAAO,GAAe;QAC3B,GAAG,GAAG;QACN,GAAG;QACH,UAAU;QACV,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjC,MAAM;QACN,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KAC3C,CAAA;IAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CACnC,KAA+B,EAC/B,GAAuD;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QAEnB,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CACd,wCAAwC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAClF,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,wBAAwB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CACvE,CAAA;QACF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC;YAAE,SAAQ;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAE1B,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACd,sCAAsC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAChF,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;QAED,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACd,mCAAmC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAC7E,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,UAAU,CAAC,IAMnB;IACA,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CAAA;IAE3D,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IAExB,8BAA8B;IAC9B,+EAA+E;IAC/E,oDAAoD;IACpD,uFAAuF;IACvF,gGAAgG;IAChG,IAAI,SAAS,GAA0C,IAAI,CAAA;IAC3D,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAE1C,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAK;QACrB,MAAM,GAAG,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;QAE/B,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACzC,IAAI,iBAAiB,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC;gBAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;YACrE,SAAS,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;QAClC,CAAC;QAED,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;IACf,CAAC;IAED,IAAI,SAAS;QAAE,OAAO,SAAS,CAAA;IAE/B,IAAI,eAAe,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,gCAAgC;IAChC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,OAAO,CAAC;YAAE,SAAQ;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAA;QAEzC,IAAI,MAAM,KAAK,MAAM;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA;QACxD,MAAM,CAAC,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAE7C,IAAI,CAAC,IAAI,eAAe;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA;IAC5D,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,SAAS,GAAG,eAAe,CAAA;AAEjC;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAgB,EAAE,KAAa,EAAE,GAAW;IACtE,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,CAAA;IACnE,MAAM,KAAK,GAAG,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAE,CAAC,CAAA;IAExE,OAAO,MAAM,IAAI,KAAK,CAAA;AACvB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,GAAW,EAAE,OAAgC;IACpF,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG;YAAE,OAAO,IAAI,CAAA;IACtC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,MAA2B,EAAE,KAA+B;IAChF,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAElC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,QAAQ,GAAa,GAAG,CAAA;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YAEnB,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;gBAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtB,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;oBACnC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBAChB,CAAC;qBAAM,CAAC;oBACP,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;gBACpC,CAAC;gBAED,MAAK;YACN,CAAC;QACF,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACnB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}

package/out/utils/cooccurrence-stats.d.ts

@@ -0,0 +1,58 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Token/label co-occurrence accounting shared by the corpus-stats builder (`tools/corpus-stats.ts`)
+ *   and the recipe-output linter (`tools/lint/recipe-output/index.ts`). The two sides must agree on the bigram key
+ *   format: the linter looks its recipe output's bigrams up in the stats file's `bigrams` table. A key built with
+ *   a different separator never matches — which silently blanks the bigram-collision check.
+ */
+/**
+ * Separator inside a bigram key (`tok1␟tok2`) and a label-bigram value (`lab1␟lab2`):
+ * U+001F unit separator, a character no address token contains.
+ *
+ * Render a key for humans with `key.split(COOCCURRENCE_KEY_SEP).join(" ")`.
+ */
+export declare const COOCCURRENCE_KEY_SEP = "\u001F";
+/**
+ * Nested tally: outer key → (inner key → count).
+ *
+ * Token → label counts, or bigram key → label-bigram counts.
+ */
+type CooccurrenceTable = Map<string, Map<string, number>>;
+/**
+ * The token + bigram co-occurrence tables one pass over labeled rows accumulates.
+ */
+export interface CooccurrenceStats {
+    tokens: CooccurrenceTable;
+    bigrams: CooccurrenceTable;
+}
+/**
+ * An empty pair of tables for {@link accumulateCooccurrences}.
+ */
+export declare function createCooccurrenceStats(): CooccurrenceStats;
+/**
+ * Fold one row's parallel `tokens`/`labels` into the tables.
+ *
+ * The caller has already checked the arrays are the same length.
+ */
+export declare function accumulateCooccurrences(stats: CooccurrenceStats, tokens: readonly string[], labels: readonly string[]): void;
+/**
+ * A parquet row projected to the two columns the co-occurrence pass reads.
+ */
+export interface TokenLabelRow {
+    tokens: string[];
+    labels: string[];
+    [key: string]: unknown;
+}
+/**
+ * Stream a parquet file's `tokens`/`labels` columns.
+ *
+ * Projected rather than read whole: parquet is columnar, so the unused columns are never touched.
+ * `limit` stops the iteration rather than filtering afterwards, so a capped
+ * run reads only the row groups it needs.
+ */
+export declare function streamTokenLabelRows(parquetPath: string, limit?: number): AsyncIterable<TokenLabelRow>;
+export {};
+//# sourceMappingURL=cooccurrence-stats.d.ts.map

package/out/utils/cooccurrence-stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cooccurrence-stats.d.ts","sourceRoot":"","sources":["../../lib/utils/cooccurrence-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,WAAM,CAAA;AAEvC;;;;GAIG;AACH,KAAK,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;AAEzD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,iBAAiB,CAAA;IACzB,OAAO,EAAE,iBAAiB,CAAA;CAC1B;AAED;;GAEG;AACH,wBAAgB,uBAAuB,IAAI,iBAAiB,CAE3D;AAaD;;;;GAIG;AACH,wBAAgB,uBAAuB,CACtC,KAAK,EAAE,iBAAiB,EACxB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,MAAM,EAAE,SAAS,MAAM,EAAE,GACvB,IAAI,CAWN;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED;;;;;;GAMG;AACH,wBAAuB,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,aAAa,CAAC,CAQ7G"}

package/out/utils/cooccurrence-stats.js

@@ -0,0 +1,62 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Token/label co-occurrence accounting shared by the corpus-stats builder (`tools/corpus-stats.ts`)
+ *   and the recipe-output linter (`tools/lint/recipe-output/index.ts`). The two sides must agree on the bigram key
+ *   format: the linter looks its recipe output's bigrams up in the stats file's `bigrams` table. A key built with
+ *   a different separator never matches — which silently blanks the bigram-collision check.
+ */
+import { openParquetRowStream } from "#parquet/streams";
+/**
+ * Separator inside a bigram key (`tok1␟tok2`) and a label-bigram value (`lab1␟lab2`):
+ * U+001F unit separator, a character no address token contains.
+ *
+ * Render a key for humans with `key.split(COOCCURRENCE_KEY_SEP).join(" ")`.
+ */
+export const COOCCURRENCE_KEY_SEP = "";
+/**
+ * An empty pair of tables for {@link accumulateCooccurrences}.
+ */
+export function createCooccurrenceStats() {
+    return { tokens: new Map(), bigrams: new Map() };
+}
+function bump(table, key, sub) {
+    let counter = table.get(key);
+    if (!counter) {
+        counter = new Map();
+        table.set(key, counter);
+    }
+    counter.set(sub, (counter.get(sub) ?? 0) + 1);
+}
+/**
+ * Fold one row's parallel `tokens`/`labels` into the tables.
+ *
+ * The caller has already checked the arrays are the same length.
+ */
+export function accumulateCooccurrences(stats, tokens, labels) {
+    for (let i = 0; i < tokens.length; i++) {
+        const tk = tokens[i];
+        const lb = labels[i];
+        bump(stats.tokens, tk, lb);
+        if (i + 1 < tokens.length) {
+            bump(stats.bigrams, tk + COOCCURRENCE_KEY_SEP + tokens[i + 1], lb + COOCCURRENCE_KEY_SEP + labels[i + 1]);
+        }
+    }
+}
+/**
+ * Stream a parquet file's `tokens`/`labels` columns.
+ *
+ * Projected rather than read whole: parquet is columnar, so the unused columns are never touched.
+ * `limit` stops the iteration rather than filtering afterwards, so a capped
+ * run reads only the row groups it needs.
+ */
+export async function* streamTokenLabelRows(parquetPath, limit) {
+    let emitted = 0;
+    for await (const row of openParquetRowStream(parquetPath, { columns: ["tokens", "labels"], limit })) {
+        yield row;
+        emitted++;
+    }
+}
+//# sourceMappingURL=cooccurrence-stats.js.map