@mailwoman/corpus 2.0.0

package/out/src/align.js

@@ -0,0 +1,139 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Alignment: turn a `CanonicalRow` (raw + components) into a `LabeledRow` (raw + tokens + BIO
+ *   labels) or a `QuarantinedRow` (raw + reason) per the Phase 1 plan.
+ *
+ *   Pipeline:
+ *
+ *   1. For each `(tag, value)` in `components`, find the value's character span in `raw`. First try a
+ *        verbatim substring match (case-insensitive, whitespace-collapsed). If that fails, fall
+ *        back to fuzzy match via `fastest-levenshtein`, with a tunable edit distance threshold.
+ *   2. If any component cannot be located, reject the row with a human-readable reason and send it to
+ *        the quarantine pile (`reason: "component-not-found:<tag>"` or
+ *        `"edit-distance-exceeded:<tag>:<dist>"`).
+ *   3. Tokenize `raw` with the supplied `Tokenizer` (defaults to the whitespace tokenizer).
+ *   4. For each token: walk the list of component spans, pick the one whose span contains the token's
+ *        character range. First token in a component span → `B-<tag>`; subsequent tokens →
+ *        `I-<tag>`; no overlap → `O`.
+ *
+ *   Two structural invariants the function preserves:
+ *
+ *   - `tokens.length === labels.length` always.
+ *   - Each component contributes at most one contiguous BIO run (no `B-tag … O … I-tag` gaps). This is
+ *       enforced by greedy first-match span assignment + ordered token iteration.
+ */
+import { distance as levenshteinDistance } from "fastest-levenshtein";
+import { whitespaceTokenizer } from "./tokenize.js";
+/** Align a single row. */
+export function alignRow(row, opts = {}) {
+    const tokenizer = opts.tokenizer ?? whitespaceTokenizer();
+    const maxEditDistance = opts.maxEditDistance ?? 2;
+    const caseInsensitive = opts.caseInsensitive ?? true;
+    if (!row.raw) {
+        return { kind: "quarantined", row: { row, reason: "raw-empty" } };
+    }
+    const componentSpans = [];
+    const claimed = [];
+    const haystack = caseInsensitive ? row.raw.toLowerCase() : row.raw;
+    for (const [tag, value] of Object.entries(row.components)) {
+        if (!value)
+            continue;
+        const needle = caseInsensitive ? value.toLowerCase() : value;
+        const span = locateSpan({ haystack, needle, raw: row.raw, claimed, maxEditDistance });
+        if (!span) {
+            return {
+                kind: "quarantined",
+                row: { row, reason: `component-not-found:${tag}` },
+            };
+        }
+        componentSpans.push({ tag, start: span.start, end: span.end });
+        claimed.push([span.start, span.end]);
+    }
+    componentSpans.sort((a, b) => a.start - b.start);
+    const tokens = tokenizer.tokenize(row.raw);
+    const labels = labelTokens(tokens, componentSpans);
+    const labeled = {
+        ...row,
+        tokens: tokens.map((t) => t.text),
+        labels,
+    };
+    return { kind: "labeled", row: labeled };
+}
+/**
+ * Locate `needle` in `haystack` (both already normalized for case if requested), preferring
+ * verbatim substring match. Falls back to a fuzzy window scan when verbatim fails and
+ * `maxEditDistance > 0`. Already-claimed spans are skipped so two components don't grab overlapping
+ * ranges.
+ *
+ * Returns the span in the original `raw` (not the lower-cased `haystack`).
+ */
+function locateSpan(args) {
+    const { haystack, needle, claimed, maxEditDistance } = args;
+    if (needle.length === 0)
+        return undefined;
+    // Pass 1: verbatim substring, leftmost non-claimed.
+    let from = 0;
+    while (true) {
+        const idx = haystack.indexOf(needle, from);
+        if (idx < 0)
+            break;
+        const end = idx + needle.length;
+        if (!overlapsClaimed(idx, end, claimed))
+            return { start: idx, end };
+        from = idx + 1;
+    }
+    if (maxEditDistance <= 0)
+        return undefined;
+    // Pass 2: fuzzy sliding-window. Walk over candidate windows of length `needle.length`
+    // across haystack, compute Levenshtein, pick the leftmost window under the threshold.
+    const len = needle.length;
+    for (let i = 0; i + len <= haystack.length; i++) {
+        if (overlapsClaimed(i, i + len, claimed))
+            continue;
+        const window = haystack.slice(i, i + len);
+        if (window === needle)
+            return { start: i, end: i + len }; // covered by pass 1, but cheap
+        const d = levenshteinDistance(window, needle);
+        if (d <= maxEditDistance)
+            return { start: i, end: i + len };
+    }
+    return undefined;
+}
+function overlapsClaimed(start, end, claimed) {
+    for (const [a, b] of claimed) {
+        if (start < b && a < end)
+            return true;
+    }
+    return false;
+}
+/**
+ * Assign BIO labels to tokens given the component spans. Components MUST be sorted by start offset.
+ * For each token, find the first component span that contains the token's start offset; if the
+ * token is the first one inside that span emit `B-<tag>`, else `I-<tag>`.
+ */
+function labelTokens(tokens, spans) {
+    const out = [];
+    const seenSpan = new Set(); // index into `spans`
+    for (const token of tokens) {
+        let assigned = "O";
+        for (let i = 0; i < spans.length; i++) {
+            const s = spans[i];
+            if (token.start >= s.start && token.end <= s.end) {
+                if (!seenSpan.has(i)) {
+                    assigned = `B-${s.tag}`;
+                    seenSpan.add(i);
+                }
+                else {
+                    assigned = `I-${s.tag}`;
+                }
+                break;
+            }
+        }
+        out.push(assigned);
+    }
+    return out;
+}
+//# sourceMappingURL=align.js.map

package/out/src/align.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"align.js","sourceRoot":"","sources":["../../src/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAGH,OAAO,EAAE,QAAQ,IAAI,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AACrE,OAAO,EAAE,mBAAmB,EAAkC,MAAM,eAAe,CAAA;AAiCnF,0BAA0B;AAC1B,MAAM,UAAU,QAAQ,CAAC,GAAiB,EAAE,OAAqB,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IACzD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;IACjD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,cAAc,GAAoB,EAAE,CAAA;IAC1C,MAAM,OAAO,GAA4B,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAA;IAElE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAA8C,EAAE,CAAC;QACxG,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;QAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;QAErF,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,uBAAuB,GAAG,EAAE,EAAE;aAClD,CAAA;QACF,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAChD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAC1C,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAElD,MAAM,OAAO,GAAe;QAC3B,GAAG,GAAG;QACN,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjC,MAAM;KACN,CAAA;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,UAAU,CAAC,IAMnB;IACA,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CAAA;IAC3D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IAEzC,oDAAoD;IACpD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,IAAI,GAAG,GAAG,CAAC;YAAE,MAAK;QAClB,MAAM,GAAG,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;QAC/B,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC;YAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;QACnE,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;IACf,CAAC;IAED,IAAI,eAAe,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IAE1C,sFAAsF;IACtF,sFAAsF;IACtF,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,OAAO,CAAC;YAAE,SAAQ;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAA;QACzC,IAAI,MAAM,KAAK,MAAM;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA,CAAC,+BAA+B;QACxF,MAAM,CAAC,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAC7C,IAAI,CAAC,IAAI,eAAe;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA;IAC5D,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,GAAW,EAAE,OAAgC;IACpF,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG;YAAE,OAAO,IAAI,CAAA;IACtC,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,MAA4B,EAAE,KAA+B;IACjF,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA,CAAC,qBAAqB;IAExD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,QAAQ,GAAa,GAAG,CAAA;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YACnB,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;gBAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtB,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;oBACnC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBAChB,CAAC;qBAAM,CAAC;oBACP,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;gBACpC,CAAC;gBACD,MAAK;YACN,CAAC;QACF,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACnB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}

package/out/src/build.d.ts

@@ -0,0 +1,104 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   End-to-end corpus build (Phase 1 task #10 in the plan).
+ *
+ *   `buildCorpus(opts)` orchestrates every stage of the pipeline:
+ *
+ *   1. **Adapter runs** — drives every adapter in turn (via `runAdapter`), writing
+ *        `<intermediate>/<adapter.id>/canonical.jsonl` shards.
+ *   2. **Synthesis** — optional. For each canonical row, every applicable augmentation in the row's
+ *        country-default policy emits an augmented row alongside the original.
+ *   3. **Alignment** — every row (original + augmented) is aligned via `alignRow`. Successes go to
+ *        `labeled.jsonl`; quarantines are appended to `quarantine.jsonl` with reasons.
+ *   4. **Splits** — `splitRows` partitions labeled `source_id`s into train/val/test by locality holdout.
+ *        Manifest written to `splits/SPLIT_MANIFEST.json` + per-split `train.txt` / `val.txt` /
+ *        `test.txt`.
+ *   5. **Parquet shards** — `writeShards` streams labeled rows into 1M-row `.parquet` shards per split
+ *        under `corpus-v<version>/{train,val,test}/part-NNNN.parquet` (SNAPPY-compressed, 50k-row
+ *        row groups), with per-shard checksums + per-stage manifest in
+ *        `corpus-v<version>/MANIFEST.json`.
+ *   6. **Top-level manifest** — `<outputDir>/MANIFEST.json` ties every per-stage manifest together with
+ *        a top-level corpus_version, built_at, and aggregate counts.
+ *
+ *   Output layout:
+ *
+ *   ```
+ *   <outputDir>/
+ *   MANIFEST.json
+ *   intermediate/
+ *     <adapter.id>/canonical.jsonl   # one per adapter
+ *     labeled.jsonl                  # post-alignment, pre-shard
+ *     quarantine.jsonl               # rows that failed alignment
+ *   splits/
+ *     SPLIT_MANIFEST.json
+ *     train.txt / val.txt / test.txt
+ *   corpus-v<version>/
+ *     MANIFEST.json
+ *     train/part-NNNN.parquet
+ *     val/part-NNNN.parquet
+ *     test/part-NNNN.parquet
+ * ```
+ *
+ *   The intermediate files live alongside the final shards for reproducibility + debugging. Operators
+ *   can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
+ *   self-contained.
+ */
+import { type ShardManifest } from "./parquet.js";
+import { type AdapterRunManifest } from "./runner.js";
+import { type SplitManifest } from "./split.js";
+import type { AdapterOptions, CorpusAdapter } from "./types.js";
+/** Stage tags surfaced to `onProgress`. */
+export type BuildStage = "adapter-run" | "align" | "split" | "shard" | "manifest";
+/** Per-invocation options for `buildCorpus`. */
+export interface BuildCorpusOptions {
+    /** Root output directory. All build artifacts land beneath it. */
+    outputDir: string;
+    /** Corpus version (e.g. `"0.1.0"`). Stamped onto every row + into the output dir name. */
+    corpusVersion: string;
+    /**
+     * Adapters to drive, in order. Defaults to `defaultAdapterRegistry.list()`. Pass an explicit list
+     * to filter (e.g. `[wofAdminAdapter]` for a smoke run).
+     */
+    adapters?: readonly CorpusAdapter[];
+    /**
+     * Per-adapter `AdapterOptions` — looked up by adapter id. Adapters whose id is missing from this
+     * map are skipped (and noted in the manifest).
+     */
+    adapterInputs: Record<string, AdapterOptions>;
+    /** Enable synthesis pass. Default `true`. Set `false` for fixture-driven smoke tests. */
+    synthesize?: boolean;
+    /** Forwarded to `writeShards`. Default 1_000_000. */
+    rowsPerShard?: number;
+    /** Progress hook. Errors thrown abort the build. */
+    onProgress?: (stage: BuildStage, message: string) => void;
+}
+/** Top-level manifest tying every stage together. */
+export interface BuildCorpusManifest {
+    corpus_version: string;
+    built_at: string;
+    adapters: AdapterRunManifest[];
+    skipped_adapters: string[];
+    splits: {
+        counts: SplitManifest["counts"];
+        holdouts: SplitManifest["holdouts"];
+    };
+    shards: {
+        counts: ShardManifest["counts"];
+        total_rows: number;
+    };
+    quarantine_count: number;
+    total_aligned_rows: number;
+}
+/**
+ * Drive the full corpus build to completion.
+ *
+ * Memory profile: the function maintains an in-memory `Map<source_id, SplitName>` to bridge the
+ * align → shard hand-off. For Phase 1 fixture-scale runs (≤ 10⁴ rows) this is trivial. For real 5M+
+ * runs, the map fits comfortably in a few hundred MB; the canonical.jsonl and labeled.jsonl
+ * payloads stream and never sit in memory.
+ */
+export declare function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpusManifest>;
+//# sourceMappingURL=build.d.ts.map

package/out/src/build.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAQH,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACjE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,YAAY,CAAA;AAEzF,2CAA2C;AAC3C,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IAClC,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAA;IAEjB,0FAA0F;IAC1F,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C,yFAAyF;IACzF,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACzD;AAED,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;CAC1B;AAED;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAgIxF"}

package/out/src/build.js

@@ -0,0 +1,201 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   End-to-end corpus build (Phase 1 task #10 in the plan).
+ *
+ *   `buildCorpus(opts)` orchestrates every stage of the pipeline:
+ *
+ *   1. **Adapter runs** — drives every adapter in turn (via `runAdapter`), writing
+ *        `<intermediate>/<adapter.id>/canonical.jsonl` shards.
+ *   2. **Synthesis** — optional. For each canonical row, every applicable augmentation in the row's
+ *        country-default policy emits an augmented row alongside the original.
+ *   3. **Alignment** — every row (original + augmented) is aligned via `alignRow`. Successes go to
+ *        `labeled.jsonl`; quarantines are appended to `quarantine.jsonl` with reasons.
+ *   4. **Splits** — `splitRows` partitions labeled `source_id`s into train/val/test by locality holdout.
+ *        Manifest written to `splits/SPLIT_MANIFEST.json` + per-split `train.txt` / `val.txt` /
+ *        `test.txt`.
+ *   5. **Parquet shards** — `writeShards` streams labeled rows into 1M-row `.parquet` shards per split
+ *        under `corpus-v<version>/{train,val,test}/part-NNNN.parquet` (SNAPPY-compressed, 50k-row
+ *        row groups), with per-shard checksums + per-stage manifest in
+ *        `corpus-v<version>/MANIFEST.json`.
+ *   6. **Top-level manifest** — `<outputDir>/MANIFEST.json` ties every per-stage manifest together with
+ *        a top-level corpus_version, built_at, and aggregate counts.
+ *
+ *   Output layout:
+ *
+ *   ```
+ *   <outputDir>/
+ *   MANIFEST.json
+ *   intermediate/
+ *     <adapter.id>/canonical.jsonl   # one per adapter
+ *     labeled.jsonl                  # post-alignment, pre-shard
+ *     quarantine.jsonl               # rows that failed alignment
+ *   splits/
+ *     SPLIT_MANIFEST.json
+ *     train.txt / val.txt / test.txt
+ *   corpus-v<version>/
+ *     MANIFEST.json
+ *     train/part-NNNN.parquet
+ *     val/part-NNNN.parquet
+ *     test/part-NNNN.parquet
+ * ```
+ *
+ *   The intermediate files live alongside the final shards for reproducibility + debugging. Operators
+ *   can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
+ *   self-contained.
+ */
+import { createReadStream, createWriteStream } from "node:fs";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { createInterface } from "node:readline";
+import { defaultAdapterRegistry } from "./adapter.js";
+import { alignRow } from "./align.js";
+import { writeShards } from "./parquet.js";
+import { runAdapter } from "./runner.js";
+import { defaultHoldouts, splitForRow, writeSplitManifestsFromLabeledFiles, } from "./split.js";
+import { defaultAugmentationsForCountry, synthesizeRow } from "./synthesize.js";
+/**
+ * Drive the full corpus build to completion.
+ *
+ * Memory profile: the function maintains an in-memory `Map<source_id, SplitName>` to bridge the
+ * align → shard hand-off. For Phase 1 fixture-scale runs (≤ 10⁴ rows) this is trivial. For real 5M+
+ * runs, the map fits comfortably in a few hundred MB; the canonical.jsonl and labeled.jsonl
+ * payloads stream and never sit in memory.
+ */
+export async function buildCorpus(opts) {
+    const adapters = opts.adapters ?? defaultAdapterRegistry.list();
+    const synthesize = opts.synthesize ?? true;
+    const rowsPerShard = opts.rowsPerShard ?? 1_000_000;
+    const built_at = new Date().toISOString();
+    await mkdir(opts.outputDir, { recursive: true });
+    const intermediateDir = join(opts.outputDir, "intermediate");
+    await mkdir(intermediateDir, { recursive: true });
+    // 1. Adapter runs.
+    const adapterRuns = [];
+    const skipped = [];
+    for (const adapter of adapters) {
+        const adapterOptions = opts.adapterInputs[adapter.id];
+        if (!adapterOptions) {
+            skipped.push(adapter.id);
+            opts.onProgress?.("adapter-run", `skipped ${adapter.id} (no input configured)`);
+            continue;
+        }
+        opts.onProgress?.("adapter-run", `running ${adapter.id}`);
+        const m = await runAdapter({
+            adapter,
+            adapterOptions,
+            outputDir: intermediateDir,
+            corpusVersion: opts.corpusVersion,
+        });
+        adapterRuns.push(m);
+    }
+    // 2 + 3. Synthesis + alignment: stream every canonical.jsonl, optionally augment, align,
+    // and route each labeled row directly to its split-specific JSONL (`labeled-{train,val,test}.
+    // jsonl`). Memory cost is O(1) — the prior in-memory `splitInputs` array + `splitByIdMap`
+    // + `SplitManifest.{train,val,test}` arrays are gone; per-row split is decided inline via
+    // `splitForRow` (a pure function of source_id + region + holdout policy).
+    const labeledPaths = {
+        train: join(intermediateDir, "labeled-train.jsonl"),
+        val: join(intermediateDir, "labeled-val.jsonl"),
+        test: join(intermediateDir, "labeled-test.jsonl"),
+    };
+    const labeledStreams = {
+        train: createWriteStream(labeledPaths.train, { encoding: "utf8" }),
+        val: createWriteStream(labeledPaths.val, { encoding: "utf8" }),
+        test: createWriteStream(labeledPaths.test, { encoding: "utf8" }),
+    };
+    const quarantinePath = join(intermediateDir, "quarantine.jsonl");
+    const quarantineStream = createWriteStream(quarantinePath, { encoding: "utf8" });
+    let aligned = 0;
+    let quarantined = 0;
+    const counts = { train: 0, val: 0, test: 0 };
+    const holdouts = defaultHoldouts();
+    const writeQuarantine = (row, reason) => {
+        quarantineStream.write(`${JSON.stringify({ row, reason })}\n`);
+    };
+    for (const adapterRun of adapterRuns) {
+        opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`);
+        for await (const row of streamJsonl(adapterRun.jsonl_path)) {
+            const fanned = [row];
+            if (synthesize) {
+                for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
+                    fanned.push(aug);
+                }
+            }
+            for (const r of fanned) {
+                const result = alignRow(r);
+                if (result.kind === "labeled") {
+                    const split = splitForRow(result.row, holdouts);
+                    labeledStreams[split].write(`${JSON.stringify(result.row)}\n`);
+                    counts[split]++;
+                    aligned++;
+                }
+                else {
+                    writeQuarantine(r, result.row.reason);
+                    quarantined++;
+                }
+            }
+        }
+    }
+    for (const s of Object.values(labeledStreams))
+        s.end();
+    quarantineStream.end();
+    await Promise.all([...Object.values(labeledStreams).map(streamEnd), streamEnd(quarantineStream)]);
+    // 4. Splits — manifest derived by streaming the per-split labeled files; no in-memory
+    // source-id arrays. `sort(1)` from coreutils produces the deterministic per-split .txt
+    // manifests with disk spill for splits that exceed in-memory thresholds.
+    opts.onProgress?.("split", `splitting ${aligned} aligned rows`);
+    const splitsDir = join(opts.outputDir, "splits");
+    const splitCounts = await writeSplitManifestsFromLabeledFiles({
+        labeledPaths,
+        outputDir: splitsDir,
+        corpusVersion: opts.corpusVersion,
+        counts,
+        holdouts,
+    });
+    // 5. Parquet shards — per-split labeled JSONL streams in, sharded `.parquet` out. The prior
+    // `splitFor(source_id)` callback (and the `Map<source_id, SplitName>` behind it) is gone.
+    opts.onProgress?.("shard", "writing parquet shards");
+    const shardManifest = await writeShards({
+        train: streamJsonl(labeledPaths.train),
+        val: streamJsonl(labeledPaths.val),
+        test: streamJsonl(labeledPaths.test),
+    }, {
+        outputDir: opts.outputDir,
+        corpusVersion: opts.corpusVersion,
+        rowsPerShard,
+    });
+    // 6. Top-level manifest.
+    opts.onProgress?.("manifest", "writing top-level MANIFEST.json");
+    const manifest = {
+        corpus_version: opts.corpusVersion,
+        built_at,
+        adapters: adapterRuns,
+        skipped_adapters: skipped,
+        splits: { counts: splitCounts, holdouts },
+        shards: { counts: shardManifest.counts, total_rows: shardManifest.total_rows },
+        quarantine_count: quarantined,
+        total_aligned_rows: aligned,
+    };
+    await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
+    return manifest;
+}
+async function* streamJsonl(path) {
+    const stream = createReadStream(path, { encoding: "utf8" });
+    const rl = createInterface({ input: stream, crlfDelay: Infinity });
+    for await (const line of rl) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        yield JSON.parse(trimmed);
+    }
+}
+function streamEnd(s) {
+    return new Promise((resolve, reject) => {
+        s.once("close", resolve);
+        s.once("error", reject);
+    });
+}
+//# sourceMappingURL=build.js.map

package/out/src/build.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAoB,MAAM,SAAS,CAAA;AAC/E,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAgD/E;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEjD,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QACrD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAC/E,SAAQ;QACT,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QACzD,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC;YAC1B,OAAO;YACP,cAAc;YACd,SAAS,EAAE,eAAe;YAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CAAA;QACF,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,yFAAyF;IACzF,8FAA8F;IAC9F,0FAA0F;IAC1F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,YAAY,GAA8B;QAC/C,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,qBAAqB,CAAC;QACnD,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,mBAAmB,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC;KACjD,CAAA;IACD,MAAM,cAAc,GAAmC;QACtD,KAAK,EAAE,iBAAiB,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAClE,GAAG,EAAE,iBAAiB,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC9D,IAAI,EAAE,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;KAChE,CAAA;IACD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAA;IAChE,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhF,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAElC,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QACnE,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YACpC,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBAC1B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;oBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAC9D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBACf,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBACrC,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;QAAE,CAAC,CAAC,GAAG,EAAE,CAAA;IACtD,gBAAgB,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAA;IAEjG,sFAAsF;IACtF,uFAAuF;IACvF,yEAAyE;IACzE,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,OAAO,eAAe,CAAC,CAAA;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IAChD,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS;QACpB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM;QACN,QAAQ;KACR,CAAC,CAAA;IAEF,4FAA4F;IAC5F,0FAA0F;IAC1F,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;IACpD,MAAM,aAAa,GAAG,MAAM,WAAW,CACtC;QACC,KAAK,EAAE,WAAW,CAAa,YAAY,CAAC,KAAK,CAAC;QAClD,GAAG,EAAE,WAAW,CAAa,YAAY,CAAC,GAAG,CAAC;QAC9C,IAAI,EAAE,WAAW,CAAa,YAAY,CAAC,IAAI,CAAC;KAChD,EACD;QACC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,YAAY;KACZ,CACD,CAAA;IAED,yBAAyB;IACzB,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAChE,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE;QAC9E,gBAAgB,EAAE,WAAW;QAC7B,kBAAkB,EAAE,OAAO;KAC3B,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACxG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAY;IAC1C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAClE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;IAC/B,CAAC;AACF,CAAC;AAED,SAAS,SAAS,CAAC,CAAc;IAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACxB,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACH,CAAC"}

package/out/src/codex/us-fips-state.d.ts

@@ -0,0 +1,44 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   US Census FIPS state codes → two-letter abbreviation + full display name.
+ *
+ *   Covers all 50 states + DC + the five primary territories (PR, GU, VI, MP, AS).
+ *
+ *   The data is public-domain US Census reference (FIPS PUB 5-2, withdrawn but still the canonical
+ *   numeric ID used by every TIGER product). It is reproduced here so the TIGER adapter can resolve
+ *   `statefp` columns (e.g. `"50"`) into a `region` component (e.g. `"VT"`) without an extra DB
+ *   join.
+ *
+ *   Salvaged 2026-05-17 from `isp-nexus/universe@6eeb7bd99643a6d62a8b8abbd50968a1e492b90b`
+ *   `tiger/state.ts` (originally copyright OpenISP, Inc.; both projects are AGPL-3.0). isp-nexus
+ *   ships these as TypeScript enums + a TypeORM-backed service layer; mailwoman only needs the flat
+ *   lookup so the file is a plain `Record` plus a small helper.
+ */
+/** Per-state record: two-letter postal abbreviation + full canonical display name. */
+export interface UsStateInfo {
+    abbreviation: string;
+    name: string;
+}
+/**
+ * FIPS state-or-territory code → `{ abbreviation, name }`. Includes all 50 states, DC, and the five
+ * primary territories (PR, GU, VI, MP, AS). Codes are two-digit zero-padded strings to match TIGER
+ * column `statefp`.
+ */
+export declare const US_FIPS_STATE: Readonly<Record<string, UsStateInfo>>;
+/** Lookup helper. Returns null when the FIPS code isn't recognized. */
+export declare function lookupFipsState(statefp: string | null | undefined): UsStateInfo | null;
+/**
+ * Inverted view: two-letter postal abbreviation → `UsStateInfo`. Built once at module load. Used by
+ * adapters whose source data ships the abbreviation rather than the FIPS code (FCC BDC, most
+ * federal CSVs).
+ */
+export declare const US_STATE_BY_ABBREVIATION: Readonly<Record<string, UsStateInfo>>;
+/**
+ * Lookup helper for adapters carrying 2-char USPS abbreviations (`"CA"`, `"VT"`). Case-folded; null
+ * for any value outside the 50 states + DC + the five primary territories.
+ */
+export declare function lookupStateAbbreviation(abbreviation: string | null | undefined): UsStateInfo | null;
+//# sourceMappingURL=us-fips-state.d.ts.map

package/out/src/codex/us-fips-state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"us-fips-state.d.ts","sourceRoot":"","sources":["../../../src/codex/us-fips-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,sFAAsF;AACtF,MAAM,WAAW,WAAW;IAC3B,YAAY,EAAE,MAAM,CAAA;IACpB,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CA0D9D,CAAA;AAEF,uEAAuE;AACvE,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,WAAW,GAAG,IAAI,CAGtF;AAED;;;;GAIG;AACH,eAAO,MAAM,wBAAwB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAE1E,CAAA;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,WAAW,GAAG,IAAI,CAGnG"}

package/out/src/codex/us-fips-state.js

@@ -0,0 +1,105 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   US Census FIPS state codes → two-letter abbreviation + full display name.
+ *
+ *   Covers all 50 states + DC + the five primary territories (PR, GU, VI, MP, AS).
+ *
+ *   The data is public-domain US Census reference (FIPS PUB 5-2, withdrawn but still the canonical
+ *   numeric ID used by every TIGER product). It is reproduced here so the TIGER adapter can resolve
+ *   `statefp` columns (e.g. `"50"`) into a `region` component (e.g. `"VT"`) without an extra DB
+ *   join.
+ *
+ *   Salvaged 2026-05-17 from `isp-nexus/universe@6eeb7bd99643a6d62a8b8abbd50968a1e492b90b`
+ *   `tiger/state.ts` (originally copyright OpenISP, Inc.; both projects are AGPL-3.0). isp-nexus
+ *   ships these as TypeScript enums + a TypeORM-backed service layer; mailwoman only needs the flat
+ *   lookup so the file is a plain `Record` plus a small helper.
+ */
+/**
+ * FIPS state-or-territory code → `{ abbreviation, name }`. Includes all 50 states, DC, and the five
+ * primary territories (PR, GU, VI, MP, AS). Codes are two-digit zero-padded strings to match TIGER
+ * column `statefp`.
+ */
+export const US_FIPS_STATE = Object.freeze({
+    "01": { abbreviation: "AL", name: "Alabama" },
+    "02": { abbreviation: "AK", name: "Alaska" },
+    "04": { abbreviation: "AZ", name: "Arizona" },
+    "05": { abbreviation: "AR", name: "Arkansas" },
+    "06": { abbreviation: "CA", name: "California" },
+    "08": { abbreviation: "CO", name: "Colorado" },
+    "09": { abbreviation: "CT", name: "Connecticut" },
+    "10": { abbreviation: "DE", name: "Delaware" },
+    "11": { abbreviation: "DC", name: "District of Columbia" },
+    "12": { abbreviation: "FL", name: "Florida" },
+    "13": { abbreviation: "GA", name: "Georgia" },
+    "15": { abbreviation: "HI", name: "Hawaii" },
+    "16": { abbreviation: "ID", name: "Idaho" },
+    "17": { abbreviation: "IL", name: "Illinois" },
+    "18": { abbreviation: "IN", name: "Indiana" },
+    "19": { abbreviation: "IA", name: "Iowa" },
+    "20": { abbreviation: "KS", name: "Kansas" },
+    "21": { abbreviation: "KY", name: "Kentucky" },
+    "22": { abbreviation: "LA", name: "Louisiana" },
+    "23": { abbreviation: "ME", name: "Maine" },
+    "24": { abbreviation: "MD", name: "Maryland" },
+    "25": { abbreviation: "MA", name: "Massachusetts" },
+    "26": { abbreviation: "MI", name: "Michigan" },
+    "27": { abbreviation: "MN", name: "Minnesota" },
+    "28": { abbreviation: "MS", name: "Mississippi" },
+    "29": { abbreviation: "MO", name: "Missouri" },
+    "30": { abbreviation: "MT", name: "Montana" },
+    "31": { abbreviation: "NE", name: "Nebraska" },
+    "32": { abbreviation: "NV", name: "Nevada" },
+    "33": { abbreviation: "NH", name: "New Hampshire" },
+    "34": { abbreviation: "NJ", name: "New Jersey" },
+    "35": { abbreviation: "NM", name: "New Mexico" },
+    "36": { abbreviation: "NY", name: "New York" },
+    "37": { abbreviation: "NC", name: "North Carolina" },
+    "38": { abbreviation: "ND", name: "North Dakota" },
+    "39": { abbreviation: "OH", name: "Ohio" },
+    "40": { abbreviation: "OK", name: "Oklahoma" },
+    "41": { abbreviation: "OR", name: "Oregon" },
+    "42": { abbreviation: "PA", name: "Pennsylvania" },
+    "44": { abbreviation: "RI", name: "Rhode Island" },
+    "45": { abbreviation: "SC", name: "South Carolina" },
+    "46": { abbreviation: "SD", name: "South Dakota" },
+    "47": { abbreviation: "TN", name: "Tennessee" },
+    "48": { abbreviation: "TX", name: "Texas" },
+    "49": { abbreviation: "UT", name: "Utah" },
+    "50": { abbreviation: "VT", name: "Vermont" },
+    "51": { abbreviation: "VA", name: "Virginia" },
+    "53": { abbreviation: "WA", name: "Washington" },
+    "54": { abbreviation: "WV", name: "West Virginia" },
+    "55": { abbreviation: "WI", name: "Wisconsin" },
+    "56": { abbreviation: "WY", name: "Wyoming" },
+    // Territories
+    "60": { abbreviation: "AS", name: "American Samoa" },
+    "66": { abbreviation: "GU", name: "Guam" },
+    "69": { abbreviation: "MP", name: "Northern Mariana Islands" },
+    "72": { abbreviation: "PR", name: "Puerto Rico" },
+    "78": { abbreviation: "VI", name: "Virgin Islands" },
+});
+/** Lookup helper. Returns null when the FIPS code isn't recognized. */
+export function lookupFipsState(statefp) {
+    if (!statefp)
+        return null;
+    return US_FIPS_STATE[statefp] ?? null;
+}
+/**
+ * Inverted view: two-letter postal abbreviation → `UsStateInfo`. Built once at module load. Used by
+ * adapters whose source data ships the abbreviation rather than the FIPS code (FCC BDC, most
+ * federal CSVs).
+ */
+export const US_STATE_BY_ABBREVIATION = Object.freeze(Object.fromEntries(Object.values(US_FIPS_STATE).map((info) => [info.abbreviation, info])));
+/**
+ * Lookup helper for adapters carrying 2-char USPS abbreviations (`"CA"`, `"VT"`). Case-folded; null
+ * for any value outside the 50 states + DC + the five primary territories.
+ */
+export function lookupStateAbbreviation(abbreviation) {
+    if (!abbreviation)
+        return null;
+    return US_STATE_BY_ABBREVIATION[abbreviation.toUpperCase()] ?? null;
+}
+//# sourceMappingURL=us-fips-state.js.map

package/out/src/codex/us-fips-state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"us-fips-state.js","sourceRoot":"","sources":["../../../src/codex/us-fips-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAQH;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAA0C,MAAM,CAAC,MAAM,CAAC;IACjF,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE;IAChD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;IACjD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,sBAAsB,EAAE;IAC1D,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;IAC/C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE;IACnD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;IAC/C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;IACjD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE;IACnD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE;IAChD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE;IAChD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE;IACpD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE;IAClD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE;IAClD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE;IAClD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE;IACpD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE;IAClD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;IAC/C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE;IAChD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE;IACnD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;IAC/C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,cAAc;IACd,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE;IACpD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,0BAA0B,EAAE;IAC9D,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;IACjD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE;CACpD,CAAC,CAAA;AAEF,uEAAuE;AACvE,MAAM,UAAU,eAAe,CAAC,OAAkC;IACjE,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IACzB,OAAO,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,CAAA;AACtC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAA0C,MAAM,CAAC,MAAM,CAC3F,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,CACzF,CAAA;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,YAAuC;IAC9E,IAAI,CAAC,YAAY;QAAE,OAAO,IAAI,CAAA;IAC9B,OAAO,wBAAwB,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,CAAA;AACpE,CAAC"}