@mailwoman/corpus 4.16.2 → 5.0.0

package/out/src/shard-recipes/house-venue.js

@@ -0,0 +1,66 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `house-venue` shard recipe — synthetic house_number+venue+street co-occurrence rows: tuples →
+ *   {@link synthesizeHouseVenueRow} → aligned LabeledRow. The v0.6.3 corrective companion to the
+ *   no-street shard: every row carries BOTH house_number AND venue, restoring the house_number
+ *   signal that no-street's distributional shift cost the model. Ported from
+ *   scripts/build-house-venue-shard.mjs.
+ */
+import { synthesizeHouseVenueRow } from "../synthesize-house-venue.js";
+import { alignAndWrite, makeLcg, readTuples, shardSourceID } from "./scaffold.js";
+const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples";
+export const houseVenueRecipe = {
+    name: "house-venue",
+    description: "House_number+venue co-occurrence rows: tuples → synthesizeHouseVenueRow → aligned LabeledRow",
+    mode: "tuples",
+    async run(opts, write) {
+        if (!opts.input)
+            throw new Error("house-venue recipe requires --input <tuples.jsonl>");
+        // Legacy build-house-venue-shard.mjs seeded the LCG via makeRandom(opts.seed) (s = seed).
+        const random = makeLcg(opts.seed);
+        const source = opts.sourceName ?? "synth-house-venue";
+        let read = 0;
+        let emitted = 0;
+        let skipped = 0;
+        for await (const tuple of readTuples(opts.input)) {
+            read++;
+            if (!tuple.locality || !tuple.region || !tuple.postcode || !tuple.country) {
+                skipped++;
+                continue;
+            }
+            for (let v = 0; v < opts.variants; v++) {
+                const synth = synthesizeHouseVenueRow(tuple, { random });
+                if (!synth) {
+                    skipped++;
+                    continue;
+                }
+                const ok = alignAndWrite(write, {
+                    raw: synth.raw,
+                    components: synth.components,
+                    country: tuple.country,
+                    locale: synth.locale,
+                    source,
+                    source_id: shardSourceID(source, {
+                        locality: tuple.locality,
+                        region: tuple.region,
+                        postcode: tuple.postcode,
+                        country: tuple.country,
+                        template: synth.template,
+                        v: String(v),
+                    }),
+                    corpus_version: "0.4.0",
+                    license: LICENSE,
+                }, synth.template);
+                if (ok)
+                    emitted++;
+                else
+                    skipped++;
+            }
+        }
+        return { read, emitted, skipped };
+    },
+};
+//# sourceMappingURL=house-venue.js.map

package/out/src/shard-recipes/house-venue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"house-venue.js","sourceRoot":"","sources":["../../../src/shard-recipes/house-venue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,uBAAuB,EAA4B,MAAM,8BAA8B,CAAA;AAChG,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAoB,MAAM,eAAe,CAAA;AAEnG,MAAM,OAAO,GAAG,6DAA6D,CAAA;AAE7E,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EAAE,8FAA8F;IAC3G,IAAI,EAAE,QAAQ;IACd,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;QACtF,0FAA0F;QAC1F,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,mBAAmB,CAAA;QACrD,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,IAAI,EAAE,CAAA;YAEN,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC3E,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,KAAK,GAAG,uBAAuB,CAAC,KAA4B,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;gBAE/E,IAAI,CAAC,KAAK,EAAE,CAAC;oBACZ,OAAO,EAAE,CAAA;oBACT,SAAQ;gBACT,CAAC;gBACD,MAAM,EAAE,GAAG,aAAa,CACvB,KAAK,EACL;oBACC,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM;oBACN,SAAS,EAAE,aAAa,CAAC,MAAM,EAAE;wBAChC,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,OAAO,EAAE,KAAK,CAAC,OAAO;wBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;qBACZ,CAAC;oBACF,cAAc,EAAE,OAAO;oBACvB,OAAO,EAAE,OAAO;iBAChB,EACD,KAAK,CAAC,QAAQ,CACd,CAAA;gBAED,IAAI,EAAE;oBAAE,OAAO,EAAE,CAAA;;oBACZ,OAAO,EAAE,CAAA;YACf,CAAC;QACF,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}

package/out/src/shard-recipes/index.d.ts

@@ -0,0 +1,18 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The synthetic-corpus SHARD RECIPE registry. Each recipe ({@link ShardRecipe}) is what one legacy
+ *   `build-*-shard.mjs` script used to be; the `mailwoman corpus shard <recipe>` command dispatches
+ *   to them. A new shard is a new entry here, not a new script.
+ */
+import type { ShardRecipe } from "./scaffold.js";
+export * from "./scaffold.js";
+/** Recipe name → recipe. */
+export declare const SHARD_RECIPES: ReadonlyMap<string, ShardRecipe>;
+/** Look up a recipe by its `<recipe>` name. */
+export declare function getShardRecipe(name: string): ShardRecipe | undefined;
+/** All recipes (for `--list` / help). */
+export declare function listShardRecipes(): readonly ShardRecipe[];
+//# sourceMappingURL=index.d.ts.map

package/out/src/shard-recipes/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAeH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAMhD,cAAc,eAAe,CAAA;AAuB7B,4BAA4B;AAC5B,eAAO,MAAM,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAA4C,CAAA;AAEvG,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEpE;AAED,yCAAyC;AACzC,wBAAgB,gBAAgB,IAAI,SAAS,WAAW,EAAE,CAEzD"}

package/out/src/shard-recipes/index.js

@@ -0,0 +1,58 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The synthetic-corpus SHARD RECIPE registry. Each recipe ({@link ShardRecipe}) is what one legacy
+ *   `build-*-shard.mjs` script used to be; the `mailwoman corpus shard <recipe>` command dispatches
+ *   to them. A new shard is a new entry here, not a new script.
+ */
+import { anchorAbsorptionRecipe } from "./anchor-absorption.js";
+import { boundaryStressRecipe } from "./boundary-stress.js";
+import { countryBalancedRecipe } from "./country-balanced.js";
+import { frAdminSplitRecipe } from "./fr-admin-split.js";
+import { frBareStreetRecipe } from "./fr-bare-street.js";
+import { frOrderRecipe } from "./fr-order.js";
+import { germanRecipe } from "./german.js";
+import { houseVenueRecipe } from "./house-venue.js";
+import { intersectionRecipe } from "./intersection.js";
+import { localeRecipe } from "./locale.js";
+import { noStreetRecipe } from "./no-street.js";
+import { poBoxCedexRecipe } from "./po-box-cedex.js";
+import { poBoxRecipe } from "./po-box.js";
+import { streetAffixRecipe } from "./street-affix.js";
+import { streetBareRecipe } from "./street-bare.js";
+import { streetRecipe } from "./street.js";
+import { unitRecipe } from "./unit.js";
+export * from "./scaffold.js";
+/** Every registered recipe, in display order. */
+const RECIPES = [
+    streetRecipe,
+    streetBareRecipe,
+    streetAffixRecipe,
+    noStreetRecipe,
+    houseVenueRecipe,
+    poBoxRecipe,
+    poBoxCedexRecipe,
+    unitRecipe,
+    intersectionRecipe,
+    germanRecipe,
+    localeRecipe,
+    frOrderRecipe,
+    frAdminSplitRecipe,
+    frBareStreetRecipe,
+    countryBalancedRecipe,
+    boundaryStressRecipe,
+    anchorAbsorptionRecipe,
+];
+/** Recipe name → recipe. */
+export const SHARD_RECIPES = new Map(RECIPES.map((r) => [r.name, r]));
+/** Look up a recipe by its `<recipe>` name. */
+export function getShardRecipe(name) {
+    return SHARD_RECIPES.get(name);
+}
+/** All recipes (for `--list` / help). */
+export function listShardRecipes() {
+    return RECIPES;
+}
+//# sourceMappingURL=index.js.map

package/out/src/shard-recipes/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,cAAc,eAAe,CAAA;AAE7B,iDAAiD;AACjD,MAAM,OAAO,GAA2B;IACvC,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,cAAc;IACd,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,qBAAqB;IACrB,oBAAoB;IACpB,sBAAsB;CACtB,CAAA;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,aAAa,GAAqC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAEvG,+CAA+C;AAC/C,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,gBAAgB;IAC/B,OAAO,OAAO,CAAA;AACf,CAAC"}

package/out/src/shard-recipes/intersection.d.ts

@@ -0,0 +1,38 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `intersection` shard recipe — the REAL-pair intersection training shard (#487). The model scored
+ *   0.0 on intersection_a/b because the training mix had ZERO intersection-labeled rows. This is
+ *   the missing data. Ported from scripts/build-intersection-shard.mjs.
+ *
+ *   STREET PAIRS ARE REAL: the same TIGER 2023 EDGES extraction as the eval builder
+ *   (scripts/eval/build-intersection-real.ts) — a node where two road edges (MTFCC S1*) with
+ *   distinct FULLNAMEs meet is a real crossing. Real pairs avoid teaching fake street-street
+ *   co-occurrences.
+ *
+ *   LEAKAGE POLICY (mirrors the affix shard's VT discipline):
+ *
+ *   - TRAIN counties: Cook IL (grid city) + Morris NJ (suburb).
+ *   - GOLDEN (`--golden`) county: Washington VT (rural) ONLY — the corpus defaultHoldout state.
+ *   - Every crossing in data/eval/external/intersection-real.jsonl is excluded from BOTH modes, by node
+ *       id AND by order-insensitive name pair (the eval shares all three counties).
+ *
+ *   RENDERING: junction-format variety — padded/TIGHT `&` and `/`, `and`, `at`, `@`, leading-phrase
+ *   `corner of` / `intersection of` — crossed with tails (bare / `, ST` / `, ST ZIP` / `, City, ST
+ *   [ZIP]`) and case variants. ZIPs are the crossing's own TIGER edge ZIPL (real); the locality
+ *   tail comes from the OA Cook-county ZIP→city majority map.
+ *
+ *   AUDIT: every emitted row is label-checked on the RAW SURFACE via the #519 char-offset span
+ *   triple. Any violation FAILS the build (throws). A JSON audit report lands next to the output.
+ *
+ *   External inputs (`--edges-dir`, opts.edgesDir; both already on disk — do not re-download):
+ *
+ *   - <edges-dir>/tl_2023_{17031,34027,50023}_edges.shp (unzipped TIGER 2023 EDGES; default
+ *       /tmp/tiger-edges)
+ *   - /tmp/oa-cache/us__il__cook.zip (ZIP→city tails)
+ */
+import { type ShardRecipe } from "./scaffold.js";
+export declare const intersectionRecipe: ShardRecipe;
+//# sourceMappingURL=intersection.d.ts.map

package/out/src/shard-recipes/intersection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"intersection.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/intersection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAYH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAsXhE,eAAO,MAAM,kBAAkB,EAAE,WAkKhC,CAAA"}

package/out/src/shard-recipes/intersection.js

@@ -0,0 +1,473 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `intersection` shard recipe — the REAL-pair intersection training shard (#487). The model scored
+ *   0.0 on intersection_a/b because the training mix had ZERO intersection-labeled rows. This is
+ *   the missing data. Ported from scripts/build-intersection-shard.mjs.
+ *
+ *   STREET PAIRS ARE REAL: the same TIGER 2023 EDGES extraction as the eval builder
+ *   (scripts/eval/build-intersection-real.ts) — a node where two road edges (MTFCC S1*) with
+ *   distinct FULLNAMEs meet is a real crossing. Real pairs avoid teaching fake street-street
+ *   co-occurrences.
+ *
+ *   LEAKAGE POLICY (mirrors the affix shard's VT discipline):
+ *
+ *   - TRAIN counties: Cook IL (grid city) + Morris NJ (suburb).
+ *   - GOLDEN (`--golden`) county: Washington VT (rural) ONLY — the corpus defaultHoldout state.
+ *   - Every crossing in data/eval/external/intersection-real.jsonl is excluded from BOTH modes, by node
+ *       id AND by order-insensitive name pair (the eval shares all three counties).
+ *
+ *   RENDERING: junction-format variety — padded/TIGHT `&` and `/`, `and`, `at`, `@`, leading-phrase
+ *   `corner of` / `intersection of` — crossed with tails (bare / `, ST` / `, ST ZIP` / `, City, ST
+ *   [ZIP]`) and case variants. ZIPs are the crossing's own TIGER edge ZIPL (real); the locality
+ *   tail comes from the OA Cook-county ZIP→city majority map.
+ *
+ *   AUDIT: every emitted row is label-checked on the RAW SURFACE via the #519 char-offset span
+ *   triple. Any violation FAILS the build (throws). A JSON audit report lands next to the output.
+ *
+ *   External inputs (`--edges-dir`, opts.edgesDir; both already on disk — do not re-download):
+ *
+ *   - <edges-dir>/tl_2023_{17031,34027,50023}_edges.shp (unzipped TIGER 2023 EDGES; default
+ *       /tmp/tiger-edges)
+ *   - /tmp/oa-cache/us__il__cook.zip (ZIP→city tails)
+ */
+import { spawnSync } from "node:child_process";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { repoRootPathBuilder } from "@mailwoman/core/utils";
+import { stableSourceID } from "../adapter.js";
+import { alignRow } from "../align.js";
+import { makeMulberry32 } from "./scaffold.js";
+const TRAIN_COUNTIES = [
+    { fips: "17031", state: "IL", regime: "grid-city" },
+    { fips: "34027", state: "NJ", regime: "suburb" },
+];
+const GOLDEN_COUNTIES = [{ fips: "50023", state: "VT", regime: "rural" }];
+const EVAL_GOLD_PATH = String(repoRootPathBuilder("data", "eval", "external", "intersection-real.jsonl"));
+const OA_COOK = { zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv" };
+const FORMS = [
+    { id: "amp", w: 0.2, render: (a, b) => `${a} & ${b}` },
+    { id: "and", w: 0.2, render: (a, b) => `${a} and ${b}` },
+    { id: "at", w: 0.12, render: (a, b) => `${a} at ${b}` },
+    { id: "slash", w: 0.08, render: (a, b) => `${a} / ${b}` },
+    { id: "slash-tight", w: 0.06, render: (a, b) => `${a}/${b}` },
+    { id: "amp-tight", w: 0.05, render: (a, b) => `${a}&${b}` },
+    { id: "at-sign", w: 0.06, render: (a, b) => `${a} @ ${b}` },
+    { id: "corner-of", w: 0.115, render: (a, b) => `corner of ${a} and ${b}` },
+    { id: "intersection-of", w: 0.115, render: (a, b) => `intersection of ${a} and ${b}` },
+];
+const TAILS = [
+    { id: "bare", w: 0.55 },
+    { id: "region", w: 0.16 },
+    { id: "region-zip", w: 0.09 },
+    { id: "city-region", w: 0.08 },
+    { id: "city-region-zip", w: 0.12 },
+];
+const CASES = [
+    { id: "as-is", w: 0.82, apply: (s) => s },
+    { id: "upper", w: 0.12, apply: (s) => s.toUpperCase() },
+    { id: "lower", w: 0.06, apply: (s) => s.toLowerCase() },
+];
+/**
+ * Words a connector may contribute as O tokens. The audit rejects any O token outside this set — an unlabeled
+ * street/locality token would surface here.
+ */
+const CONNECTOR_O_TOKENS = new Set(["and", "at", "of", "corner", "intersection"]);
+/**
+ * Street names that would make the connector ambiguous or break verbatim alignment: embedded connector punctuation, or
+ * a standalone "and"/"at" word.
+ */
+const BAD_NAME = /[,&@/]|\b(and|at)\b/i;
+/** Punctuation a connector form may leave between spans (besides whitespace): `, & @ /`. */
+const CONNECTOR_PUNCT_RE = /^[\s,&@/]*$/;
+function weightedPick(items, random) {
+    const total = items.reduce((s, x) => s + x.w, 0);
+    let r = random() * total;
+    for (const item of items) {
+        r -= item.w;
+        if (r <= 0)
+            return item;
+    }
+    return items[items.length - 1];
+}
+/** Minimal RFC-4180-ish splitter (handles quoted fields) — same as the affix builder. */
+function splitCSV(line) {
+    const out = [];
+    let cur = "";
+    let inQ = false;
+    for (let i = 0; i < line.length; i++) {
+        const c = line[i];
+        if (inQ) {
+            if (c === '"') {
+                if (line[i + 1] === '"') {
+                    cur += '"';
+                    i++;
+                }
+                else
+                    inQ = false;
+            }
+            else
+                cur += c;
+        }
+        else if (c === '"')
+            inQ = true;
+        else if (c === ",") {
+            out.push(cur);
+            cur = "";
+        }
+        else
+            cur += c;
+    }
+    out.push(cur);
+    return out;
+}
+/** Order-insensitive crossing key, for eval-leakage exclusion + pair dedup. */
+const pairKey = (a, b) => [a.toLowerCase(), b.toLowerCase()].sort().join("\x1f");
+/** Load the eval's crossings so neither train nor golden ever sees them. */
+function readEvalExclusions() {
+    const nodes = new Set();
+    const pairs = new Set();
+    if (!existsSync(EVAL_GOLD_PATH)) {
+        console.error(`  WARN: eval gold not found at ${EVAL_GOLD_PATH} — no eval-leakage exclusion applied`);
+        return { nodes, pairs };
+    }
+    for (const line of readFileSync(EVAL_GOLD_PATH, "utf8").split("\n")) {
+        if (!line)
+            continue;
+        const row = JSON.parse(line);
+        nodes.add(Number(row.node));
+        pairs.add(pairKey(row.components.intersection_a, row.components.intersection_b));
+    }
+    return { nodes, pairs };
+}
+/**
+ * Extract real crossings from one county's TIGER EDGES shapefile. Same query shape as the eval builder (2 incident
+ * distinct S1* FULLNAMEs at a node, both names >=6 chars), plus the edge ZIPL so tails can carry the crossing's own
+ * ZIP. Hash-ordered for seed-stable determinism.
+ */
+async function extractCrossings(db, edgesDir, county, seed) {
+    const shp = `${edgesDir}/tl_2023_${county.fips}_edges.shp`;
+    const result = await db.runAndReadAll(`
+		WITH incidence AS (
+			SELECT TNIDF AS node, FULLNAME AS name, ZIPL AS zip
+			FROM ST_Read('${shp}') WHERE MTFCC LIKE 'S1%' AND FULLNAME IS NOT NULL
+			UNION ALL
+			SELECT TNIDT AS node, FULLNAME AS name, ZIPL AS zip
+			FROM ST_Read('${shp}') WHERE MTFCC LIKE 'S1%' AND FULLNAME IS NOT NULL
+		),
+		nodes AS (
+			SELECT node,
+				list_sort(list_distinct(list(name))) AS names,
+				max(zip) AS zip
+			FROM incidence GROUP BY node
+			HAVING len(list_distinct(list(name))) = 2
+		)
+		SELECT node, names[1] AS a, names[2] AS b, zip,
+			hash(node::VARCHAR || '${seed}') AS h
+		FROM nodes
+		WHERE len(names[1]) >= 6 AND len(names[2]) >= 6
+		ORDER BY h
+	`);
+    const out = [];
+    for (const r of result.getRowObjects()) {
+        out.push({
+            a: String(r.a),
+            b: String(r.b),
+            zip: r.zip == null ? null : String(r.zip),
+            node: Number(r.node),
+            fips: county.fips,
+            state: county.state,
+        });
+    }
+    return out;
+}
+/** ZIP → majority city from the cached OA Cook-county CSV (real ZIP/city pairings). */
+function buildZipCityMap() {
+    const r = spawnSync("unzip", ["-p", OA_COOK.zip, OA_COOK.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" });
+    if (r.status !== 0) {
+        console.error(`  WARN: unzip failed for ${OA_COOK.zip} — city tails disabled`);
+        return new Map();
+    }
+    const lines = r.stdout.toString("utf8").split(/\r?\n/);
+    if (lines.length < 2)
+        return new Map();
+    const header = splitCSV(lines[0]).map((h) => h.trim().toLowerCase());
+    const iCity = header.indexOf("city");
+    const iPost = header.indexOf("postcode");
+    const counts = new Map();
+    // zip → Map(city → n)
+    for (let li = 1; li < lines.length; li++) {
+        if (!lines[li])
+            continue;
+        const cells = splitCSV(lines[li]);
+        const city = (cells[iCity] ?? "").trim();
+        const zip = (cells[iPost] ?? "").trim();
+        if (!city || !/^\d{5}$/.test(zip) || BAD_NAME.test(city))
+            continue;
+        let byCity = counts.get(zip);
+        if (!byCity)
+            counts.set(zip, (byCity = new Map()));
+        byCity.set(city, (byCity.get(city) ?? 0) + 1);
+    }
+    const map = new Map();
+    for (const [zip, byCity] of counts) {
+        let best = null;
+        for (const [city, n] of byCity)
+            if (!best || n > best.n)
+                best = { city, n };
+        if (best)
+            map.set(zip, best.city);
+    }
+    return map;
+}
+/**
+ * Render one crossing → { raw, components, formID, tailID, caseID }. Components are inserted in claim order (streets
+ * first) so alignment can't grab a region/postcode lookalike inside a street.
+ */
+function renderRow(random, crossing, zipCity) {
+    const form = weightedPick(FORMS, random);
+    const body = form.render(crossing.a, crossing.b);
+    let tail = weightedPick(TAILS, random);
+    const city = crossing.zip ? (zipCity.get(crossing.zip) ?? null) : null;
+    // Downgrade unsatisfiable tails (no ZIP on the edge / no city for the ZIP) to the region tail.
+    if ((tail.id === "region-zip" || tail.id === "city-region-zip") && !crossing.zip)
+        tail = TAILS[1];
+    if ((tail.id === "city-region" || tail.id === "city-region-zip") && !city)
+        tail = TAILS[1];
+    const components = { intersection_a: crossing.a, intersection_b: crossing.b };
+    let raw = body;
+    if (tail.id === "region") {
+        raw = `${body}, ${crossing.state}`;
+        components.region = crossing.state;
+    }
+    else if (tail.id === "region-zip") {
+        raw = `${body}, ${crossing.state} ${crossing.zip}`;
+        components.region = crossing.state;
+        components.postcode = crossing.zip;
+    }
+    else if (tail.id === "city-region") {
+        raw = `${body}, ${city}, ${crossing.state}`;
+        components.locality = city;
+        components.region = crossing.state;
+    }
+    else if (tail.id === "city-region-zip") {
+        raw = `${body}, ${city}, ${crossing.state} ${crossing.zip}`;
+        components.locality = city;
+        components.region = crossing.state;
+        components.postcode = crossing.zip;
+    }
+    const casing = weightedPick(CASES, random);
+    raw = casing.apply(raw);
+    // Components keep their original case; alignRow matches case-insensitively and labels the
+    // tokens of the (cased) raw — the parquet row carries tokens+labels only.
+    return { raw, components, formID: form.id, tailID: tail.id, caseID: casing.id };
+}
+/**
+ * Label-correctness audit for one aligned row, on the RAW SURFACE via the #519 span triple. Returns a list of
+ * violations (empty = clean). Re-derives the span checks independent of `alignRow`'s own assertion, so a builder bug
+ * can't vouch for itself.
+ */
+function auditRow(row, components) {
+    const errors = [];
+    const { raw, tokens, labels, span_starts, span_ends, span_tags } = row;
+    if (tokens.length !== labels.length)
+        errors.push("tokens/labels length mismatch");
+    if (!span_starts || !span_ends || !span_tags) {
+        errors.push("missing the char-offset span triple (#519)");
+        return errors;
+    }
+    if (span_starts.length !== span_ends.length || span_starts.length !== span_tags.length) {
+        errors.push(`span triple not parallel: ${span_starts.length}/${span_ends.length}/${span_tags.length}`);
+        return errors;
+    }
+    for (let i = 0; i < span_starts.length; i++) {
+        if (!(span_starts[i] >= 0 && span_starts[i] < span_ends[i] && span_ends[i] <= raw.length)) {
+            errors.push(`span ${span_tags[i]}@[${span_starts[i]}, ${span_ends[i]}) out of bounds`);
+        }
+        if (i > 0 && span_starts[i] < span_ends[i - 1]) {
+            errors.push(`spans unsorted/overlapping at index ${i}`);
+        }
+    }
+    if (errors.length > 0)
+        return errors;
+    const compCount = Object.keys(components).length;
+    if (span_tags.length !== compCount)
+        errors.push(`span count ${span_tags.length} != components ${compCount}`);
+    // Raw-surface reconstruction: each component's single span slices raw to the component verbatim.
+    for (const [tag, value] of Object.entries(components)) {
+        if (value == null)
+            continue;
+        const indices = span_tags.map((t, i) => (t === tag ? i : -1)).filter((i) => i >= 0);
+        if (indices.length !== 1) {
+            errors.push(`${tag}: expected 1 span, got ${indices.length}`);
+            continue;
+        }
+        const idx = indices[0];
+        const got = raw.slice(span_starts[idx], span_ends[idx]);
+        if (got.toLowerCase() !== value.toLowerCase())
+            errors.push(`${tag} span "${got}" != component "${value}"`);
+    }
+    // Negative space: every char outside the spans must be connector material.
+    let cursor = 0;
+    const uncovered = [];
+    for (let i = 0; i < span_starts.length; i++) {
+        if (span_starts[i] > cursor)
+            uncovered.push(raw.slice(cursor, span_starts[i]));
+        cursor = span_ends[i];
+    }
+    if (cursor < raw.length)
+        uncovered.push(raw.slice(cursor));
+    for (const segment of uncovered) {
+        const words = segment.match(/[\p{L}\p{N}]+/gu) ?? [];
+        for (const word of words) {
+            if (!CONNECTOR_O_TOKENS.has(word.toLowerCase()))
+                errors.push(`illegal uncovered word "${word}"`);
+        }
+        const punctOnly = segment.replace(/[\p{L}\p{N}]+/gu, "");
+        if (!CONNECTOR_PUNCT_RE.test(punctOnly))
+            errors.push(`illegal uncovered punctuation in "${segment}"`);
+    }
+    return errors;
+}
+export const intersectionRecipe = {
+    name: "intersection",
+    description: "Real-pair intersection rows (US): TIGER 2023 EDGES crossings → audited intersection_a/b labels",
+    mode: "generate",
+    options: [{ flag: "--edges-dir <dir>", description: "Unzipped TIGER 2023 EDGES dir. Default /tmp/tiger-edges" }],
+    async run(opts, write) {
+        // Legacy build-intersection-shard.mjs seeded `mulberry32(opts.seed)`.
+        const random = makeMulberry32(opts.seed);
+        const count = opts.count ?? 40000;
+        const source = opts.sourceName ?? "synth-intersection";
+        const edgesDir = opts.edgesDir ?? "/tmp/tiger-edges";
+        const counties = opts.golden ? GOLDEN_COUNTIES : TRAIN_COUNTIES;
+        const exclusions = readEvalExclusions();
+        console.error(`  eval exclusions: ${exclusions.nodes.size} nodes, ${exclusions.pairs.size} pairs`);
+        const { DuckDBInstance } = await import("@duckdb/node-api");
+        const instance = await DuckDBInstance.create();
+        const db = await instance.connect();
+        await db.run("INSTALL spatial; LOAD spatial;");
+        // Pool real crossings: eval-excluded, connector-safe names, one crossing per distinct pair.
+        const pool = [];
+        const seenPairs = new Set();
+        const stats = { evalExcluded: 0, badName: 0, dupPair: 0 };
+        for (const county of counties) {
+            const crossings = await extractCrossings(db, edgesDir, county, opts.seed);
+            let kept = 0;
+            for (const c of crossings) {
+                const key = pairKey(c.a, c.b);
+                if (exclusions.nodes.has(c.node) || exclusions.pairs.has(key)) {
+                    stats.evalExcluded++;
+                    continue;
+                }
+                if (BAD_NAME.test(c.a) || BAD_NAME.test(c.b) || c.a.includes(c.b) || c.b.includes(c.a)) {
+                    stats.badName++;
+                    continue;
+                }
+                if (seenPairs.has(key)) {
+                    stats.dupPair++;
+                    continue;
+                }
+                seenPairs.add(key);
+                pool.push(c);
+                kept++;
+            }
+            console.error(`  ${county.fips} (${county.state}, ${county.regime}): ${crossings.length} crossings, ${kept} kept`);
+        }
+        if (pool.length === 0) {
+            throw new Error(`No crossings found — are the TIGER EDGES shapefiles present in ${edgesDir}?`);
+        }
+        const zipCity = opts.golden ? new Map() : buildZipCityMap();
+        if (!opts.golden)
+            console.error(`  zip→city map: ${zipCity.size} ZIPs (OA Cook)`);
+        let emitted = 0;
+        let skipped = 0;
+        let guard = 0;
+        const formCounts = {};
+        const tailCounts = {};
+        const caseCounts = {};
+        const countyCounts = {};
+        const usedCrossings = new Set();
+        const seenRaw = new Set();
+        const auditErrors = [];
+        const samples = [];
+        while (emitted < count && guard++ < count * 10) {
+            const crossing = pool[Math.floor(random() * pool.length)];
+            const { raw, components, formID, tailID, caseID } = renderRow(random, crossing, zipCity);
+            if (seenRaw.has(raw)) {
+                skipped++;
+                continue;
+            }
+            if (opts.golden) {
+                seenRaw.add(raw);
+                write(JSON.stringify({ raw, components, country: "US", form: formID }) + "\n");
+                formCounts[formID] = (formCounts[formID] ?? 0) + 1;
+                tailCounts[tailID] = (tailCounts[tailID] ?? 0) + 1;
+                caseCounts[caseID] = (caseCounts[caseID] ?? 0) + 1;
+                countyCounts[crossing.fips] = (countyCounts[crossing.fips] ?? 0) + 1;
+                usedCrossings.add(crossing.node);
+                emitted++;
+                continue;
+            }
+            const canonical = {
+                raw,
+                components,
+                country: "US",
+                locale: "en-US",
+                source,
+                source_id: stableSourceID(source, components),
+                corpus_version: "0.4.0",
+                license: "TIGER/Line 2023 EDGES (US Census, public domain) real street pairs; OA Cook IL zip-to-city tails",
+            };
+            // Verbatim-only alignment: raw is built from the component values, so a fuzzy fallback could
+            // only ever mislabel (e.g. claim a lookalike window for a near-duplicate street).
+            const aligned = alignRow(canonical, { maxEditDistance: 0 });
+            if (aligned.kind !== "labeled" || !aligned.row) {
+                skipped++;
+                continue;
+            }
+            const violations = auditRow(aligned.row, components);
+            if (violations.length > 0) {
+                auditErrors.push({ raw, violations });
+                continue;
+            }
+            seenRaw.add(raw);
+            write(JSON.stringify({ ...aligned.row, synth_method: "intersection", synth_base_id: null }) + "\n");
+            formCounts[formID] = (formCounts[formID] ?? 0) + 1;
+            tailCounts[tailID] = (tailCounts[tailID] ?? 0) + 1;
+            caseCounts[caseID] = (caseCounts[caseID] ?? 0) + 1;
+            countyCounts[crossing.fips] = (countyCounts[crossing.fips] ?? 0) + 1;
+            usedCrossings.add(crossing.node);
+            if (samples.length < FORMS.length && !samples.some((s) => s.form === formID)) {
+                samples.push({ form: formID, raw, tokens: aligned.row.tokens, labels: aligned.row.labels });
+            }
+            emitted++;
+        }
+        const report = {
+            mode: opts.golden ? "golden" : "train",
+            rows: emitted,
+            skipped,
+            pool: { crossings: pool.length, used: usedCrossings.size, ...stats },
+            per_county: countyCounts,
+            forms: formCounts,
+            tails: tailCounts,
+            cases: caseCounts,
+            audit: { errors: auditErrors.length, examples: auditErrors.slice(0, 10) },
+            seed: opts.seed,
+            source: "TIGER2023 EDGES via DuckDB ST_Read; node = 2 distinct S1* FULLNAMEs; eval crossings excluded",
+            samples,
+        };
+        writeFileSync(opts.output.replace(/\.jsonl$/, ".report.json"), JSON.stringify(report, null, "\t"));
+        console.error(`Done: emitted ${emitted} rows (skipped ${skipped}) from ${usedCrossings.size}/${pool.length} real crossings. → ${opts.output}\n` +
+            `  forms: ${JSON.stringify(formCounts)}\n` +
+            `  tails: ${JSON.stringify(tailCounts)}\n` +
+            `  cases: ${JSON.stringify(caseCounts)}\n` +
+            `  audit: ${auditErrors.length} violation(s)`);
+        if (auditErrors.length > 0) {
+            throw new Error(`AUDIT FAILED — first violation: ${JSON.stringify(auditErrors[0])}`);
+        }
+        return { emitted, skipped };
+    },
+};
+//# sourceMappingURL=intersection.js.map