@mailwoman/corpus 8.6.0 → 9.1.0

package/out/src/shard-recipes/scaffold.d.ts

@@ -8,6 +8,9 @@
  *   canonical → `alignRow` → `LabeledRow` JSONL emit step. A recipe ({@link ShardRecipe}) supplies
  *   only its synthesis + filter; the `mailwoman corpus shard <recipe>` command supplies the I/O.
  */
+import type { PathBuilderLike } from "path-ts";
+import type { AsyncChunkIterator, AsyncDataResource } from "spliterator";
+import { AsyncSequence } from "spliterator";
 /**
  * {@link stableSourceID}, but accepting arbitrary disambiguator keys (e.g. a variant index `v`) that aren't
  * `ComponentTag`s. `stableSourceID` sorts + hashes EVERY key it's given, so passing extra keys is how the legacy
@@ -25,20 +28,42 @@ export interface ShardTuple {
     [k: string]: unknown;
 }
 /**
- * The deterministic LCG (`s = s*1664525 + 1013904223 mod 2^32`) the street/po-box/anchor builders used. A recipe whose
- * legacy `.mjs` seeded this must create it here so `--seed N` is byte-reproducible.
+ * The two seeded generators the legacy `build-*-shard` scripts used, re-exported from their home in
+ * `@mailwoman/core/utils` so a recipe keeps importing everything it needs from this one scaffold module.
+ *
+ * - `makeLcg` (`s = s*1664525 + 1013904223 mod 2^32`) — what the street/po-box/anchor builders seeded.
+ * - `makeMulberry32` — what the MAJORITY of them used (german, locale, boundary-stress, unit, fr-order, country-balanced,
+ *   intersection, fr-admin-split, street-affix, street-bare, po-box-cedex).
+ *
+ * A recipe must seed the same one its `.mjs` did, the same way it did (usually `seed`, but some derive a per-stream
+ * seed), or `--seed N` stops being byte-reproducible.
  */
-export declare function makeLcg(seed: number): () => number;
+export { makeLcg, mulberry32 as makeMulberry32, makeLcg as makeRandom } from "@mailwoman/core/utils";
 /**
- * Back-compat alias for {@link makeLcg}.
+ * One CSV record, keyed by its lower-cased header name, every value trimmed. A column the header does not declare reads
+ * as `undefined`; a column the header declares but this record does not reach reads as `""`.
  */
-export declare const makeRandom: typeof makeLcg;
+export type CSVRecord = Record<string, string | undefined>;
 /**
- * Mulberry32 — the PRNG the MAJORITY of the legacy `build-*-shard` scripts used (german, locale, boundary-stress, unit,
- * fr-order, country-balanced, intersection, fr-admin-split, street-affix, street-bare, po-box-cedex). A recipe must
- * seed it EXACTLY as its `.mjs` did (usually `seed`, but some derive a per-stream seed) to stay byte-reproducible.
+ * Read a CSV as header-keyed records.
+ *
+ * Returns the spliterator's own {@linkcode AsyncSequence}, so a caller composes `take`/`drop`/`filter` onto it — those
+ * ops fuse into one pull loop, and a `take` that is satisfied closes the source's file handle on the way out. Wrapping
+ * this in an `async function*` costs an async frame per row and takes those ops away; don't.
+ *
+ * A source at or below the spliterator's 128 KiB bulk threshold is read whole and parsed by the synchronous engine, so
+ * this is also the right reader for small sources — there is no buffered variant to reach for.
  */
-export declare function makeMulberry32(seed: number): () => number;
+export declare function readCSVRecords(source: AsyncDataResource | AsyncChunkIterator): AsyncSequence<CSVRecord>;
+/**
+ * {@link readCSVRecords} over one member of a zip archive — what every recipe reading a cached OA source wants.
+ *
+ * A source a checkout has not cached yields nothing, after saying so. A lab holds the archives for the countries it has
+ * built, so a recipe naming ten sources routinely finds three, and the `unzip -p` subprocesses these replaced behaved
+ * the same way by accident — a non-zero exit warned and returned no rows. A recipe that ends up with NO tuples at all
+ * still throws; that is the case where the cache, not the recipe, is the problem.
+ */
+export declare function readZippedCSVRecords(archivePath: PathBuilderLike, entryName: string): AsyncSequence<CSVRecord>;
 /**
  * Stream-parse a tuples JSONL file, yielding each parsed object (blank/invalid lines skipped).
  */
@@ -117,6 +142,30 @@ export interface ShardRecipeOpts {
      * `no-fragment` knob 3: minimum digit count for a number to count as "long" and be boosted. Default 3.
      */
     longNumberMinDigits?: number;
+    /**
+     * `sub-venue`: the sub-venue lexicon JSON. Default = the committed `corpus/data/sub-venue-lexicon.json`, resolved
+     * through the package manifest so it works from the source tree and from `out/`.
+     */
+    lexicon?: string;
+    /**
+     * `sub-venue`: directory of `sub-venue-extract` JSONLs, one per region. Default
+     * `$MAILWOMAN_DATA_ROOT/sub-venue/extracts`.
+     */
+    extractsDir?: string;
+    /**
+     * `sub-venue`: the `poi.db` spatial layer, read for the en-US and fr-FR venue + confound pools (the two of poi.db's
+     * four countries this shard has legs for). Default `$MAILWOMAN_DATA_ROOT/poi/poi.db`.
+     */
+    poiDb?: string;
+    /**
+     * `sub-venue`: GB/US/FR address-context tuples JSONL. Default the house-venue v3 tuples
+     * (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v3.jsonl`); DE and ES read OpenAddresses directly.
+     */
+    subVenueTuples?: string;
+    /**
+     * `sub-venue`: share of emitted rows that are confound NEGATIVES. Default 0.3.
+     */
+    negativeFraction?: number;
 }
 /**
  * Tally a recipe returns.

package/out/src/shard-recipes/scaffold.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scaffold.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,CAElG;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;CACpB;AAED;;;GAGG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CAQlD;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,gBAAU,CAAA;AAEjC;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CAUzD;AAED;;GAEG;AACH,wBAAuB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAgB3E;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC5B,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAC7B,SAAS,EAAE,iBAAiB,EAC5B,WAAW,EAAE,MAAM,EACnB,WAAW,GAAE,MAAM,GAAG,IAAW,GAC/B,OAAO,CAOT;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,IAAI,EAAE,QAAQ,GAAG,UAAU,CAAA;IAC3B;;OAEG;IACH,OAAO,CAAC,EAAE,iBAAiB,EAAE,CAAA;IAC7B;;;OAGG;IACH,GAAG,CAAC,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;CAC9E"}
+{"version":3,"file":"scaffold.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAQH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAC9C,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACxE,OAAO,EAAE,aAAa,EAAkB,MAAM,aAAa,CAAA;AAE3D;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,CAElG;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;CACpB;AAED;;;;;;;;;;GAUG;AACH,OAAO,EAAE,OAAO,EAAE,UAAU,IAAI,cAAc,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAEpG;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAA;AA4C1D;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,GAAG,aAAa,CAAC,SAAS,CAAC,CAYvG;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,CAQ9G;AAED;;GAEG;AACH,wBAAuB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAgB3E;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC5B,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAC7B,SAAS,EAAE,iBAAiB,EAC5B,WAAW,EAAE,MAAM,EACnB,WAAW,GAAE,MAAM,GAAG,IAAW,GAC/B,OAAO,CAOT;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,IAAI,EAAE,QAAQ,GAAG,UAAU,CAAA;IAC3B;;OAEG;IACH,OAAO,CAAC,EAAE,iBAAiB,EAAE,CAAA;IAC7B;;;OAGG;IACH,GAAG,CAAC,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;CAC9E"}

package/out/src/shard-recipes/scaffold.js

@@ -8,8 +8,12 @@
  *   canonical → `alignRow` → `LabeledRow` JSONL emit step. A recipe ({@link ShardRecipe}) supplies
  *   only its synthesis + filter; the `mailwoman corpus shard <recipe>` command supplies the I/O.
  */
-import { stableSourceID } from "../adapter.js";
-import { alignRow } from "../align.js";
+import { existsSync } from "node:fs";
+import { readZipEntry } from "@mailwoman/core/fs/zip";
+import { tryParsingJSON } from "@mailwoman/core/objects";
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils";
+import { alignRow } from "@mailwoman/corpus/utils";
+import { AsyncSequence, CSVSpliterator } from "spliterator";
 /**
  * {@link stableSourceID}, but accepting arbitrary disambiguator keys (e.g. a variant index `v`) that aren't
  * `ComponentTag`s. `stableSourceID` sorts + hashes EVERY key it's given, so passing extra keys is how the legacy
@@ -19,50 +23,104 @@ export function shardSourceID(adapterID, parts) {
     return stableSourceID(adapterID, parts);
 }
 /**
- * The deterministic LCG (`s = s*1664525 + 1013904223 mod 2^32`) the street/po-box/anchor builders used. A recipe whose
- * legacy `.mjs` seeded this must create it here so `--seed N` is byte-reproducible.
+ * The two seeded generators the legacy `build-*-shard` scripts used, re-exported from their home in
+ * `@mailwoman/core/utils` so a recipe keeps importing everything it needs from this one scaffold module.
+ *
+ * - `makeLcg` (`s = s*1664525 + 1013904223 mod 2^32`) — what the street/po-box/anchor builders seeded.
+ * - `makeMulberry32` — what the MAJORITY of them used (german, locale, boundary-stress, unit, fr-order, country-balanced,
+ *   intersection, fr-admin-split, street-affix, street-bare, po-box-cedex).
+ *
+ * A recipe must seed the same one its `.mjs` did, the same way it did (usually `seed`, but some derive a per-stream
+ * seed), or `--seed N` stops being byte-reproducible.
+ */
+export { makeLcg, mulberry32 as makeMulberry32, makeLcg as makeRandom } from "@mailwoman/core/utils";
+/**
+ * Quote handling spans the ROW split, not only the column split: a newline inside a quoted field belongs to a single
+ * record, so the record boundaries can only be found by a scanner that already knows where the quotes are. Find the
+ * lines first and unquote afterwards and such a record splits in two — the first half short by however many columns
+ * followed the newline, which is how a `street` value comes to be read as `city`.
+ *
+ * `header: false` keeps the first record in the stream, so this module owns the lower-casing rather than
+ * `normalizeKeys` — see {@link toHeader}.
  */
-export function makeLcg(seed) {
-    let s = seed >>> 0;
-    return () => {
-        s = (s * 1_664_525 + 1_013_904_223) % 4_294_967_296;
-        return s / 4_294_967_296;
-    };
+const CSV_INIT = { header: false, enableQuoteHandling: true };
+/**
+ * Header names are lower-cased on the way in. `normalizeKeys` will not do it: it leaves an ALL CAPS header alone, and
+ * OpenAddresses ships `LON,LAT,NUMBER,STREET` while other extracts ship the same names lower-case. A recipe names its
+ * columns in lower case either way.
+ */
+function toHeader(cells) {
+    return cells.map((name) => name.trim().toLowerCase());
 }
 /**
- * Back-compat alias for {@link makeLcg}.
+ * Line breaks inside a value become single spaces. A quote-aware parse is the first thing here able to return a value
+ * CONTAINING one — `us/ia/statewide.csv` has 12, all unit designators like `"#2\n#2"` — and every consumer synthesizes
+ * one-line address text from these cells with no guard, because until that parse landed no value could carry one.
+ * Collapsing keeps the record (the address is fine; the source's line break is not part of it) without emitting a
+ * training row with a newline inside it.
+ *
+ * Only `\r` and `\n`, deliberately — NOT `\s`. Runs of spaces and tabs pass through exactly as the source wrote them
+ * (OA's IA extract writes `NORTH`, three spaces, `MAIN STREET`), because those could always appear and every shard
+ * built to date contains them. Widening this to `\s+` silently rewrites values on rows with no line break at all.
+ * `scaffold.test.ts` pins both halves, against both readers.
  */
-export const makeRandom = makeLcg;
+function toRecord(header, cells) {
+    const record = {};
+    for (let i = 0; i < header.length; i++) {
+        record[header[i]] = (cells[i] ?? "").replaceAll(/[\r\n]+/g, " ").trim();
+    }
+    return record;
+}
 /**
- * Mulberry32 — the PRNG the MAJORITY of the legacy `build-*-shard` scripts used (german, locale, boundary-stress, unit,
- * fr-order, country-balanced, intersection, fr-admin-split, street-affix, street-bare, po-box-cedex). A recipe must
- * seed it EXACTLY as its `.mjs` did (usually `seed`, but some derive a per-stream seed) to stay byte-reproducible.
+ * Read a CSV as header-keyed records.
+ *
+ * Returns the spliterator's own {@linkcode AsyncSequence}, so a caller composes `take`/`drop`/`filter` onto it — those
+ * ops fuse into one pull loop, and a `take` that is satisfied closes the source's file handle on the way out. Wrapping
+ * this in an `async function*` costs an async frame per row and takes those ops away; don't.
+ *
+ * A source at or below the spliterator's 128 KiB bulk threshold is read whole and parsed by the synchronous engine, so
+ * this is also the right reader for small sources — there is no buffered variant to reach for.
  */
-export function makeMulberry32(seed) {
-    let a = seed >>> 0;
-    return () => {
-        a = (a + 0x6d_2b_79_f5) | 0;
-        let t = Math.imul(a ^ (a >>> 15), 1 | a);
-        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
-        return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296;
-    };
+export function readCSVRecords(source) {
+    let header = null;
+    return CSVSpliterator.fromAsync(source, CSV_INIT)
+        .filter((cells) => {
+        if (header)
+            return true;
+        header = toHeader(cells);
+        return false;
+    })
+        .map((cells) => toRecord(header, cells));
+}
+/**
+ * {@link readCSVRecords} over one member of a zip archive — what every recipe reading a cached OA source wants.
+ *
+ * A source a checkout has not cached yields nothing, after saying so. A lab holds the archives for the countries it has
+ * built, so a recipe naming ten sources routinely finds three, and the `unzip -p` subprocesses these replaced behaved
+ * the same way by accident — a non-zero exit warned and returned no rows. A recipe that ends up with NO tuples at all
+ * still throws; that is the case where the cache, not the recipe, is the problem.
+ */
+export function readZippedCSVRecords(archivePath, entryName) {
+    if (!existsSync(String(archivePath))) {
+        console.error(`  WARN: ${archivePath} is not cached — skipping ${entryName}`);
+        return AsyncSequence.from([]);
+    }
+    return readCSVRecords(readZipEntry(archivePath, entryName));
 }
 /**
  * Stream-parse a tuples JSONL file, yielding each parsed object (blank/invalid lines skipped).
  */
 export async function* readTuples(input) {
     // TextSpliterator (not JSONSpliterator) so a malformed line is SKIPPED, not thrown — the
-    // per-line try/catch below is the tolerance this reader has always had.
+    // per-line tryParsingJSON below is the tolerance this reader has always had.
     const { TextSpliterator } = await import("spliterator");
     for await (const line of TextSpliterator.fromAsync(input)) {
         const trimmed = line.trim();
         if (!trimmed)
             continue;
-        try {
-            yield JSON.parse(trimmed);
-        }
-        catch {
-            // skip malformed line
+        const tuple = tryParsingJSON(trimmed);
+        if (tuple !== null) {
+            yield tuple;
         }
     }
 }

package/out/src/shard-recipes/scaffold.js.map

@@ -1 +1 @@
-{"version":3,"file":"scaffold.js","sourceRoot":"","sources":["../../../src/shard-recipes/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,SAAiB,EAAE,KAAyC;IACzF,OAAO,cAAc,CAAC,SAAS,EAAE,KAAwD,CAAC,CAAA;AAC3F,CAAC;AAaD;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IACnC,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,CAAA;IAElB,OAAO,GAAG,EAAE;QACX,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,GAAG,aAAa,CAAC,GAAG,aAAa,CAAA;QAEnD,OAAO,CAAC,GAAG,aAAa,CAAA;IACzB,CAAC,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,OAAO,CAAA;AAEjC;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,CAAA;IAElB,OAAO,GAAG,EAAE;QACX,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QACxC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;QAE9C,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,aAAa,CAAA;IAChD,CAAC,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,KAAa;IAC9C,yFAAyF;IACzF,wEAAwE;IACxE,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAA;IAEvD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAe,CAAA;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,sBAAsB;QACvB,CAAC;IACF,CAAC;AACF,CAAC;AAgBD;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC5B,KAA6B,EAC7B,SAA4B,EAC5B,WAAmB,EACnB,cAA6B,IAAI;IAEjC,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;IAErE,IAAI,CAAC,OAAO,CAAC,GAAG;QAAE,OAAO,KAAK,CAAA;IAC9B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;IAEvG,OAAO,IAAI,CAAA;AACZ,CAAC"}
+{"version":3,"file":"scaffold.js","sourceRoot":"","sources":["../../../src/shard-recipes/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAA;AACjE,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAA;AAGlD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE3D;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,SAAiB,EAAE,KAAyC;IACzF,OAAO,cAAc,CAAC,SAAS,EAAE,KAAwD,CAAC,CAAA;AAC3F,CAAC;AAaD;;;;;;;;;;GAUG;AACH,OAAO,EAAE,OAAO,EAAE,UAAU,IAAI,cAAc,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAQpG;;;;;;;;GAQG;AACH,MAAM,QAAQ,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,IAAI,EAAW,CAAA;AAEtE;;;;GAIG;AACH,SAAS,QAAQ,CAAC,KAAwB;IACzC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;AACtD,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,QAAQ,CAAC,MAAyB,EAAE,KAAwB;IACpE,MAAM,MAAM,GAAc,EAAE,CAAA;IAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACzE,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,cAAc,CAAC,MAA8C;IAC5E,IAAI,MAAM,GAAoB,IAAI,CAAA;IAElC,OAAO,cAAc,CAAC,SAAS,CAAW,MAAM,EAAE,QAAQ,CAAC;SACzD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,IAAI,MAAM;YAAE,OAAO,IAAI,CAAA;QAEvB,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAExB,OAAO,KAAK,CAAA;IACb,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAO,EAAE,KAAK,CAAC,CAAC,CAAA;AAC3C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,WAA4B,EAAE,SAAiB;IACnF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;QACtC,OAAO,CAAC,KAAK,CAAC,WAAW,WAAW,6BAA6B,SAAS,EAAE,CAAC,CAAA;QAE7E,OAAO,aAAa,CAAC,IAAI,CAAY,EAAE,CAAC,CAAA;IACzC,CAAC;IAED,OAAO,cAAc,CAAC,YAAY,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC,CAAA;AAC5D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,KAAa;IAC9C,yFAAyF;IACzF,6EAA6E;IAC7E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAA;IAEvD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,MAAM,KAAK,GAAG,cAAc,CAAa,OAAO,CAAC,CAAA;QAEjD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACpB,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;AACF,CAAC;AAgBD;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC5B,KAA6B,EAC7B,SAA4B,EAC5B,WAAmB,EACnB,cAA6B,IAAI;IAEjC,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;IAErE,IAAI,CAAC,OAAO,CAAC,GAAG;QAAE,OAAO,KAAK,CAAA;IAC9B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;IAEvG,OAAO,IAAI,CAAA;AACZ,CAAC"}

package/out/src/shard-recipes/street-affix.d.ts

@@ -26,10 +26,64 @@
  *   to keep the postcode-ORDER distribution multi-locale so a US-heavy affix shard doesn't dilute
  *   FR/DE postcode (the v0.9.8 blemish).
  */
+import type { ComponentTag } from "@mailwoman/core/types";
 import { type ShardRecipe } from "./scaffold.ts";
+/**
+ * A real US skeleton tuple read from a cached OA zip.
+ */
+interface USTuple {
+    house_number: string;
+    street: string;
+    locality: string;
+    region: string;
+    postcode: string;
+    base_source_id: string;
+}
+export type SuffixBoundaryClass = "terminal-only" | "terminal-contrast";
+/**
+ * Classify a real street surface for #1569. `terminal-only` has an ambiguous suffix-eligible name word immediately
+ * before a different terminal type (`Blue Hill Rd`); `terminal-contrast` ends at the ambiguous word itself (`Sutton
+ * Hollow`). The contrast check intentionally wins when both final words are name-prone: only the LAST token is the
+ * suffix under the canonical rule.
+ */
+export declare function classifySuffixBoundaryStreet(street: string): SuffixBoundaryClass | null;
+/**
+ * Layout-shell options for {@link renderRow}. `cuts` are the cumulative random() cutoffs for [full, bare, street-only] —
+ * the remainder is the venue shell. Defaults reproduce the original street-affix distribution (40/25/20/15) with the
+ * six template venues.
+ */
+interface RenderRowOpts {
+    venues?: readonly string[];
+    cuts?: readonly [number, number, number];
+}
+/**
+ * Embed the rendered street in a RANDOM layout so the model recognizes affixes wherever the street sits: full address,
+ * bare house+street, street-only (pure affix parse), or venue-prefixed.
+ */
+export declare function renderRow(random: () => number, base: USTuple, street: string, streetComponents: Partial<Record<ComponentTag, string>>, opts?: RenderRowOpts): {
+    fmt: string;
+    raw: string;
+    components: Partial<Record<ComponentTag, string>>;
+};
 /**
  * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
  * and `description` below for the surface form it generates.
  */
 export declare const streetAffixRecipe: ShardRecipe;
+/**
+ * #1569 root-fix shard. Both classes come from real non-Vermont OA streets and use the affix recipe's existing layout
+ * diversity. v4.3.1 makes terminal-only 80% of the mix: the first 40/60 run moved a 100-row TRAIN sample only 4→11
+ * while contrast was already 95/100 before training (93/100 after). Post-run audit found that the global affix relabel
+ * pass corrupts many already-decomposed target rows into double suffixes; do not retrain this recipe until relabel is
+ * idempotent over a decomposed street family. The 20% contrast leg remains explicit, additive to the already-strong
+ * base distribution, and B2 still gates it unchanged.
+ *
+ * V2 (corpus 0.19.0, 2026-08-10 recipe review): the v4.3.3 board split (rich venue-led rows 5/43 vs bare 53/65) showed
+ * the model separating template rows from real ones, and the venue shell was the giveaway — six fixed venue strings. v2
+ * draws the venue shell from thousands of REAL HRSA facility names and raises its share (venue 30%, full 35%, bare 20%,
+ * street-only 15%). Dose policy moved to the recipe's config side: weight ≤4 effective passes per run (Muennighoff
+ * 2023's repetition knee) and the source is excluded from the augmentation pool — see the v4.4.0 config.
+ */
+export declare const suffixBoundaryRecipe: ShardRecipe;
+export {};
 //# sourceMappingURL=street-affix.d.ts.map

package/out/src/shard-recipes/street-affix.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"street-affix.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/street-affix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAkBH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AA6XhE;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,WAiM/B,CAAA"}
+{"version":3,"file":"street-affix.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/street-affix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAWH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAOzD,OAAO,EAAuE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAuFrH;;GAEG;AACH,UAAU,OAAO;IAChB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;CACtB;AAmGD,MAAM,MAAM,mBAAmB,GAAG,eAAe,GAAG,mBAAmB,CAAA;AAEvE;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAevF;AAsFD;;;;GAIG;AACH,UAAU,aAAa;IACtB,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B,IAAI,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;CACxC;AAED;;;GAGG;AACH,wBAAgB,SAAS,CACxB,MAAM,EAAE,MAAM,MAAM,EACpB,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,EACvD,IAAI,GAAE,aAAkB,GACtB;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;CAAE,CA8BjF;AAyDD;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,WAiM/B,CAAA;AAED;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,oBAAoB,EAAE,WA0JlC,CAAA"}