@mailwoman/corpus 9.0.0 → 9.2.0

package/out/src/shard-recipes/scaffold.d.ts

@@ -8,20 +8,52 @@
  *   canonical → `alignRow` → `LabeledRow` JSONL emit step. A recipe ({@link ShardRecipe}) supplies
  *   only its synthesis + filter; the `mailwoman corpus shard <recipe>` command supplies the I/O.
  */
+import type { PathBuilderLike } from "path-ts";
+import type { AsyncChunkIterator, AsyncDataResource } from "spliterator";
+import { AsyncSequence } from "spliterator";
 /**
- * {@link stableSourceID}, but accepting arbitrary disambiguator keys (e.g. a variant index `v`) that aren't
- * `ComponentTag`s. `stableSourceID` sorts + hashes EVERY key it's given, so passing extra keys is how the legacy
- * builders kept per-variant ids unique — the strict typing is just too narrow for that. Centralizes the one cast.
+ * {@link stableSourceIDFromParts} under the name the recipes use: arbitrary disambiguator keys (e.g. a variant index
+ * `v`) that aren't `ComponentTag`s, which is how the legacy builders kept per-variant ids unique.
  */
 export declare function shardSourceID(adapterID: string, parts: Record<string, string | undefined>): string;
+/**
+ * Where a country's convention writes the postcode inside the `«locality», «region»[, «country»]` admin tail.
+ *
+ * Load-bearing rather than cosmetic: the same digits change TAG with position. Measured on the shipped model,
+ * `Heladería Frappé Manía, Avenida Country Club, Barcelona 6001, Anzoátegui, Venezuela` tags `6001` as `house_number`
+ * and loses the locality into the street, while the identical row written `… 6001 Barcelona, Anzoátegui, Venezuela`
+ * tags it `postcode` and recovers `locality: Barcelona`. So a shard emitting one placement teaches one family of
+ * countries.
+ *
+ * Each is attested by a gauntlet board row, which is the bar for adding another:
+ *
+ * - `leading` — `«postcode» «locality», «region»`. `Rua da Praia, 15, 8600-315 Lagos, Algarve, Portugal`
+ *   (`pt_structured`). The default, and what every tuple written before this field existed means.
+ * - `after_locality` — `«locality» «postcode», «region»`. `…, Barcelona 6001, Anzoátegui, Venezuela`
+ *   (`ve_city_postcode_trailing_state`).
+ * - `after_region` — `«locality», «region» «postcode»`. `12 MG Road, Indiranagar, Bengaluru, Karnataka 560038, India`
+ *   (`in_structured`).
+ */
+export type PostcodePlacement = "leading" | "after_locality" | "after_region";
 /**
  * A (locality, region, postcode, country) source tuple — the input to tuples-mode recipes.
  */
 export interface ShardTuple {
     locality?: string;
+    /**
+     * The segment before the locality, when the source has one. A shard whose every row BEGINS with the locality teaches
+     * that the first named segment is the locality, which flips the model's default — measured on the v4.8.0 candidate,
+     * where `Ye Three Lords, 27 Minories, London EC3N 1DE` came back `locality: "Ye Three Lords"`.
+     */
+    dependentLocality?: string;
     region?: string;
     postcode?: string;
     country?: string;
+    /**
+     * Defaults to `leading` when absent, which is what every tuple file written before this field existed means — so an
+     * old tuples file produces byte-identical rows.
+     */
+    postcodePlacement?: PostcodePlacement;
     [k: string]: unknown;
 }
 /**
@@ -42,29 +74,25 @@ export { makeLcg, mulberry32 as makeMulberry32, makeLcg as makeRandom } from "@m
  */
 export type CSVRecord = Record<string, string | undefined>;
 /**
- * Read an in-memory CSV (an `unzip -p` buffer, say) as header-keyed records.
+ * Read a CSV as header-keyed records.
  *
- * Quote handling spans the ROW split, not only the column split: a newline inside a quoted field belongs to a single
- * record, so the record boundaries can only be found by a scanner that already knows where the quotes are. Find the
- * lines first and unquote afterwards and such a record splits in two — the first half short by however many columns
- * followed the newline, which is how a `street` value comes to be read as `city`.
+ * Returns the spliterator's own {@linkcode AsyncSequence}, so a caller composes `take`/`drop`/`filter` onto it — those
+ * ops fuse into one pull loop, and a `take` that is satisfied closes the source's file handle on the way out. Wrapping
+ * this in an `async function*` costs an async frame per row and takes those ops away; don't.
  *
- * Header names are lower-cased on the way in. `normalizeKeys` will not do it: it leaves an ALL CAPS header alone, and
- * OpenAddresses ships `LON,LAT,NUMBER,STREET` while other extracts ship the same names lower-case. A recipe names its
- * columns in lower case either way.
- *
- * Line breaks inside a value become single spaces. A quote-aware parse is the first thing here able to return a value
- * CONTAINING one — `us/ia/statewide.csv` has 12, all unit designators like `"#2\n#2"` — and every consumer synthesizes
- * one-line address text from these cells with no guard, because until that parse landed no value could carry one.
- * Collapsing keeps the record (the address is fine; the source's line break is not part of it) without emitting a
- * training row with a newline inside it.
+ * A source at or below the spliterator's 128 KiB bulk threshold is read whole and parsed by the synchronous engine, so
+ * this is also the right reader for small sources — there is no buffered variant to reach for.
+ */
+export declare function readCSVRecords(source: AsyncDataResource | AsyncChunkIterator): AsyncSequence<CSVRecord>;
+/**
+ * {@link readCSVRecords} over one member of a zip archive — what every recipe reading a cached OA source wants.
  *
- * Only `\r` and `\n`, deliberately — NOT `\s`. Runs of spaces and tabs pass through exactly as the source wrote them
- * (OA's IA extract writes `NORTH`, three spaces, `MAIN STREET`), because those could always appear and every shard
- * built to date contains them. Widening this to `\s+` silently rewrites values on rows with no line break at all.
- * `scaffold.test.ts` pins both halves.
+ * A source a checkout has not cached yields nothing, after saying so. A lab holds the archives for the countries it has
+ * built, so a recipe naming ten sources routinely finds three, and the `unzip -p` subprocesses these replaced behaved
+ * the same way by accident — a non-zero exit warned and returned no rows. A recipe that ends up with NO tuples at all
+ * still throws; that is the case where the cache, not the recipe, is the problem.
  */
-export declare function readCSVRecords(source: Uint8Array): Generator<CSVRecord>;
+export declare function readZippedCSVRecords(archivePath: PathBuilderLike, entryName: string): AsyncSequence<CSVRecord>;
 /**
  * Stream-parse a tuples JSONL file, yielding each parsed object (blank/invalid lines skipped).
  */
@@ -157,7 +185,7 @@ export interface ShardRecipeOpts {
      * `sub-venue`: the `poi.db` spatial layer, read for the en-US and fr-FR venue + confound pools (the two of poi.db's
      * four countries this shard has legs for). Default `$MAILWOMAN_DATA_ROOT/poi/poi.db`.
      */
-    poiDb?: string;
+    poiDB?: string;
     /**
      * `sub-venue`: GB/US/FR address-context tuples JSONL. Default the house-venue v3 tuples
      * (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v3.jsonl`); DE and ES read OpenAddresses directly.

package/out/src/shard-recipes/scaffold.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scaffold.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAQH;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,CAElG;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;CACpB;AAED;;;;;;;;;;GAUG;AACH,OAAO,EAAE,OAAO,EAAE,UAAU,IAAI,cAAc,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAEpG;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAA;AAE1D;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAiB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,CAmBxE;AAED;;GAEG;AACH,wBAAuB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAgB3E;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC5B,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAC7B,SAAS,EAAE,iBAAiB,EAC5B,WAAW,EAAE,MAAM,EACnB,WAAW,GAAE,MAAM,GAAG,IAAW,GAC/B,OAAO,CAOT;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,IAAI,EAAE,QAAQ,GAAG,UAAU,CAAA;IAC3B;;OAEG;IACH,OAAO,CAAC,EAAE,iBAAiB,EAAE,CAAA;IAC7B;;;OAGG;IACH,GAAG,CAAC,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;CAC9E"}
+{"version":3,"file":"scaffold.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAC9C,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACxE,OAAO,EAAE,aAAa,EAAkB,MAAM,aAAa,CAAA;AAK3D;;;GAGG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,CAElG;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,gBAAgB,GAAG,cAAc,CAAA;AAE7E;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAA;IACrC,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;CACpB;AAED;;;;;;;;;;GAUG;AACH,OAAO,EAAE,OAAO,EAAE,UAAU,IAAI,cAAc,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAEpG;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAA;AA4C1D;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,GAAG,aAAa,CAAC,SAAS,CAAC,CAYvG;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,CAQ9G;AAED;;GAEG;AACH,wBAAuB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAgB3E;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC5B,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAC7B,SAAS,EAAE,iBAAiB,EAC5B,WAAW,EAAE,MAAM,EACnB,WAAW,GAAE,MAAM,GAAG,IAAW,GAC/B,OAAO,CAOT;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,IAAI,EAAE,QAAQ,GAAG,UAAU,CAAA;IAC3B;;OAEG;IACH,OAAO,CAAC,EAAE,iBAAiB,EAAE,CAAA;IAC7B;;;OAGG;IACH,GAAG,CAAC,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;CAC9E"}

package/out/src/shard-recipes/scaffold.js

@@ -8,17 +8,18 @@
  *   canonical → `alignRow` → `LabeledRow` JSONL emit step. A recipe ({@link ShardRecipe}) supplies
  *   only its synthesis + filter; the `mailwoman corpus shard <recipe>` command supplies the I/O.
  */
+import { existsSync } from "node:fs";
+import { readZipEntry } from "@mailwoman/core/fs/zip";
 import { tryParsingJSON } from "@mailwoman/core/objects";
-import { CSVSpliterator } from "spliterator";
-import { stableSourceID } from "../adapter.js";
-import { alignRow } from "../align.js";
+import { AsyncSequence, CSVSpliterator } from "spliterator";
+import { stableSourceIDFromParts } from "#adapters/utils";
+import { alignRow } from "#utils";
 /**
- * {@link stableSourceID}, but accepting arbitrary disambiguator keys (e.g. a variant index `v`) that aren't
- * `ComponentTag`s. `stableSourceID` sorts + hashes EVERY key it's given, so passing extra keys is how the legacy
- * builders kept per-variant ids unique — the strict typing is just too narrow for that. Centralizes the one cast.
+ * {@link stableSourceIDFromParts} under the name the recipes use: arbitrary disambiguator keys (e.g. a variant index
+ * `v`) that aren't `ComponentTag`s, which is how the legacy builders kept per-variant ids unique.
  */
 export function shardSourceID(adapterID, parts) {
-    return stableSourceID(adapterID, parts);
+    return stableSourceIDFromParts(adapterID, parts);
 }
 /**
  * The two seeded generators the legacy `build-*-shard` scripts used, re-exported from their home in
@@ -33,17 +34,24 @@ export function shardSourceID(adapterID, parts) {
  */
 export { makeLcg, mulberry32 as makeMulberry32, makeLcg as makeRandom } from "@mailwoman/core/utils";
 /**
- * Read an in-memory CSV (an `unzip -p` buffer, say) as header-keyed records.
- *
  * Quote handling spans the ROW split, not only the column split: a newline inside a quoted field belongs to a single
  * record, so the record boundaries can only be found by a scanner that already knows where the quotes are. Find the
  * lines first and unquote afterwards and such a record splits in two — the first half short by however many columns
  * followed the newline, which is how a `street` value comes to be read as `city`.
  *
+ * `header: false` keeps the first record in the stream, so this module owns the lower-casing rather than
+ * `normalizeKeys` — see {@link toHeader}.
+ */
+const CSV_INIT = { header: false, enableQuoteHandling: true };
+/**
  * Header names are lower-cased on the way in. `normalizeKeys` will not do it: it leaves an ALL CAPS header alone, and
  * OpenAddresses ships `LON,LAT,NUMBER,STREET` while other extracts ship the same names lower-case. A recipe names its
  * columns in lower case either way.
- *
+ */
+function toHeader(cells) {
+    return cells.map((name) => name.trim().toLowerCase());
+}
+/**
  * Line breaks inside a value become single spaces. A quote-aware parse is the first thing here able to return a value
  * CONTAINING one — `us/ia/statewide.csv` has 12, all unit designators like `"#2\n#2"` — and every consumer synthesizes
  * one-line address text from these cells with no guard, because until that parse landed no value could carry one.
@@ -53,22 +61,50 @@ export { makeLcg, mulberry32 as makeMulberry32, makeLcg as makeRandom } from "@m
  * Only `\r` and `\n`, deliberately — NOT `\s`. Runs of spaces and tabs pass through exactly as the source wrote them
  * (OA's IA extract writes `NORTH`, three spaces, `MAIN STREET`), because those could always appear and every shard
  * built to date contains them. Widening this to `\s+` silently rewrites values on rows with no line break at all.
- * `scaffold.test.ts` pins both halves.
+ * `scaffold.test.ts` pins both halves, against both readers.
  */
-export function* readCSVRecords(source) {
+function toRecord(header, cells) {
+    const record = {};
+    for (let i = 0; i < header.length; i++) {
+        record[header[i]] = (cells[i] ?? "").replaceAll(/[\r\n]+/g, " ").trim();
+    }
+    return record;
+}
+/**
+ * Read a CSV as header-keyed records.
+ *
+ * Returns the spliterator's own {@linkcode AsyncSequence}, so a caller composes `take`/`drop`/`filter` onto it — those
+ * ops fuse into one pull loop, and a `take` that is satisfied closes the source's file handle on the way out. Wrapping
+ * this in an `async function*` costs an async frame per row and takes those ops away; don't.
+ *
+ * A source at or below the spliterator's 128 KiB bulk threshold is read whole and parsed by the synchronous engine, so
+ * this is also the right reader for small sources — there is no buffered variant to reach for.
+ */
+export function readCSVRecords(source) {
     let header = null;
-    // `header: false` keeps the first record in the stream so this reader owns the lower-casing.
-    for (const cells of CSVSpliterator.from(source, { header: false, enableQuoteHandling: true })) {
-        if (!header) {
-            header = cells.map((name) => name.trim().toLowerCase());
-            continue;
-        }
-        const record = {};
-        for (let i = 0; i < header.length; i++) {
-            record[header[i]] = (cells[i] ?? "").replaceAll(/[\r\n]+/g, " ").trim();
-        }
-        yield record;
+    return CSVSpliterator.fromAsync(source, CSV_INIT)
+        .filter((cells) => {
+        if (header)
+            return true;
+        header = toHeader(cells);
+        return false;
+    })
+        .map((cells) => toRecord(header, cells));
+}
+/**
+ * {@link readCSVRecords} over one member of a zip archive — what every recipe reading a cached OA source wants.
+ *
+ * A source a checkout has not cached yields nothing, after saying so. A lab holds the archives for the countries it has
+ * built, so a recipe naming ten sources routinely finds three, and the `unzip -p` subprocesses these replaced behaved
+ * the same way by accident — a non-zero exit warned and returned no rows. A recipe that ends up with NO tuples at all
+ * still throws; that is the case where the cache, not the recipe, is the problem.
+ */
+export function readZippedCSVRecords(archivePath, entryName) {
+    if (!existsSync(String(archivePath))) {
+        console.error(`  WARN: ${archivePath} is not cached — skipping ${entryName}`);
+        return AsyncSequence.from([]);
     }
+    return readCSVRecords(readZipEntry(archivePath, entryName));
 }
 /**
  * Stream-parse a tuples JSONL file, yielding each parsed object (blank/invalid lines skipped).

package/out/src/shard-recipes/scaffold.js.map

@@ -1 +1 @@
-{"version":3,"file":"scaffold.js","sourceRoot":"","sources":["../../../src/shard-recipes/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,SAAiB,EAAE,KAAyC;IACzF,OAAO,cAAc,CAAC,SAAS,EAAE,KAAwD,CAAC,CAAA;AAC3F,CAAC;AAaD;;;;;;;;;;GAUG;AACH,OAAO,EAAE,OAAO,EAAE,UAAU,IAAI,cAAc,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAQpG;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,SAAS,CAAC,CAAC,cAAc,CAAC,MAAkB;IACjD,IAAI,MAAM,GAAoB,IAAI,CAAA;IAElC,6FAA6F;IAC7F,KAAK,MAAM,KAAK,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QAC/F,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;YAEvD,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAc,EAAE,CAAA;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;QACzE,CAAC;QAED,MAAM,MAAM,CAAA;IACb,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,KAAa;IAC9C,yFAAyF;IACzF,6EAA6E;IAC7E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAA;IAEvD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,MAAM,KAAK,GAAG,cAAc,CAAa,OAAO,CAAC,CAAA;QAEjD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACpB,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;AACF,CAAC;AAgBD;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC5B,KAA6B,EAC7B,SAA4B,EAC5B,WAAmB,EACnB,cAA6B,IAAI;IAEjC,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;IAErE,IAAI,CAAC,OAAO,CAAC,GAAG;QAAE,OAAO,KAAK,CAAA;IAC9B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;IAEvG,OAAO,IAAI,CAAA;AACZ,CAAC"}
+{"version":3,"file":"scaffold.js","sourceRoot":"","sources":["../../../src/shard-recipes/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAGxD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAEjC;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,SAAiB,EAAE,KAAyC;IACzF,OAAO,uBAAuB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;AACjD,CAAC;AA4CD;;;;;;;;;;GAUG;AACH,OAAO,EAAE,OAAO,EAAE,UAAU,IAAI,cAAc,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAQpG;;;;;;;;GAQG;AACH,MAAM,QAAQ,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,IAAI,EAAW,CAAA;AAEtE;;;;GAIG;AACH,SAAS,QAAQ,CAAC,KAAwB;IACzC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;AACtD,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,QAAQ,CAAC,MAAyB,EAAE,KAAwB;IACpE,MAAM,MAAM,GAAc,EAAE,CAAA;IAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACzE,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,cAAc,CAAC,MAA8C;IAC5E,IAAI,MAAM,GAAoB,IAAI,CAAA;IAElC,OAAO,cAAc,CAAC,SAAS,CAAW,MAAM,EAAE,QAAQ,CAAC;SACzD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,IAAI,MAAM;YAAE,OAAO,IAAI,CAAA;QAEvB,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAExB,OAAO,KAAK,CAAA;IACb,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAO,EAAE,KAAK,CAAC,CAAC,CAAA;AAC3C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,WAA4B,EAAE,SAAiB;IACnF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;QACtC,OAAO,CAAC,KAAK,CAAC,WAAW,WAAW,6BAA6B,SAAS,EAAE,CAAC,CAAA;QAE7E,OAAO,aAAa,CAAC,IAAI,CAAY,EAAE,CAAC,CAAA;IACzC,CAAC;IAED,OAAO,cAAc,CAAC,YAAY,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC,CAAA;AAC5D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,KAAa;IAC9C,yFAAyF;IACzF,6EAA6E;IAC7E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAA;IAEvD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,MAAM,KAAK,GAAG,cAAc,CAAa,OAAO,CAAC,CAAA;QAEjD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACpB,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;AACF,CAAC;AAgBD;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC5B,KAA6B,EAC7B,SAA4B,EAC5B,WAAmB,EACnB,cAA6B,IAAI;IAEjC,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;IAErE,IAAI,CAAC,OAAO,CAAC,GAAG;QAAE,OAAO,KAAK,CAAA;IAC9B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;IAEvG,OAAO,IAAI,CAAA;AACZ,CAAC"}

package/out/src/shard-recipes/street-affix.d.ts

@@ -26,10 +26,64 @@
  *   to keep the postcode-ORDER distribution multi-locale so a US-heavy affix shard doesn't dilute
  *   FR/DE postcode (the v0.9.8 blemish).
  */
+import type { ComponentTag } from "@mailwoman/core/types";
 import { type ShardRecipe } from "./scaffold.ts";
+/**
+ * A real US skeleton tuple read from a cached OA zip.
+ */
+interface USTuple {
+    house_number: string;
+    street: string;
+    locality: string;
+    region: string;
+    postcode: string;
+    base_source_id: string;
+}
+export type SuffixBoundaryClass = "terminal-only" | "terminal-contrast";
+/**
+ * Classify a real street surface for #1569. `terminal-only` has an ambiguous suffix-eligible name word immediately
+ * before a different terminal type (`Blue Hill Rd`); `terminal-contrast` ends at the ambiguous word itself (`Sutton
+ * Hollow`). The contrast check intentionally wins when both final words are name-prone: only the LAST token is the
+ * suffix under the canonical rule.
+ */
+export declare function classifySuffixBoundaryStreet(street: string): SuffixBoundaryClass | null;
+/**
+ * Layout-shell options for {@link renderRow}. `cuts` are the cumulative random() cutoffs for [full, bare, street-only] —
+ * the remainder is the venue shell. Defaults reproduce the original street-affix distribution (40/25/20/15) with the
+ * six template venues.
+ */
+interface RenderRowOpts {
+    venues?: readonly string[];
+    cuts?: readonly [number, number, number];
+}
+/**
+ * Embed the rendered street in a RANDOM layout so the model recognizes affixes wherever the street sits: full address,
+ * bare house+street, street-only (pure affix parse), or venue-prefixed.
+ */
+export declare function renderRow(random: () => number, base: USTuple, street: string, streetComponents: Partial<Record<ComponentTag, string>>, opts?: RenderRowOpts): {
+    fmt: string;
+    raw: string;
+    components: Partial<Record<ComponentTag, string>>;
+};
 /**
  * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
  * and `description` below for the surface form it generates.
  */
 export declare const streetAffixRecipe: ShardRecipe;
+/**
+ * #1569 root-fix shard. Both classes come from real non-Vermont OA streets and use the affix recipe's existing layout
+ * diversity. v4.3.1 makes terminal-only 80% of the mix: the first 40/60 run moved a 100-row TRAIN sample only 4→11
+ * while contrast was already 95/100 before training (93/100 after). Post-run audit found that the global affix relabel
+ * pass corrupts many already-decomposed target rows into double suffixes; do not retrain this recipe until relabel is
+ * idempotent over a decomposed street family. The 20% contrast leg remains explicit, additive to the already-strong
+ * base distribution, and B2 still gates it unchanged.
+ *
+ * V2 (corpus 0.19.0, 2026-08-10 recipe review): the v4.3.3 board split (rich venue-led rows 5/43 vs bare 53/65) showed
+ * the model separating template rows from real ones, and the venue shell was the giveaway — six fixed venue strings. v2
+ * draws the venue shell from thousands of REAL HRSA facility names and raises its share (venue 30%, full 35%, bare 20%,
+ * street-only 15%). Dose policy moved to the recipe's config side: weight ≤4 effective passes per run (Muennighoff
+ * 2023's repetition knee) and the source is excluded from the augmentation pool — see the v4.4.0 config.
+ */
+export declare const suffixBoundaryRecipe: ShardRecipe;
+export {};
 //# sourceMappingURL=street-affix.d.ts.map

package/out/src/shard-recipes/street-affix.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"street-affix.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/street-affix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAmBH,OAAO,EAAkC,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAuVhF;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,WAiM/B,CAAA"}
+{"version":3,"file":"street-affix.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/street-affix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAWH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAQzD,OAAO,EAAuE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAuFrH;;GAEG;AACH,UAAU,OAAO;IAChB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,MAAM,CAAA;CACtB;AAmGD,MAAM,MAAM,mBAAmB,GAAG,eAAe,GAAG,mBAAmB,CAAA;AAEvE;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAevF;AAsFD;;;;GAIG;AACH,UAAU,aAAa;IACtB,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B,IAAI,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;CACxC;AAED;;;GAGG;AACH,wBAAgB,SAAS,CACxB,MAAM,EAAE,MAAM,MAAM,EACpB,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,EACvD,IAAI,GAAE,aAAkB,GACtB;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;CAAE,CA8BjF;AAyDD;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,WAiM/B,CAAA;AAED;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,oBAAoB,EAAE,WA0JlC,CAAA"}