@mailwoman/corpus 8.6.0 → 9.1.0

package/src/shard-recipes/scaffold.ts

@@ -9,8 +9,15 @@
  *   only its synthesis + filter; the `mailwoman corpus shard <recipe>` command supplies the I/O.
  */
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
+import { existsSync } from "node:fs"
+
+import { readZipEntry } from "@mailwoman/core/fs/zip"
+import { tryParsingJSON } from "@mailwoman/core/objects"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import { alignRow } from "@mailwoman/corpus/utils"
+import type { PathBuilderLike } from "path-ts"
+import type { AsyncChunkIterator, AsyncDataResource } from "spliterator"
+import { AsyncSequence, CSVSpliterator } from "spliterator"
 
 /**
  * {@link stableSourceID}, but accepting arbitrary disambiguator keys (e.g. a variant index `v`) that aren't
@@ -33,39 +40,106 @@ export interface ShardTuple {
 }
 
 /**
- * The deterministic LCG (`s = s*1664525 + 1013904223 mod 2^32`) the street/po-box/anchor builders used. A recipe whose
- * legacy `.mjs` seeded this must create it here so `--seed N` is byte-reproducible.
+ * The two seeded generators the legacy `build-*-shard` scripts used, re-exported from their home in
+ * `@mailwoman/core/utils` so a recipe keeps importing everything it needs from this one scaffold module.
+ *
+ * - `makeLcg` (`s = s*1664525 + 1013904223 mod 2^32`) — what the street/po-box/anchor builders seeded.
+ * - `makeMulberry32` — what the MAJORITY of them used (german, locale, boundary-stress, unit, fr-order, country-balanced,
+ *   intersection, fr-admin-split, street-affix, street-bare, po-box-cedex).
+ *
+ * A recipe must seed the same one its `.mjs` did, the same way it did (usually `seed`, but some derive a per-stream
+ * seed), or `--seed N` stops being byte-reproducible.
+ */
+export { makeLcg, mulberry32 as makeMulberry32, makeLcg as makeRandom } from "@mailwoman/core/utils"
+
+/**
+ * One CSV record, keyed by its lower-cased header name, every value trimmed. A column the header does not declare reads
+ * as `undefined`; a column the header declares but this record does not reach reads as `""`.
  */
-export function makeLcg(seed: number): () => number {
-	let s = seed >>> 0
+export type CSVRecord = Record<string, string | undefined>
 
-	return () => {
-		s = (s * 1_664_525 + 1_013_904_223) % 4_294_967_296
+/**
+ * Quote handling spans the ROW split, not only the column split: a newline inside a quoted field belongs to a single
+ * record, so the record boundaries can only be found by a scanner that already knows where the quotes are. Find the
+ * lines first and unquote afterwards and such a record splits in two — the first half short by however many columns
+ * followed the newline, which is how a `street` value comes to be read as `city`.
+ *
+ * `header: false` keeps the first record in the stream, so this module owns the lower-casing rather than
+ * `normalizeKeys` — see {@link toHeader}.
+ */
+const CSV_INIT = { header: false, enableQuoteHandling: true } as const
 
-		return s / 4_294_967_296
-	}
+/**
+ * Header names are lower-cased on the way in. `normalizeKeys` will not do it: it leaves an ALL CAPS header alone, and
+ * OpenAddresses ships `LON,LAT,NUMBER,STREET` while other extracts ship the same names lower-case. A recipe names its
+ * columns in lower case either way.
+ */
+function toHeader(cells: readonly string[]): string[] {
+	return cells.map((name) => name.trim().toLowerCase())
 }
 
 /**
- * Back-compat alias for {@link makeLcg}.
+ * Line breaks inside a value become single spaces. A quote-aware parse is the first thing here able to return a value
+ * CONTAINING one — `us/ia/statewide.csv` has 12, all unit designators like `"#2\n#2"` — and every consumer synthesizes
+ * one-line address text from these cells with no guard, because until that parse landed no value could carry one.
+ * Collapsing keeps the record (the address is fine; the source's line break is not part of it) without emitting a
+ * training row with a newline inside it.
+ *
+ * Only `\r` and `\n`, deliberately — NOT `\s`. Runs of spaces and tabs pass through exactly as the source wrote them
+ * (OA's IA extract writes `NORTH`, three spaces, `MAIN STREET`), because those could always appear and every shard
+ * built to date contains them. Widening this to `\s+` silently rewrites values on rows with no line break at all.
+ * `scaffold.test.ts` pins both halves, against both readers.
  */
-export const makeRandom = makeLcg
+function toRecord(header: readonly string[], cells: readonly string[]): CSVRecord {
+	const record: CSVRecord = {}
+
+	for (let i = 0; i < header.length; i++) {
+		record[header[i]!] = (cells[i] ?? "").replaceAll(/[\r\n]+/g, " ").trim()
+	}
+
+	return record
+}
 
 /**
- * Mulberry32 — the PRNG the MAJORITY of the legacy `build-*-shard` scripts used (german, locale, boundary-stress, unit,
- * fr-order, country-balanced, intersection, fr-admin-split, street-affix, street-bare, po-box-cedex). A recipe must
- * seed it EXACTLY as its `.mjs` did (usually `seed`, but some derive a per-stream seed) to stay byte-reproducible.
+ * Read a CSV as header-keyed records.
+ *
+ * Returns the spliterator's own {@linkcode AsyncSequence}, so a caller composes `take`/`drop`/`filter` onto it — those
+ * ops fuse into one pull loop, and a `take` that is satisfied closes the source's file handle on the way out. Wrapping
+ * this in an `async function*` costs an async frame per row and takes those ops away; don't.
+ *
+ * A source at or below the spliterator's 128 KiB bulk threshold is read whole and parsed by the synchronous engine, so
+ * this is also the right reader for small sources — there is no buffered variant to reach for.
  */
-export function makeMulberry32(seed: number): () => number {
-	let a = seed >>> 0
+export function readCSVRecords(source: AsyncDataResource | AsyncChunkIterator): AsyncSequence<CSVRecord> {
+	let header: string[] | null = null
 
-	return () => {
-		a = (a + 0x6d_2b_79_f5) | 0
-		let t = Math.imul(a ^ (a >>> 15), 1 | a)
-		t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
+	return CSVSpliterator.fromAsync<string[]>(source, CSV_INIT)
+		.filter((cells) => {
+			if (header) return true
 
-		return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296
+			header = toHeader(cells)
+
+			return false
+		})
+		.map((cells) => toRecord(header!, cells))
+}
+
+/**
+ * {@link readCSVRecords} over one member of a zip archive — what every recipe reading a cached OA source wants.
+ *
+ * A source a checkout has not cached yields nothing, after saying so. A lab holds the archives for the countries it has
+ * built, so a recipe naming ten sources routinely finds three, and the `unzip -p` subprocesses these replaced behaved
+ * the same way by accident — a non-zero exit warned and returned no rows. A recipe that ends up with NO tuples at all
+ * still throws; that is the case where the cache, not the recipe, is the problem.
+ */
+export function readZippedCSVRecords(archivePath: PathBuilderLike, entryName: string): AsyncSequence<CSVRecord> {
+	if (!existsSync(String(archivePath))) {
+		console.error(`  WARN: ${archivePath} is not cached — skipping ${entryName}`)
+
+		return AsyncSequence.from<CSVRecord>([])
 	}
+
+	return readCSVRecords(readZipEntry(archivePath, entryName))
 }
 
 /**
@@ -73,7 +147,7 @@ export function makeMulberry32(seed: number): () => number {
  */
 export async function* readTuples(input: string): AsyncGenerator<ShardTuple> {
 	// TextSpliterator (not JSONSpliterator) so a malformed line is SKIPPED, not thrown — the
-	// per-line try/catch below is the tolerance this reader has always had.
+	// per-line tryParsingJSON below is the tolerance this reader has always had.
 	const { TextSpliterator } = await import("spliterator")
 
 	for await (const line of TextSpliterator.fromAsync(input)) {
@@ -81,10 +155,10 @@ export async function* readTuples(input: string): AsyncGenerator<ShardTuple> {
 
 		if (!trimmed) continue
 
-		try {
-			yield JSON.parse(trimmed) as ShardTuple
-		} catch {
-			// skip malformed line
+		const tuple = tryParsingJSON<ShardTuple>(trimmed)
+
+		if (tuple !== null) {
+			yield tuple
 		}
 	}
 }
@@ -178,6 +252,30 @@ export interface ShardRecipeOpts {
 	 * `no-fragment` knob 3: minimum digit count for a number to count as "long" and be boosted. Default 3.
 	 */
 	longNumberMinDigits?: number
+	/**
+	 * `sub-venue`: the sub-venue lexicon JSON. Default = the committed `corpus/data/sub-venue-lexicon.json`, resolved
+	 * through the package manifest so it works from the source tree and from `out/`.
+	 */
+	lexicon?: string
+	/**
+	 * `sub-venue`: directory of `sub-venue-extract` JSONLs, one per region. Default
+	 * `$MAILWOMAN_DATA_ROOT/sub-venue/extracts`.
+	 */
+	extractsDir?: string
+	/**
+	 * `sub-venue`: the `poi.db` spatial layer, read for the en-US and fr-FR venue + confound pools (the two of poi.db's
+	 * four countries this shard has legs for). Default `$MAILWOMAN_DATA_ROOT/poi/poi.db`.
+	 */
+	poiDb?: string
+	/**
+	 * `sub-venue`: GB/US/FR address-context tuples JSONL. Default the house-venue v3 tuples
+	 * (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v3.jsonl`); DE and ES read OpenAddresses directly.
+	 */
+	subVenueTuples?: string
+	/**
+	 * `sub-venue`: share of emitted rows that are confound NEGATIVES. Default 0.3.
+	 */
+	negativeFraction?: number
 }
 
 /**