@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/tools/{slice → overlay}/translit.ts

@@ -3,46 +3,50 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Build per-script parquet slices from the DeepSeek-generated transliteration JSONL and emit the
- *   corpus-v0.4.0 MANIFEST that combines them with the existing kryptonite + v0.3.0 slices.
+ *   Build per-script parquet files from the DeepSeek-generated transliteration jsonl and emit the
+ *   corpus-v0.4.0 manifest that combines them with the existing kryptonite + v0.3.0 files.
  *
- *   Sibling to `slice-kryptonite.ts`. The two modules share the same composition pattern: take a
- *   base MANIFEST, append new slices, write a combined MANIFEST. Differences specific to
+ *   This module sits beside `kryptonite.ts`. Both modules use the same composition pattern: take a
+ *   base manifest, append new parquet files, write a combined manifest. Differences specific to
  *   transliteration:
  *
- *   - One JSONL contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
- *       `source` and write one slice per script so `audit.ts` can attribute each slice to its
+ *   - One jsonl contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
+ *       `source` and write one parquet file per script so `audit.ts` can attribute each file to its
  *       synthetic source without relying on filename-prefix inference.
- *   - Each slice is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
+ *   - Each file is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
  *       `part-0000.parquet`, which v0.4.0's first builder already produced).
- *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 slice paths are
- *       rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined MANIFEST so
+ *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 file paths are
+ *       rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined manifest so
  *       all paths share one container-friendly form.
  *
- *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for prompts, model, and the
- *   reproducibility contract.
+ *   See docs/records/engineering/CORPUS_V0_4_0_GENERATION.mdx for prompts, model and the
+ *   reproducibility interface.
  *
  *   Invoke via `mailwoman corpus slice translit \
  *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
- *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/MANIFEST.json \
+ *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/manifest.json \
  *   --out-dir /data/corpus/versioned/v0.4.0`
  */
 
-import { mailwomanDataRoot } from "@mailwoman/core/data-root"
+import { dataRootPath } from "@mailwoman/core/data-root"
+import { delimitedSource } from "@mailwoman/core/fs/delimited"
 import { pathExists, readLocalJSONFile, tryStat } from "@mailwoman/core/fs/readers"
 import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers"
 import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
 import { JSONSpliterator } from "spliterator"
 
-import type { CanonicalRow, LabeledRow } from "#types"
-import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, rowToParquet, SLICE_COMPRESSION, writeParquetRows } from "#utils"
-import type { SliceDescriptor, SliceManifest } from "#utils"
+import { corpusDirectoryName } from "#directory"
+import { PARQUET_COLUMNS, PARQUET_COMPRESSION, ROW_GROUP_SIZE, rowToParquet } from "#parquet/schema"
+import type { ParquetFileDescriptor, ParquetManifest } from "#parquet/writers"
+import { writeParquetFile } from "#parquet/writers"
+import { type CanonicalRow, type LabeledRow, requireSurface } from "#types"
+import { alignRow } from "#utils"
 
-export interface SliceTranslitOptions {
+export interface TranslitOverlayOptions {
 	jsonl: string
 	baseManifest: string
-	outDir: string
+	outDir: PathBuilderLike
 	/**
 	 * Default `"0.4.0"`.
 	 */
@@ -52,9 +56,11 @@ export interface SliceTranslitOptions {
 	 */
 	canonicalPathPrefix?: string
 	/**
-	 * Prefix the base manifest's slice paths currently carry, to be rewritten to
-	 * {@link SliceTranslitOptions.canonicalPathPrefix}. Defaults to `mailwomanDataRoot()` with a trailing slash — the
-	 * root that WROTE those paths. Pass it explicitly when translating a manifest generated under a different
+	 * Prefix currently used in the base manifest's file paths, to be rewritten
+	 * to {@link TranslitOverlayOptions.canonicalPathPrefix}.
+	 *
+	 * Defaults to `dataRootPath()` with a trailing slash — the root that wrote those paths.
+	 * Pass it explicitly when translating a manifest generated under a different
 	 * `$MAILWOMAN_DATA_ROOT` than the one you are running with.
 	 */
 	legacyPathPrefix?: string
@@ -70,15 +76,23 @@ function toCanonicalRow(raw: Record<string, unknown>, corpusVersion: string): Ca
 		source_id: raw["source_id"] as string,
 		corpus_version: corpusVersion,
 		license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
-		synth: raw["synth"] as CanonicalRow["synth"],
+		recipe: raw["recipe"] as CanonicalRow["recipe"],
+		register: (raw["register"] as string | null) ?? null,
+		surface: requireSurface(raw, "translit"),
 	}
 }
 
 /**
- * Write one slice for a single source slug. Returns the populated SliceDescriptor + a list of quarantine reasons for
- * rows that failed alignment.
+ * Write one parquet file for a single source slug.
+ *
+ * @returns The populated ParquetFileDescriptor + a list of quarantine reasons
+ * for rows that failed alignment.
  */
-async function writeOneSlice(rows: readonly LabeledRow[], outPath: string, source: string): Promise<SliceDescriptor> {
+async function writeOneFile(
+	rows: readonly LabeledRow[],
+	outPath: string,
+	source: string
+): Promise<ParquetFileDescriptor> {
 	let firstSourceID = ""
 	let lastSourceID = ""
 
@@ -90,7 +104,7 @@ async function writeOneSlice(rows: readonly LabeledRow[], outPath: string, sourc
 		lastSourceID = row.source_id
 	}
 
-	await writeParquetRows(rows.map(rowToParquet), outPath)
+	await writeParquetFile(rows.map(rowToParquet), outPath)
 
 	const fileStat = await tryStat(outPath)
 	const sha256 = await sha256File(outPath)
@@ -99,46 +113,47 @@ async function writeOneSlice(rows: readonly LabeledRow[], outPath: string, sourc
 		split: "train",
 		path: outPath,
 		format: "parquet",
-		compression: SLICE_COMPRESSION,
+		compression: PARQUET_COMPRESSION,
 		rows: rows.length,
 		bytes: fileStat?.size ?? 0,
 		sha256,
 		first_source_id: firstSourceID,
 		last_source_id: lastSourceID,
-		// Stamp source so audit.ts attributes the slice without falling back to filename-prefix inference.
+		// Stamp source so audit.ts attributes the file without falling back to filename-prefix inference.
 		source,
 	}
 }
 
-function canonicalizeSlicePath(path: string, legacyPrefix: string, canonicalPrefix: string): string {
+function canonicalizeFilePath(path: string, legacyPrefix: string, canonicalPrefix: string): string {
 	if (path.startsWith(legacyPrefix)) return canonicalPrefix + path.slice(legacyPrefix.length)
 
 	return path
 }
 
-export async function buildTranslitSlice(
-	options: SliceTranslitOptions,
+export async function buildTranslitOverlay(
+	options: TranslitOverlayOptions,
 	report?: (line: string) => void
 ): Promise<void> {
 	const corpusVersion = options.corpusVersion ?? "0.4.0"
-	// This is a portable manifest namespace, not a host filesystem default.
+	// This is a portable manifest namespace rather than a host filesystem default.
 	const canonicalPathPrefix = options.canonicalPathPrefix ?? "/data/"
-	const legacyPathPrefix = options.legacyPathPrefix ?? `${mailwomanDataRoot()}/`
+	const legacyPathPrefix = options.legacyPathPrefix ?? `${dataRootPath()}/`
 
 	if (!(await pathExists(options.jsonl))) throw new Error(`jsonl not found: ${options.jsonl}`)
 
 	if (!(await pathExists(options.baseManifest))) throw new Error(`base-manifest not found: ${options.baseManifest}`)
 
-	const corpusDir = join(options.outDir, `corpus-v${corpusVersion}`)
-	const trainDir = join(corpusDir, "train")
+	const corpusDir = PathBuilder.from(options.outDir)(corpusDirectoryName(corpusVersion))
+	const trainDir = corpusDir("train")
 	await makeDirectories(trainDir)
 
-	// Bucket canonical rows by source. Quarantined rows are logged.
+	// Bucket canonical rows by source.
+	// Quarantined rows are logged.
 	const buckets = new Map<string, LabeledRow[]>()
 	const quarantine: string[] = []
 	let totalIn = 0
 
-	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(options.jsonl)) {
+	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(delimitedSource(options.jsonl))) {
 		totalIn++
 		const canon = toCanonicalRow(raw, corpusVersion)
 		const result = alignRow(canon)
@@ -160,42 +175,43 @@ export async function buildTranslitSlice(
 
 	report?.(`read ${totalIn} rows; ${quarantine.length} quarantined; ${buckets.size} script buckets`)
 
-	const newSlices: SliceDescriptor[] = []
+	const newFiles: ParquetFileDescriptor[] = []
 	const sortedKeys = [...buckets.keys()].toSorted()
 
 	for (const source of sortedKeys) {
 		const rows = buckets.get(source)!
 		const slug = source.startsWith("deepseek-translit-") ? source.slice("deepseek-translit-".length) : source
-		const outPath = join(trainDir, `part-translit-${slug}.parquet`)
-		const descriptor = await writeOneSlice(rows, outPath, source)
-		newSlices.push(descriptor)
+		// A string, because the manifest records it.
+		const outPath = trainDir(`part-translit-${slug}.parquet`).toString()
+		const descriptor = await writeOneFile(rows, outPath, source)
+		newFiles.push(descriptor)
 		report?.(`  ${source}: ${descriptor.rows} rows → ${outPath} (${descriptor.bytes} bytes)`)
 	}
 
 	if (quarantine.length) {
-		const qPath = join(corpusDir, "quarantine-transliteration.tsv")
-		await writeLocalTextFile(quarantine.join("\n") + "\n", qPath)
+		const qPath = corpusDir("quarantine-transliteration.tsv")
+		await writeLocalTextFile(quarantine, qPath)
 		report?.(`quarantine log → ${qPath} (${quarantine.length} rows)`)
 	}
 
-	// Compose final MANIFEST: rewrite base.slices paths from /mnt/playpen/... → /data/... and append
-	// the new translit slices. Kryptonite slice already lives in the base manifest (it was written
-	// there by Thread B).
-	const base = await readLocalJSONFile<SliceManifest>(options.baseManifest)
+	// Compose final manifest: rewrite the base's file paths from the data root → /data/...
+	// And append the new translit files.
+	// The kryptonite file already lives in the base manifest (it was written there by Thread B).
+	const base = await readLocalJSONFile<ParquetManifest>(options.baseManifest)
 
-	const rewrittenBase = base.slices.map((sh) => ({
-		...sh,
-		path: canonicalizeSlicePath(sh.path, legacyPathPrefix, canonicalPathPrefix),
+	const rewrittenBase = base.slices.map((file) => ({
+		...file,
+		path: canonicalizeFilePath(file.path, legacyPathPrefix, canonicalPathPrefix),
 	}))
 
-	const newTrainRows = newSlices.reduce((sum, sh) => sum + sh.rows, 0)
+	const newTrainRows = newFiles.reduce((sum, file) => sum + file.rows, 0)
 
-	const combined: SliceManifest = {
+	const combined: ParquetManifest = {
 		corpus_version: corpusVersion,
 		schema: PARQUET_COLUMNS,
 		rows_per_slice: base.rows_per_slice,
 		row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
-		slices: [...rewrittenBase, ...newSlices],
+		slices: [...rewrittenBase, ...newFiles],
 		counts: {
 			train: base.counts.train + newTrainRows,
 			val: base.counts.val,
@@ -204,15 +220,15 @@ export async function buildTranslitSlice(
 		total_rows: base.total_rows + newTrainRows,
 	}
 
-	const combinedPath = join(corpusDir, "MANIFEST.json")
+	const combinedPath = corpusDir("MANIFEST.json")
 	await writeLocalJSONFile(combined, combinedPath)
 	report?.(`wrote combined manifest → ${combinedPath}`)
 	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newTrainRows})`)
-	report?.(`  slices=${combined.slices.length} (base=${base.slices.length}, added=${newSlices.length})`)
-	report?.(`  compression=${SLICE_COMPRESSION}`)
+	report?.(`  files=${combined.slices.length} (base=${base.slices.length}, added=${newFiles.length})`)
+	report?.(`  compression=${PARQUET_COMPRESSION}`)
 	const pathFix = rewrittenBase.filter((s, i) => s.path !== base.slices[i]!.path).length
 
 	if (pathFix > 0) {
-		report?.(`  path-canonicalized base slices: ${pathFix} (legacy '${legacyPathPrefix}' → '${canonicalPathPrefix}')`)
+		report?.(`  path-canonicalized base files: ${pathFix} (legacy '${legacyPathPrefix}' → '${canonicalPathPrefix}')`)
 	}
 }

package/lib/tools/overture-subvenue.ts

@@ -3,89 +3,30 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Overture Places source for the sub-venue lexicon (#35 wave 2) — read the sub-venue-bearing
- *   category slices of `poi.db` (spatial layer #1, `$MAILWOMAN_DATA_ROOT/poi/poi.db`) and yield
- *   {@link SubVenueHarvestRow}s the lexicon builder consumes exactly like an OSM extract.
+ * Read selected Overture Places categories from `poi.db` and return candidate sub-venue names with
+ * country and category provenance.
  *
- *   ── WHY OVERTURE AT ALL, when the OSM extractor already exists ────────────────────────────────────
- *   Wave 1 measured `concourse` at 21 real Overture rows against 4 in the whole Great Britain OSM
- *   extract, 3 of which are a street called CONCOURSE WAY. Overture's `airport_terminal` category is
- *   curated venue-interior naming; OSM's `aeroway=terminal` is a building footprint that usually
- *   carries the AIRPORT's name. The two sources fail differently, so both are read.
- *
- *   ── THE CATEGORY SET IS MEASURED, NOT GUESSED ────────────────────────────────────────────────────
- *   A full scan of all 13,681,698 rows (2026-08-05, poi.db vintage 2026-05-20.0) counted, per
- *   category, how many named rows carry a designator token. The ranking is NOT what a category name
- *   predicts — `gas_station` leads the whole table with 12,996 hits, every one of them the token
- *   `station` inside "Holiday Station" / "Chevron Station Seward", and `shoe_store` contributes 708
- *   hits of `wing` because Red Wing sells boots. {@link OVERTURE_SUBVENUE_CATEGORIES} is the four
- *   categories whose hits survived reading the distribution; the rejects are listed below it so nobody
- *   re-proposes them.
- *
- *   ── poi.db IS FOUR COUNTRIES ─────────────────────────────────────────────────────────────────────
- *   Measured the same day: US 11,521,612 / CA 794,418 / FR 721,352 / MX 644,316, and nothing else. The
- *   shipped layer is not a world gazetteer, so Overture can attest en-US, en-CA, fr-FR and es-MX
- *   surfaces and NOTHING ELSE. Every non-Latin designator the corpus task asks for — `ターミナル`,
- *   `Halle`, `Flügel` — has to come from the OSM leg. Do not read a zero count here as evidence of
- *   absence in the world; it is evidence of absence in four countries.
- *
- *   ── The row shape fits; the PROVENANCE STAMP did not ─────────────────────────────────────────────
- *   Wave 1's `OSMSubVenueRow` was written to accept a non-OSM row, and it does: an Overture row is
- *   `{ designatorID, name }` with no `ref` and no `localizedNames`. What did NOT fit is
- *   `extractAttestedPhrases`, which hardcoded `osm:name` as the surface's `source`. Feeding Overture
- *   rows through it unchanged would have labelled every Overture surface as OSM-attested — a
- *   provenance lie, and under ODbL a consequential one. Hence the `source`/`region` options on that
- *   function and the source-neutral {@link SubVenueHarvestRow} name.
+ * `poi.db` covers four countries — US, CA, FR and MX — so a zero count here is absence in four countries rather
+ * than evidence of absence in the world. Every non-Latin designator comes from the OSM leg.
  */
 
 import { DatabaseClient } from "@mailwoman/sqlite/client"
+import type { PathBuilderLike } from "path-ts"
 
 import type { SubVenueHarvestRow } from "#tools/sub/venue/lexicon"
 
 /**
- * Overture Places category → the designator its rows attest.
- *
- * Each entry carries the 2026-08-05 full-scan measurement in a comment: `rows` is the category's total, `hits` is how
- * many of its named rows contain any designator token, and the token list is the top of that distribution.
- *
- * REJECTED, with the number that rejected them — do not re-add without a fresh census:
- *
- * - `gas_station` (12,996 hits) — all `station` inside a brand name (Holiday Station, Chevron Station Seward).
- * - `fire_station` (10,377) — same, plus 330 `hall` from fire halls, which are venues rather than sub-venues.
- * - `town_hall` (5,528) — `City Hall` is a whole building, the `venue` tier, not interior structure.
- * - `building_supply_store` (4,030) — `building` inside "Allied Building Products".
- * - `jehovahs_witness_place_of_worship` (2,334) — every hit is "Kingdom Hall of Jehovah's Witnesses".
- * - `shoe_store` (909, of which 708 `wing`) — Red Wing. This one is a CONFOUND BOARD entry, not a source.
- * - `college_university` (3,697) — 2,082 `campus`, but the row names the whole institution; `campus_building` is the
- *   interior slice and is kept instead.
- * - `airport` (6,000 rows, 4,302 hits) — 4,071 of them are the token `airport` in the aerodrome's own name. Venue tier,
- *   already covered by OurAirports, and it drowns the interior signal.
- * - `transport_interchange` (1 row), `rail_facility_or_service` (81 rows, 2 hits) — too small to matter.
- * - `public_transit_facility_or_service` (2,274 rows, 185 hits) — 161 are `station` naming the station itself.
+ * Overture categories selected for sub-venue designator evidence.
  */
 export const OVERTURE_SUBVENUE_CATEGORIES: Readonly<Record<string, string>> = {
-	// 1,022 rows / 669 hits — terminal 266, concourse 19, gate 13, hall 4. The densest interior naming
-	// in the layer and the reason this reader exists.
 	airport_terminal: "terminal",
-	// 7,366 rows / 3,382 hits — hall 2,134, building 1,019, campus 209. US academic halls and numbered
-	// campus buildings: "UAA Cuddy Hall", "UAA Science Building". The row is a building ON a campus, so
-	// `campus` is its CONTEXT designator; the phrase found inside the name decides the record it
-	// attests, which for these rows is mostly `hall`.
 	campus_building: "campus",
-	// 443 rows / 301 hits — pier 282, terminal 17.
 	pier: "pier",
-	// 388 rows / 66 hits — airport 36, terminal 20, concourse 5, gate 3. Small, but the hits are almost
-	// all genuine ("Delta Sky Club Concourse B").
 	airport_lounge: "terminal",
 }
 
 /**
- * The `poi` and `poi_category_codes` columns this reader touches, declared LOCALLY.
- *
- * `@mailwoman/resolver-wof-sqlite` owns the full `POIDatabase` interface, and `@mailwoman/corpus` does not depend on it
- * — the same dependency-direction call `sub-venue-lexicon.ts` makes for `@mailwoman/osm`'s row type. This is a
- * read-only projection of four columns; `overture-subvenue.test.ts` builds a fixture with exactly this DDL, so a column
- * rename upstream fails a test here rather than throwing at runtime against a 3.9 GB database nobody has in CI.
+ * Read-only subset of POI database tables used by this extractor.
  */
 interface POIReadDatabase {
 	poi: {
@@ -104,14 +45,9 @@ interface POIReadDatabase {
 }
 
 /**
- * The layer's `source_vintage` — the Overture release the rows came from (`2026-05-20.0`), for the lexicon's
- * `sources[]`.
- *
- * Read off the database rather than passed in, because a vintage a caller types is a vintage that goes stale silently.
- * The layer-contract tables are part of every layer database by construction — see `docs/engineering/reference/
- * layer-contract.mdx` — so there is no version of poi.db where this is absent.
+ * Read the Overture release recorded in the POI layer manifest.
  */
-export async function readOvertureLayerVintage(databasePath: string): Promise<string> {
+export async function readOvertureLayerVintage(databasePath: PathBuilderLike): Promise<string> {
 	using kdb = new DatabaseClient<POIReadDatabase>(databasePath, { readOnly: true })
 
 	const row = await kdb
@@ -125,44 +61,35 @@ export async function readOvertureLayerVintage(databasePath: string): Promise<st
 
 export interface ReadOvertureSubVenuesOptions {
 	/**
-	 * Path to `poi.db`. Typically `dataRootPath("poi", "poi.db")`.
+	 * Path to `poi.db`.
 	 */
-	databasePath: string
+	databasePath: PathBuilderLike
 	/**
-	 * Category → designator map. Defaults to {@link OVERTURE_SUBVENUE_CATEGORIES}.
+	 * Category-to-designator mapping.
 	 */
 	categories?: Readonly<Record<string, string>>
 	/**
-	 * Restrict to these ISO 3166-1 alpha-2 countries. Omit for all four the layer carries.
+	 * Optional ISO alpha-2 country filter.
 	 */
 	countries?: readonly string[]
 }
 
 /**
- * One Overture row plus the country it came from — {@link SubVenueHarvestRow} widened by the field the layer has and an
- * OSM extract does not (a Geofabrik extract's country is a property of the invocation, so OSM rows carry `country:
- * ""`).
+ * Overture sub-venue row with its country and category provenance.
  */
 export interface OvertureSubVenueRow extends SubVenueHarvestRow {
 	/**
-	 * ISO 3166-1 alpha-2, straight off the Overture partition.
+	 * ISO alpha-2 code from the Overture partition.
 	 */
 	country: string
 	/**
-	 * The Overture category the row was read from, so a surface's provenance survives past the designator.
+	 * Overture category used to select the row.
 	 */
 	category: string
 }
 
 /**
- * Read the sub-venue-bearing category slices of `poi.db`.
- *
- * Cold path, already async, no interface constraint — so Kysely, per the repo's inline-SQL rule.
- *
- * `category_id` is the SECOND component of poi.db's clustered `(h3_cell, category_id, …)` primary key, which reads like
- * a full 13.7M-row scan and is not one: SQLite skip-scans the leading `h3_cell` and the whole four-category read
- * returned 9,219 rows in **1.4 s** on the shipped layer (2026-08-05). A JS-side scan of every row to survey the same
- * question took 52 s, which is the number to remember if you are tempted to filter in JS instead.
+ * Read POI rows with names in selected sub-venue categories, optionally filtered by country.
  */
 export async function readOvertureSubVenues(options: ReadOvertureSubVenuesOptions): Promise<OvertureSubVenueRow[]> {
 	const categories = options.categories ?? OVERTURE_SUBVENUE_CATEGORIES