@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/tools/overlay/translit.ts

@@ -0,0 +1,236 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build per-script parquet files from the DeepSeek-generated transliteration jsonl and emit the
+ *   corpus-v0.4.0 manifest that combines them with the existing kryptonite + v0.3.0 files.
+ *
+ *   This module sits beside `kryptonite.ts`. Both modules use the same composition pattern: take a
+ *   base manifest, append new parquet files, write a combined manifest. Differences specific to
+ *   transliteration:
+ *
+ *   - One jsonl contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
+ *       `source` and write one parquet file per script so `audit.ts` can attribute each file to its
+ *       synthetic source without relying on filename-prefix inference.
+ *   - Each file is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
+ *       `part-0000.parquet`, which v0.4.0's first builder already produced).
+ *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 file paths are
+ *       rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined manifest so
+ *       all paths share one container-friendly form.
+ *
+ *   See docs/records/engineering/CORPUS_V0_4_0_GENERATION.mdx for prompts, model and the
+ *   reproducibility interface.
+ *
+ *   Invoke via `mailwoman corpus slice translit \
+ *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/manifest.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+
+import { dataRootPath } from "@mailwoman/core/data-root"
+import { delimitedSource } from "@mailwoman/core/fs/delimited"
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
+import { pathExists, tryStat } from "@mailwoman/core/fs/readers/stat"
+import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+import { JSONSpliterator } from "spliterator"
+
+import { corpusDirectoryName } from "#directory"
+import { PARQUET_COLUMNS, PARQUET_COMPRESSION, ROW_GROUP_SIZE, rowToParquet } from "#parquet/schema"
+import type { ParquetFileDescriptor, ParquetManifest } from "#parquet/writers"
+import { writeParquetFile } from "#parquet/writers"
+import { type CanonicalRow, type LabeledRow, requireSurface } from "#types"
+import { alignRow } from "#utils"
+
+export interface TranslitOverlayOptions {
+	jsonl: string
+	baseManifest: string
+	outDir: PathBuilderLike
+	/**
+	 * @defaultValue `"0.4.0"`
+	 */
+	corpusVersion?: string
+	/**
+	 * @defaultValue `"/data/"`
+	 */
+	canonicalPathPrefix?: string
+	/**
+	 * Prefix currently used in the base manifest's file paths, to be rewritten
+	 * to {@link TranslitOverlayOptions.canonicalPathPrefix}.
+	 *
+	 * Pass it explicitly when translating a manifest generated under a different
+	 * `$MAILWOMAN_DATA_ROOT` than the one you are running with.
+	 *
+	 * @defaultValue `dataRootPath()` with a trailing slash — the root that wrote those paths.
+	 */
+	legacyPathPrefix?: string
+}
+
+function toCanonicalRow(raw: Record<string, unknown>, corpusVersion: string): CanonicalRow {
+	return {
+		raw: raw["raw"] as string,
+		components: raw["components"] as Record<string, string>,
+		country: (raw["country"] as string) ?? "US",
+		locale: (raw["locale"] as string) ?? undefined,
+		source: raw["source"] as string,
+		source_id: raw["source_id"] as string,
+		corpus_version: corpusVersion,
+		license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
+		recipe: raw["recipe"] as CanonicalRow["recipe"],
+		register: (raw["register"] as string | null) ?? null,
+		surface: requireSurface(raw, "translit"),
+	}
+}
+
+/**
+ * Write one parquet file for a single source slug.
+ *
+ * @returns The populated ParquetFileDescriptor + a list of quarantine reasons
+ * for rows that failed alignment.
+ */
+async function writeOneFile(
+	rows: readonly LabeledRow[],
+	outPath: string,
+	source: string
+): Promise<ParquetFileDescriptor> {
+	let firstSourceID = ""
+	let lastSourceID = ""
+
+	for (const row of rows) {
+		if (firstSourceID === "") {
+			firstSourceID = row.source_id
+		}
+
+		lastSourceID = row.source_id
+	}
+
+	await writeParquetFile(rows.map(rowToParquet), outPath)
+
+	const fileStat = await tryStat(outPath)
+	const sha256 = await sha256File(outPath)
+
+	return {
+		split: "train",
+		path: outPath,
+		format: "parquet",
+		compression: PARQUET_COMPRESSION,
+		rows: rows.length,
+		bytes: fileStat?.size ?? 0,
+		sha256,
+		first_source_id: firstSourceID,
+		last_source_id: lastSourceID,
+		// Stamp source so audit.ts attributes the file without falling back to filename-prefix inference.
+		source,
+	}
+}
+
+function canonicalizeFilePath(path: string, legacyPrefix: string, canonicalPrefix: string): string {
+	if (path.startsWith(legacyPrefix)) return canonicalPrefix + path.slice(legacyPrefix.length)
+
+	return path
+}
+
+export async function buildTranslitOverlay(
+	options: TranslitOverlayOptions,
+	report?: (line: string) => void
+): Promise<void> {
+	const corpusVersion = options.corpusVersion ?? "0.4.0"
+	// This is a portable manifest namespace rather than a host filesystem default.
+	const canonicalPathPrefix = options.canonicalPathPrefix ?? "/data/"
+	const legacyPathPrefix = options.legacyPathPrefix ?? `${dataRootPath()}/`
+
+	if (!(await pathExists(options.jsonl))) throw new Error(`jsonl not found: ${options.jsonl}`)
+
+	if (!(await pathExists(options.baseManifest))) throw new Error(`base-manifest not found: ${options.baseManifest}`)
+
+	const corpusDir = PathBuilder.from(options.outDir)(corpusDirectoryName(corpusVersion))
+	const trainDir = corpusDir("train")
+	await makeDirectories(trainDir)
+
+	// Bucket canonical rows by source.
+	// Quarantined rows are logged.
+	const buckets = new Map<string, LabeledRow[]>()
+	const quarantine: string[] = []
+	let totalIn = 0
+
+	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(delimitedSource(options.jsonl))) {
+		totalIn++
+		const canon = toCanonicalRow(raw, corpusVersion)
+		const result = alignRow(canon)
+
+		if (result.kind !== "labeled") {
+			quarantine.push(`${canon.source_id}\t${result.row.reason}`)
+
+			continue
+		}
+
+		const bucket = buckets.get(canon.source)
+
+		if (bucket) {
+			bucket.push(result.row)
+		} else {
+			buckets.set(canon.source, [result.row])
+		}
+	}
+
+	report?.(`read ${totalIn} rows; ${quarantine.length} quarantined; ${buckets.size} script buckets`)
+
+	const newFiles: ParquetFileDescriptor[] = []
+	const sortedKeys = [...buckets.keys()].toSorted()
+
+	for (const source of sortedKeys) {
+		const rows = buckets.get(source)!
+		const slug = source.startsWith("deepseek-translit-") ? source.slice("deepseek-translit-".length) : source
+		// A string, because the manifest records it.
+		const outPath = trainDir(`part-translit-${slug}.parquet`).toString()
+		const descriptor = await writeOneFile(rows, outPath, source)
+		newFiles.push(descriptor)
+		report?.(`  ${source}: ${descriptor.rows} rows → ${outPath} (${descriptor.bytes} bytes)`)
+	}
+
+	if (quarantine.length) {
+		const qPath = corpusDir("quarantine-transliteration.tsv")
+		await writeLocalTextFile(quarantine, qPath)
+		report?.(`quarantine log → ${qPath} (${quarantine.length} rows)`)
+	}
+
+	// Compose final manifest: rewrite the base's file paths from the data root → /data/...
+	// And append the new translit files.
+	// The kryptonite file already lives in the base manifest (it was written there by Thread B).
+	const base = await readLocalJSONFile<ParquetManifest>(options.baseManifest)
+
+	const rewrittenBase = base.slices.map((file) => ({
+		...file,
+		path: canonicalizeFilePath(file.path, legacyPathPrefix, canonicalPathPrefix),
+	}))
+
+	const newTrainRows = newFiles.reduce((sum, file) => sum + file.rows, 0)
+
+	const combined: ParquetManifest = {
+		corpus_version: corpusVersion,
+		schema: PARQUET_COLUMNS,
+		rows_per_slice: base.rows_per_slice,
+		row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
+		slices: [...rewrittenBase, ...newFiles],
+		counts: {
+			train: base.counts.train + newTrainRows,
+			val: base.counts.val,
+			test: base.counts.test,
+		},
+		total_rows: base.total_rows + newTrainRows,
+	}
+
+	const combinedPath = corpusDir("MANIFEST.json")
+	await writeLocalJSONFile(combined, combinedPath)
+	report?.(`wrote combined manifest → ${combinedPath}`)
+	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newTrainRows})`)
+	report?.(`  files=${combined.slices.length} (base=${base.slices.length}, added=${newFiles.length})`)
+	report?.(`  compression=${PARQUET_COMPRESSION}`)
+	const pathFix = rewrittenBase.filter((s, i) => s.path !== base.slices[i]!.path).length
+
+	if (pathFix > 0) {
+		report?.(`  path-canonicalized base files: ${pathFix} (legacy '${legacyPathPrefix}' → '${canonicalPathPrefix}')`)
+	}
+}

package/lib/tools/overture-subvenue.ts

@@ -3,89 +3,30 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Overture Places source for the sub-venue lexicon (#35 wave 2) — read the sub-venue-bearing
- *   category slices of `poi.db` (spatial layer #1, `$MAILWOMAN_DATA_ROOT/poi/poi.db`) and yield
- *   {@link SubVenueHarvestRow}s the lexicon builder consumes exactly like an OSM extract.
+ * Read selected Overture Places categories from `poi.db` and return candidate sub-venue names with
+ * country and category provenance.
  *
- *   ── WHY OVERTURE AT ALL, when the OSM extractor already exists ────────────────────────────────────
- *   Wave 1 measured `concourse` at 21 real Overture rows against 4 in the whole Great Britain OSM
- *   extract, 3 of which are a street called CONCOURSE WAY. Overture's `airport_terminal` category is
- *   curated venue-interior naming; OSM's `aeroway=terminal` is a building footprint that usually
- *   carries the AIRPORT's name. The two sources fail differently, so both are read.
- *
- *   ── THE CATEGORY SET IS MEASURED, NOT GUESSED ────────────────────────────────────────────────────
- *   A full scan of all 13,681,698 rows (2026-08-05, poi.db vintage 2026-05-20.0) counted, per
- *   category, how many named rows carry a designator token. The ranking is NOT what a category name
- *   predicts — `gas_station` leads the whole table with 12,996 hits, every one of them the token
- *   `station` inside "Holiday Station" / "Chevron Station Seward", and `shoe_store` contributes 708
- *   hits of `wing` because Red Wing sells boots. {@link OVERTURE_SUBVENUE_CATEGORIES} is the four
- *   categories whose hits survived reading the distribution; the rejects are listed below it so nobody
- *   re-proposes them.
- *
- *   ── poi.db IS FOUR COUNTRIES ─────────────────────────────────────────────────────────────────────
- *   Measured the same day: US 11,521,612 / CA 794,418 / FR 721,352 / MX 644,316, and nothing else. The
- *   shipped layer is not a world gazetteer, so Overture can attest en-US, en-CA, fr-FR and es-MX
- *   surfaces and NOTHING ELSE. Every non-Latin designator the corpus task asks for — `ターミナル`,
- *   `Halle`, `Flügel` — has to come from the OSM leg. Do not read a zero count here as evidence of
- *   absence in the world; it is evidence of absence in four countries.
- *
- *   ── The row shape fits; the PROVENANCE STAMP did not ─────────────────────────────────────────────
- *   Wave 1's `OSMSubVenueRow` was written to accept a non-OSM row, and it does: an Overture row is
- *   `{ designatorID, name }` with no `ref` and no `localizedNames`. What did NOT fit is
- *   `extractAttestedPhrases`, which hardcoded `osm:name` as the surface's `source`. Feeding Overture
- *   rows through it unchanged would have labelled every Overture surface as OSM-attested — a
- *   provenance lie, and under ODbL a consequential one. Hence the `source`/`region` options on that
- *   function and the source-neutral {@link SubVenueHarvestRow} name.
+ * `poi.db` covers four countries — US, CA, FR and MX — so a zero count here is absence in four countries rather
+ * than evidence of absence in the world. Every non-Latin designator comes from the OSM leg.
  */
 
 import { DatabaseClient } from "@mailwoman/sqlite/client"
+import type { PathBuilderLike } from "path-ts"
 
-import type { SubVenueHarvestRow } from "#tools/sub/venue/lexicon"
+import type { SubVenueHarvestRow } from "#subvenue/harvest"
 
 /**
- * Overture Places category → the designator its rows attest.
- *
- * Each entry carries the 2026-08-05 full-scan measurement in a comment: `rows` is the category's total, `hits` is how
- * many of its named rows contain any designator token, and the token list is the top of that distribution.
- *
- * REJECTED, with the number that rejected them — do not re-add without a fresh census:
- *
- * - `gas_station` (12,996 hits) — all `station` inside a brand name (Holiday Station, Chevron Station Seward).
- * - `fire_station` (10,377) — same, plus 330 `hall` from fire halls, which are venues rather than sub-venues.
- * - `town_hall` (5,528) — `City Hall` is a whole building, the `venue` tier, not interior structure.
- * - `building_supply_store` (4,030) — `building` inside "Allied Building Products".
- * - `jehovahs_witness_place_of_worship` (2,334) — every hit is "Kingdom Hall of Jehovah's Witnesses".
- * - `shoe_store` (909, of which 708 `wing`) — Red Wing. This one is a CONFOUND BOARD entry, not a source.
- * - `college_university` (3,697) — 2,082 `campus`, but the row names the whole institution; `campus_building` is the
- *   interior slice and is kept instead.
- * - `airport` (6,000 rows, 4,302 hits) — 4,071 of them are the token `airport` in the aerodrome's own name. Venue tier,
- *   already covered by OurAirports, and it drowns the interior signal.
- * - `transport_interchange` (1 row), `rail_facility_or_service` (81 rows, 2 hits) — too small to matter.
- * - `public_transit_facility_or_service` (2,274 rows, 185 hits) — 161 are `station` naming the station itself.
+ * Overture categories selected for sub-venue designator evidence.
  */
 export const OVERTURE_SUBVENUE_CATEGORIES: Readonly<Record<string, string>> = {
-	// 1,022 rows / 669 hits — terminal 266, concourse 19, gate 13, hall 4. The densest interior naming
-	// in the layer and the reason this reader exists.
 	airport_terminal: "terminal",
-	// 7,366 rows / 3,382 hits — hall 2,134, building 1,019, campus 209. US academic halls and numbered
-	// campus buildings: "UAA Cuddy Hall", "UAA Science Building". The row is a building ON a campus, so
-	// `campus` is its CONTEXT designator; the phrase found inside the name decides the record it
-	// attests, which for these rows is mostly `hall`.
 	campus_building: "campus",
-	// 443 rows / 301 hits — pier 282, terminal 17.
 	pier: "pier",
-	// 388 rows / 66 hits — airport 36, terminal 20, concourse 5, gate 3. Small, but the hits are almost
-	// all genuine ("Delta Sky Club Concourse B").
 	airport_lounge: "terminal",
 }
 
 /**
- * The `poi` and `poi_category_codes` columns this reader touches, declared LOCALLY.
- *
- * `@mailwoman/resolver-wof-sqlite` owns the full `POIDatabase` interface, and `@mailwoman/corpus` does not depend on it
- * — the same dependency-direction call `sub-venue-lexicon.ts` makes for `@mailwoman/osm`'s row type. This is a
- * read-only projection of four columns; `overture-subvenue.test.ts` builds a fixture with exactly this DDL, so a column
- * rename upstream fails a test here rather than throwing at runtime against a 3.9 GB database nobody has in CI.
+ * Read-only subset of POI database tables used by this extractor.
  */
 interface POIReadDatabase {
 	poi: {
@@ -104,14 +45,9 @@ interface POIReadDatabase {
 }
 
 /**
- * The layer's `source_vintage` — the Overture release the rows came from (`2026-05-20.0`), for the lexicon's
- * `sources[]`.
- *
- * Read off the database rather than passed in, because a vintage a caller types is a vintage that goes stale silently.
- * The layer-contract tables are part of every layer database by construction — see `docs/engineering/reference/
- * layer-contract.mdx` — so there is no version of poi.db where this is absent.
+ * Read the Overture release recorded in the POI layer manifest.
  */
-export async function readOvertureLayerVintage(databasePath: string): Promise<string> {
+export async function readOvertureLayerVintage(databasePath: PathBuilderLike): Promise<string> {
 	using kdb = new DatabaseClient<POIReadDatabase>(databasePath, { readOnly: true })
 
 	const row = await kdb
@@ -125,44 +61,35 @@ export async function readOvertureLayerVintage(databasePath: string): Promise<st
 
 export interface ReadOvertureSubVenuesOptions {
 	/**
-	 * Path to `poi.db`. Typically `dataRootPath("poi", "poi.db")`.
+	 * Path to `poi.db`.
 	 */
-	databasePath: string
+	databasePath: PathBuilderLike
 	/**
-	 * Category → designator map. Defaults to {@link OVERTURE_SUBVENUE_CATEGORIES}.
+	 * Category-to-designator mapping.
 	 */
 	categories?: Readonly<Record<string, string>>
 	/**
-	 * Restrict to these ISO 3166-1 alpha-2 countries. Omit for all four the layer carries.
+	 * Optional ISO alpha-2 country filter.
 	 */
 	countries?: readonly string[]
 }
 
 /**
- * One Overture row plus the country it came from — {@link SubVenueHarvestRow} widened by the field the layer has and an
- * OSM extract does not (a Geofabrik extract's country is a property of the invocation, so OSM rows carry `country:
- * ""`).
+ * Overture sub-venue row with its country and category provenance.
  */
 export interface OvertureSubVenueRow extends SubVenueHarvestRow {
 	/**
-	 * ISO 3166-1 alpha-2, straight off the Overture partition.
+	 * ISO alpha-2 code from the Overture partition.
 	 */
 	country: string
 	/**
-	 * The Overture category the row was read from, so a surface's provenance survives past the designator.
+	 * Overture category used to select the row.
 	 */
 	category: string
 }
 
 /**
- * Read the sub-venue-bearing category slices of `poi.db`.
- *
- * Cold path, already async, no interface constraint — so Kysely, per the repo's inline-SQL rule.
- *
- * `category_id` is the SECOND component of poi.db's clustered `(h3_cell, category_id, …)` primary key, which reads like
- * a full 13.7M-row scan and is not one: SQLite skip-scans the leading `h3_cell` and the whole four-category read
- * returned 9,219 rows in **1.4 s** on the shipped layer (2026-08-05). A JS-side scan of every row to survey the same
- * question took 52 s, which is the number to remember if you are tempted to filter in JS instead.
+ * Read POI rows with names in selected sub-venue categories, optionally filtered by country.
  */
 export async function readOvertureSubVenues(options: ReadOvertureSubVenuesOptions): Promise<OvertureSubVenueRow[]> {
 	const categories = options.categories ?? OVERTURE_SUBVENUE_CATEGORIES