@mailwoman/corpus 9.1.0 → 9.3.0

package/lib/tools/slice-kryptonite.ts

@@ -0,0 +1,137 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build a parquet slice from the DeepSeek-generated kryptonite JSONL and emit the corpus-v0.4.0
+ *   MANIFEST. corpus-v0.4.0 is a pure adapter-addition revision: it points at every slice from
+ *   v0.3.0 plus the new kryptonite slice(s). No v0.3.0 bytes are touched or re-shuffled.
+ *
+ *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for the why; that doc also pins the
+ *   DeepSeek model version + prompt versions used to produce the JSONL.
+ *
+ *   Invoke via `mailwoman corpus slice kryptonite \
+ *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+
+import { pathExists, readLocalJSONFile } from "@mailwoman/core/fs/readers"
+import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers"
+import { join } from "path-ts"
+import { JSONSpliterator } from "spliterator"
+
+import type { CanonicalRow, LabeledRow } from "#types"
+import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, SLICE_COMPRESSION, writeSlices } from "#utils"
+import type { SliceManifest } from "#utils"
+
+export interface SliceKryptoniteOptions {
+	jsonl: string
+	baseManifest: string
+	outDir: string
+	/**
+	 * Default `"0.4.0"`.
+	 */
+	corpusVersion?: string
+	/**
+	 * Default `"deepseek-kryptonite"`.
+	 */
+	source?: string
+}
+
+async function* canonicalRows(jsonl: string, corpusVersion: string): AsyncIterable<CanonicalRow> {
+	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(jsonl)) {
+		// Strip sidecar underscore-prefixed fields the generator left behind for debugging.
+		const components = raw["components"] as Record<string, string>
+
+		yield {
+			raw: raw["raw"] as string,
+			components,
+			country: (raw["country"] as string) ?? "US",
+			locale: (raw["locale"] as string) ?? undefined,
+			source: (raw["source"] as string) ?? "deepseek-kryptonite",
+			source_id: raw["source_id"] as string,
+			corpus_version: corpusVersion,
+			license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
+			synth: raw["synth"] as CanonicalRow["synth"],
+		}
+	}
+}
+
+async function* labeledRows(jsonl: string, corpusVersion: string, quarantineLog: string[]): AsyncIterable<LabeledRow> {
+	for await (const row of canonicalRows(jsonl, corpusVersion)) {
+		const result = alignRow(row)
+
+		if (result.kind === "labeled") {
+			yield result.row
+		} else {
+			quarantineLog.push(`${row.source_id}\t${result.row.reason}`)
+		}
+	}
+}
+
+export async function buildKryptoniteSlice(
+	options: SliceKryptoniteOptions,
+	report?: (line: string) => void
+): Promise<void> {
+	const corpusVersion = options.corpusVersion ?? "0.4.0"
+	const source = options.source ?? "deepseek-kryptonite"
+
+	if (!(await pathExists(options.jsonl))) throw new Error(`jsonl not found: ${options.jsonl}`)
+
+	if (!(await pathExists(options.baseManifest))) throw new Error(`base-manifest not found: ${options.baseManifest}`)
+
+	await makeDirectories(options.outDir)
+
+	const quarantine: string[] = []
+
+	const newManifest = await writeSlices(
+		{ train: labeledRows(options.jsonl, corpusVersion, quarantine) },
+		{ outputDir: options.outDir, corpusVersion }
+	)
+
+	report?.(
+		`wrote ${newManifest.total_rows} rows into ${newManifest.slices.length} slice(s); ` +
+			`quarantined ${quarantine.length}`
+	)
+
+	if (quarantine.length) {
+		const qPath = join(options.outDir, `corpus-v${corpusVersion}`, "quarantine-kryptonite.tsv")
+		await writeLocalTextFile(quarantine.join("\n") + "\n", qPath)
+		report?.(`quarantine log → ${qPath}`)
+	}
+
+	// Stamp the new slice's source field for audit.ts (which prefers slice.source over
+	// first_source_id-prefix inference). Without this, deepseek-kryptonite IDs would have
+	// to match a prefix in KNOWN_SOURCE_PREFIXES — we add it there too as a belt-and-braces.
+	for (const sh of newManifest.slices) {
+		sh.source = source
+	}
+
+	// Compose the final corpus-v0.4.0 manifest: every slice from base + the new slice(s).
+	const base = await readLocalJSONFile<SliceManifest>(options.baseManifest)
+
+	const combined: SliceManifest = {
+		corpus_version: corpusVersion,
+		schema: PARQUET_COLUMNS,
+		rows_per_slice: base.rows_per_slice,
+		row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
+		slices: [...base.slices, ...newManifest.slices],
+		counts: {
+			train: base.counts.train + (newManifest.counts.train ?? 0),
+			val: base.counts.val,
+			test: base.counts.test,
+		},
+		total_rows: base.total_rows + newManifest.total_rows,
+	}
+
+	// Stamp source on the legacy v0.3.0 slices too, so audit's slice.source path is the
+	// authoritative one. v0.3.0 slices mix sources; we use the first_source_id-prefix
+	// inference for them (audit.ts will re-derive on its own when slice.source is absent).
+	const combinedPath = join(options.outDir, `corpus-v${corpusVersion}`, "MANIFEST.json")
+	await writeLocalJSONFile(combined, combinedPath)
+	report?.(`wrote combined manifest → ${combinedPath}`)
+	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newManifest.total_rows})`)
+	report?.(`  slices=${combined.slices.length} (base=${base.slices.length}, added=${newManifest.slices.length})`)
+	report?.(`  compression=${SLICE_COMPRESSION}`)
+}

package/lib/tools/slice-translit.ts

@@ -0,0 +1,218 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build per-script parquet slices from the DeepSeek-generated transliteration JSONL and emit the
+ *   corpus-v0.4.0 MANIFEST that combines them with the existing kryptonite + v0.3.0 slices.
+ *
+ *   Sibling to `slice-kryptonite.ts`. The two modules share the same composition pattern: take a
+ *   base MANIFEST, append new slices, write a combined MANIFEST. Differences specific to
+ *   transliteration:
+ *
+ *   - One JSONL contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
+ *       `source` and write one slice per script so `audit.ts` can attribute each slice to its
+ *       synthetic source without relying on filename-prefix inference.
+ *   - Each slice is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
+ *       `part-0000.parquet`, which v0.4.0's first builder already produced).
+ *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 slice paths are
+ *       rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined MANIFEST so
+ *       all paths share one container-friendly form.
+ *
+ *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for prompts, model, and the
+ *   reproducibility contract.
+ *
+ *   Invoke via `mailwoman corpus slice translit \
+ *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+
+import { mailwomanDataRoot } from "@mailwoman/core/data-root"
+import { pathExists, readLocalJSONFile, tryStat } from "@mailwoman/core/fs/readers"
+import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { join } from "path-ts"
+import { JSONSpliterator } from "spliterator"
+
+import type { CanonicalRow, LabeledRow } from "#types"
+import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, rowToParquet, SLICE_COMPRESSION, writeParquetRows } from "#utils"
+import type { SliceDescriptor, SliceManifest } from "#utils"
+
+export interface SliceTranslitOptions {
+	jsonl: string
+	baseManifest: string
+	outDir: string
+	/**
+	 * Default `"0.4.0"`.
+	 */
+	corpusVersion?: string
+	/**
+	 * Default `"/data/"`.
+	 */
+	canonicalPathPrefix?: string
+	/**
+	 * Prefix the base manifest's slice paths currently carry, to be rewritten to
+	 * {@link SliceTranslitOptions.canonicalPathPrefix}. Defaults to `mailwomanDataRoot()` with a trailing slash — the
+	 * root that WROTE those paths. Pass it explicitly when translating a manifest generated under a different
+	 * `$MAILWOMAN_DATA_ROOT` than the one you are running with.
+	 */
+	legacyPathPrefix?: string
+}
+
+function toCanonicalRow(raw: Record<string, unknown>, corpusVersion: string): CanonicalRow {
+	return {
+		raw: raw["raw"] as string,
+		components: raw["components"] as Record<string, string>,
+		country: (raw["country"] as string) ?? "US",
+		locale: (raw["locale"] as string) ?? undefined,
+		source: raw["source"] as string,
+		source_id: raw["source_id"] as string,
+		corpus_version: corpusVersion,
+		license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
+		synth: raw["synth"] as CanonicalRow["synth"],
+	}
+}
+
+/**
+ * Write one slice for a single source slug. Returns the populated SliceDescriptor + a list of quarantine reasons for
+ * rows that failed alignment.
+ */
+async function writeOneSlice(rows: readonly LabeledRow[], outPath: string, source: string): Promise<SliceDescriptor> {
+	let firstSourceID = ""
+	let lastSourceID = ""
+
+	for (const row of rows) {
+		if (firstSourceID === "") {
+			firstSourceID = row.source_id
+		}
+
+		lastSourceID = row.source_id
+	}
+
+	await writeParquetRows(rows.map(rowToParquet), outPath)
+
+	const fileStat = await tryStat(outPath)
+	const sha256 = await sha256File(outPath)
+
+	return {
+		split: "train",
+		path: outPath,
+		format: "parquet",
+		compression: SLICE_COMPRESSION,
+		rows: rows.length,
+		bytes: fileStat?.size ?? 0,
+		sha256,
+		first_source_id: firstSourceID,
+		last_source_id: lastSourceID,
+		// Stamp source so audit.ts attributes the slice without falling back to filename-prefix inference.
+		source,
+	}
+}
+
+function canonicalizeSlicePath(path: string, legacyPrefix: string, canonicalPrefix: string): string {
+	if (path.startsWith(legacyPrefix)) return canonicalPrefix + path.slice(legacyPrefix.length)
+
+	return path
+}
+
+export async function buildTranslitSlice(
+	options: SliceTranslitOptions,
+	report?: (line: string) => void
+): Promise<void> {
+	const corpusVersion = options.corpusVersion ?? "0.4.0"
+	// This is a portable manifest namespace, not a host filesystem default.
+	const canonicalPathPrefix = options.canonicalPathPrefix ?? "/data/"
+	const legacyPathPrefix = options.legacyPathPrefix ?? `${mailwomanDataRoot()}/`
+
+	if (!(await pathExists(options.jsonl))) throw new Error(`jsonl not found: ${options.jsonl}`)
+
+	if (!(await pathExists(options.baseManifest))) throw new Error(`base-manifest not found: ${options.baseManifest}`)
+
+	const corpusDir = join(options.outDir, `corpus-v${corpusVersion}`)
+	const trainDir = join(corpusDir, "train")
+	await makeDirectories(trainDir)
+
+	// Bucket canonical rows by source. Quarantined rows are logged.
+	const buckets = new Map<string, LabeledRow[]>()
+	const quarantine: string[] = []
+	let totalIn = 0
+
+	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(options.jsonl)) {
+		totalIn++
+		const canon = toCanonicalRow(raw, corpusVersion)
+		const result = alignRow(canon)
+
+		if (result.kind !== "labeled") {
+			quarantine.push(`${canon.source_id}\t${result.row.reason}`)
+
+			continue
+		}
+
+		const bucket = buckets.get(canon.source)
+
+		if (bucket) {
+			bucket.push(result.row)
+		} else {
+			buckets.set(canon.source, [result.row])
+		}
+	}
+
+	report?.(`read ${totalIn} rows; ${quarantine.length} quarantined; ${buckets.size} script buckets`)
+
+	const newSlices: SliceDescriptor[] = []
+	const sortedKeys = [...buckets.keys()].toSorted()
+
+	for (const source of sortedKeys) {
+		const rows = buckets.get(source)!
+		const slug = source.startsWith("deepseek-translit-") ? source.slice("deepseek-translit-".length) : source
+		const outPath = join(trainDir, `part-translit-${slug}.parquet`)
+		const descriptor = await writeOneSlice(rows, outPath, source)
+		newSlices.push(descriptor)
+		report?.(`  ${source}: ${descriptor.rows} rows → ${outPath} (${descriptor.bytes} bytes)`)
+	}
+
+	if (quarantine.length) {
+		const qPath = join(corpusDir, "quarantine-transliteration.tsv")
+		await writeLocalTextFile(quarantine.join("\n") + "\n", qPath)
+		report?.(`quarantine log → ${qPath} (${quarantine.length} rows)`)
+	}
+
+	// Compose final MANIFEST: rewrite base.slices paths from /mnt/playpen/... → /data/... and append
+	// the new translit slices. Kryptonite slice already lives in the base manifest (it was written
+	// there by Thread B).
+	const base = await readLocalJSONFile<SliceManifest>(options.baseManifest)
+
+	const rewrittenBase = base.slices.map((sh) => ({
+		...sh,
+		path: canonicalizeSlicePath(sh.path, legacyPathPrefix, canonicalPathPrefix),
+	}))
+
+	const newTrainRows = newSlices.reduce((sum, sh) => sum + sh.rows, 0)
+
+	const combined: SliceManifest = {
+		corpus_version: corpusVersion,
+		schema: PARQUET_COLUMNS,
+		rows_per_slice: base.rows_per_slice,
+		row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
+		slices: [...rewrittenBase, ...newSlices],
+		counts: {
+			train: base.counts.train + newTrainRows,
+			val: base.counts.val,
+			test: base.counts.test,
+		},
+		total_rows: base.total_rows + newTrainRows,
+	}
+
+	const combinedPath = join(corpusDir, "MANIFEST.json")
+	await writeLocalJSONFile(combined, combinedPath)
+	report?.(`wrote combined manifest → ${combinedPath}`)
+	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newTrainRows})`)
+	report?.(`  slices=${combined.slices.length} (base=${base.slices.length}, added=${newSlices.length})`)
+	report?.(`  compression=${SLICE_COMPRESSION}`)
+	const pathFix = rewrittenBase.filter((s, i) => s.path !== base.slices[i]!.path).length
+
+	if (pathFix > 0) {
+		report?.(`  path-canonicalized base slices: ${pathFix} (legacy '${legacyPathPrefix}' → '${canonicalPathPrefix}')`)
+	}
+}

package/lib/tools/sub-venue/harvest.ts

@@ -0,0 +1,234 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file The harvest side of the sub-venue lexicon: what one row of a harvestable source looks like,
+ *   how a file of them is read back, and what a pass over them extracts — attested surfaces plus the
+ *   measured shape of each designator's identifier half.
+ *
+ *   {@link SubVenueHarvestRow} is SOURCE-NEUTRAL, and verified so: an Overture Places row from the
+ *   `airport_terminal` category is `{ designatorID, name }` and fits unchanged. It is declared here
+ *   rather than imported so the builder does not take a dependency on `@mailwoman/osm` (which
+ *   `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
+ *
+ *   Two measurements shape {@link extractAttestedPhrases}, and both are the reason it is safe to run
+ *   over raw OSM at all:
+ *
+ *   **A feature's `name` is usually the VENUE's name, not a sub-venue phrase.** On the Berlin extract
+ *   (2,060 matched features) a `railway=platform` is named `Stendaler Straße` and an `amenity=university`
+ *   is named `Hertie School`; across 250,116 named Great Britain features only 6,003 (2.40%) contain a
+ *   designator token at all. So names are not harvested wholesale — a name contributes only when it
+ *   CONTAINS a phrase already in the surface index, which is what makes `Terminal E (Untere Ebene)`
+ *   evidence and `Otto Lilienthal Flughafen Berlin Tegel` not.
+ *
+ *   **A matched phrase belongs to the record the PHRASE names, not the record the ROW carries.**
+ *   Attributing every hit to `row.designatorID` — the rule that matched the FEATURE — produced `west →
+ *   platform`, `hall → platform` and `biggin → platform` on the GB extract, because a bus stop tagged
+ *   `public_transport=platform` is named "Village Hall" or "West Kensington"; 108 of 133 OSM-derived
+ *   surfaces named a different record than the one they pointed at. Attribution therefore runs through
+ *   the index, and the row's own designator is kept as `context` — exactly the axis a confound board
+ *   needs, since a `hall` seen on a platform is a confound and a `hall` seen on a terminal is evidence.
+ */
+
+import { readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { parseJSONStrict } from "@mailwoman/core/json"
+import { TextSpliterator } from "spliterator"
+
+import { nameContainsSurfaces, type SurfaceIndex } from "#tools/sub-venue/surfaces"
+import type { IdentifierShape, SubVenueSurface } from "#tools/sub-venue/table"
+
+/**
+ * One row of a harvestable source, as read back off JSONL or out of a layer database.
+ *
+ * SOURCE-NEUTRAL by design, and verified so in wave 2: an Overture Places row from the `airport_terminal` category is
+ * `{ designatorID, name }` and fits unchanged. What did NOT fit was the harvest function's hardcoded `osm:name` source
+ * stamp — see `overture-subvenue.ts`'s docstring. Declared locally so the builder does not import `@mailwoman/osm`
+ * (which `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
+ */
+export interface SubVenueHarvestRow {
+	/**
+	 * The designator the SOURCE's rule assigned to the FEATURE. Not necessarily the record a matched phrase names — see
+	 * this file's header on phrase attribution. Carried into {@link SubVenueSurface.context}.
+	 */
+	designatorID: string
+	/**
+	 * `venue` (station, airport, campus) or `sub_venue`, when the source's rule assigns one. The harvest itself does not
+	 * read it; the sub-venue slice recipe fills its venue slot from it.
+	 */
+	tier?: string
+	name?: string | null
+	ref?: string | null
+	localizedNames?: Record<string, string>
+}
+
+/**
+ * Classify an OSM `ref` into an {@link IdentifierShape} class.
+ *
+ * The classes are the ones Berlin's gates actually produced, plus the two aviation forms the corpus task names
+ * (`Terminal 2F` is digit-letter, `Concourse B` is letter). `range` covers both separators OSM uses for a gate serving
+ * more than one stand: `16-18` and `0/1`.
+ */
+export function classifyIdentifier(ref: string): string {
+	const value = ref.trim()
+
+	if (/^[0-9]+$/.test(value)) return "digit"
+
+	if (/^[A-Za-z]$/.test(value)) return "letter"
+
+	if (/^[A-Za-z]+[0-9]+$/.test(value)) return "letter-digit"
+
+	if (/^[0-9]+[A-Za-z]+$/.test(value)) return "digit-letter"
+
+	if (/^[0-9A-Za-z]+\s*[-/]\s*[0-9A-Za-z]+$/.test(value)) return "range"
+
+	return "other"
+}
+
+/**
+ * How many real `ref` values each {@link IdentifierShape} keeps.
+ *
+ * Eight, not "all" and not one. The field exists so a slice author can see what a class actually CONTAINS — GB's
+ * `other` class turned out to be semicolon multi-values (`1;2;3`, `13;14`), which one example would have hidden and
+ * which the class name does not say. Eight fits a terminal line and covers the variety inside every class the GB
+ * extract produced. The COUNT lives in `observations`; this is a sample, not a census.
+ */
+const IDENTIFIER_EXAMPLES_PER_SHAPE = 8
+
+/**
+ * Options for one harvest pass — which source stamp its surfaces carry and which region they were attested in.
+ *
+ * Both default to the OSM/unknown-region values wave 1 hardcoded, so an existing caller is unchanged.
+ */
+export interface HarvestOptions {
+	/**
+	 * Source family: `osm` yields `osm:name` / `osm:name:<lang>`, `overture` yields `overture:name`.
+	 */
+	source?: string
+	/**
+	 * ISO 3166-1 alpha-2 of the extract or partition. `""` when unknown.
+	 */
+	region?: string
+}
+
+/**
+ * Harvest attested phrases and identifier shapes out of a source's rows.
+ *
+ * `index` gates the name harvest — a name contributes only when it CONTAINS a phrase already in the table. That filter
+ * is the whole reason this function is safe to run over raw OSM: see this file's header for the Berlin measurement that
+ * motivated it. The index also decides ATTRIBUTION: a hit is a surface of the record the PHRASE names, and the row's
+ * own designator is recorded as `context`.
+ *
+ * Returns surfaces with real `observations` counts, so the lexicon can rank `terminal` above a phrase attested once.
+ */
+export function extractAttestedPhrases(
+	rows: Iterable<SubVenueHarvestRow>,
+	index: SurfaceIndex,
+	options: HarvestOptions = {}
+): { surfaces: SubVenueSurface[]; identifierShapes: IdentifierShape[] } {
+	const source = options.source ?? "osm"
+	const region = options.region ?? ""
+	/**
+	 * `phrase\0lang\0source` → { count, context }.
+	 */
+	const surfaceCounts = new Map<string, { count: number; context: Map<string, number> }>()
+	/**
+	 * `designatorID\0shape` → { count, examples }.
+	 */
+	const shapes = new Map<string, { count: number; examples: Set<string> }>()
+
+	const note = (phrase: string, lang: string, sourceTag: string, context: string): void => {
+		const key = `${phrase}\0${lang}\0${sourceTag}`
+		const entry = surfaceCounts.get(key) ?? { count: 0, context: new Map<string, number>() }
+
+		entry.count++
+		entry.context.set(context, (entry.context.get(context) ?? 0) + 1)
+		surfaceCounts.set(key, entry)
+	}
+
+	for (const row of rows) {
+		if (row.name) {
+			for (const hit of nameContainsSurfaces(row.name, index)) {
+				// `und` — the default `name` tag carries no language. Overture's `name` is the same: a
+				// primary name in whatever language the place uses, untagged.
+				note(hit, "und", `${source}:name`, row.designatorID)
+			}
+		}
+
+		for (const [lang, localized] of Object.entries(row.localizedNames ?? {})) {
+			for (const hit of nameContainsSurfaces(localized, index)) {
+				note(hit, lang, `${source}:name:${lang}`, row.designatorID)
+			}
+		}
+
+		if (row.ref) {
+			const shape = classifyIdentifier(row.ref)
+			const key = `${row.designatorID}\0${shape}`
+			const entry = shapes.get(key) ?? { count: 0, examples: new Set<string>() }
+
+			entry.count++
+
+			if (entry.examples.size < IDENTIFIER_EXAMPLES_PER_SHAPE) {
+				entry.examples.add(row.ref.trim())
+			}
+
+			shapes.set(key, entry)
+		}
+	}
+
+	const surfaces: SubVenueSurface[] = [...surfaceCounts].map(([key, entry]) => {
+		const [phrase, lang, sourceTag] = key.split("\0") as [string, string, string]
+		const record = index.get(phrase)!
+
+		return {
+			phrase,
+			recordID: record.recordID,
+			recordKind: record.recordKind,
+			lang,
+			region,
+			source: sourceTag,
+			curated: false,
+			observations: entry.count,
+			context: Object.fromEntries([...entry.context].toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))),
+		}
+	})
+
+	const identifierShapes: IdentifierShape[] = [...shapes].map(([key, entry]) => {
+		const [designatorID, shape] = key.split("\0") as [string, string]
+
+		return {
+			designatorID,
+			region,
+			shape,
+			observations: entry.count,
+			examples: [...entry.examples].toSorted((a, b) => a.localeCompare(b)),
+		}
+	})
+
+	return { surfaces, identifierShapes }
+}
+
+/**
+ * Read a JSONL file of {@link SubVenueHarvestRow}s. Blank lines and unparseable rows are skipped rather than fatal — an
+ * extract is a build output, and one malformed line should not cost the whole lexicon.
+ */
+export async function readSubVenueJSONL(path: string): Promise<SubVenueHarvestRow[]> {
+	const out: SubVenueHarvestRow[] = []
+
+	// `TextSpliterator` rather than `split("\n")` — a whole-country extract runs to 250,000 lines
+	// (52 MB for Great Britain), and materializing every segment before reading the first is exactly
+	// what the repo lint rule exists to prevent.
+	for (const line of TextSpliterator.from(await readLocalTextFile(path))) {
+		const trimmed = line.trim()
+
+		if (!trimmed) continue
+
+		try {
+			out.push(parseJSONStrict<SubVenueHarvestRow>(trimmed))
+		} catch {
+			continue
+		}
+	}
+
+	return out
+}