@mailwoman/corpus 9.2.0 → 9.3.0

package/src/tools/shard-kryptonite.ts

@@ -1,139 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Build a parquet shard from the DeepSeek-generated kryptonite JSONL and emit the corpus-v0.4.0
- *   MANIFEST. corpus-v0.4.0 is a pure adapter-addition revision: it points at every shard from
- *   v0.3.0 plus the new kryptonite shard(s). No v0.3.0 bytes are touched or re-shuffled.
- *
- *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for the why; that doc also pins the
- *   DeepSeek model version + prompt versions used to produce the JSONL.
- *
- *   Invoke via `mailwoman corpus shard kryptonite \
- *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
- *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/MANIFEST.json \
- *   --out-dir /data/corpus/versioned/v0.4.0`
- */
-
-import { existsSync, readFileSync, writeFileSync } from "node:fs"
-import { mkdir } from "node:fs/promises"
-import { join } from "node:path"
-
-import { parseJSONStrict } from "@mailwoman/core/objects"
-import { JSONSpliterator } from "spliterator"
-
-import type { CanonicalRow, LabeledRow } from "#types"
-import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, SHARD_COMPRESSION, writeShards } from "#utils"
-import type { ShardManifest } from "#utils"
-
-export interface ShardKryptoniteOptions {
-	jsonl: string
-	baseManifest: string
-	outDir: string
-	/**
-	 * Default `"0.4.0"`.
-	 */
-	corpusVersion?: string
-	/**
-	 * Default `"deepseek-kryptonite"`.
-	 */
-	source?: string
-}
-
-async function* canonicalRows(jsonl: string, corpusVersion: string): AsyncIterable<CanonicalRow> {
-	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(jsonl)) {
-		// Strip sidecar underscore-prefixed fields the generator left behind for debugging.
-		const components = raw["components"] as Record<string, string>
-
-		yield {
-			raw: raw["raw"] as string,
-			components,
-			country: (raw["country"] as string) ?? "US",
-			locale: (raw["locale"] as string) ?? undefined,
-			source: (raw["source"] as string) ?? "deepseek-kryptonite",
-			source_id: raw["source_id"] as string,
-			corpus_version: corpusVersion,
-			license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
-			synth: raw["synth"] as CanonicalRow["synth"],
-		}
-	}
-}
-
-async function* labeledRows(jsonl: string, corpusVersion: string, quarantineLog: string[]): AsyncIterable<LabeledRow> {
-	for await (const row of canonicalRows(jsonl, corpusVersion)) {
-		const result = alignRow(row)
-
-		if (result.kind === "labeled") {
-			yield result.row
-		} else {
-			quarantineLog.push(`${row.source_id}\t${result.row.reason}`)
-		}
-	}
-}
-
-export async function buildKryptoniteShard(
-	options: ShardKryptoniteOptions,
-	report?: (line: string) => void
-): Promise<void> {
-	const corpusVersion = options.corpusVersion ?? "0.4.0"
-	const source = options.source ?? "deepseek-kryptonite"
-
-	if (!existsSync(options.jsonl)) throw new Error(`jsonl not found: ${options.jsonl}`)
-
-	if (!existsSync(options.baseManifest)) throw new Error(`base-manifest not found: ${options.baseManifest}`)
-
-	await mkdir(options.outDir, { recursive: true })
-
-	const quarantine: string[] = []
-
-	const newManifest = await writeShards(
-		{ train: labeledRows(options.jsonl, corpusVersion, quarantine) },
-		{ outputDir: options.outDir, corpusVersion }
-	)
-
-	report?.(
-		`wrote ${newManifest.total_rows} rows into ${newManifest.shards.length} shard(s); ` +
-			`quarantined ${quarantine.length}`
-	)
-
-	if (quarantine.length) {
-		const qPath = join(options.outDir, `corpus-v${corpusVersion}`, "quarantine-kryptonite.tsv")
-		writeFileSync(qPath, quarantine.join("\n") + "\n", "utf8")
-		report?.(`quarantine log → ${qPath}`)
-	}
-
-	// Stamp the new shard's source field for audit.ts (which prefers shard.source over
-	// first_source_id-prefix inference). Without this, deepseek-kryptonite IDs would have
-	// to match a prefix in KNOWN_SOURCE_PREFIXES — we add it there too as a belt-and-braces.
-	for (const sh of newManifest.shards) {
-		sh.source = source
-	}
-
-	// Compose the final corpus-v0.4.0 manifest: every shard from base + the new shard(s).
-	const base = parseJSONStrict<ShardManifest>(readFileSync(options.baseManifest, "utf8"))
-
-	const combined: ShardManifest = {
-		corpus_version: corpusVersion,
-		schema: PARQUET_COLUMNS,
-		rows_per_shard: base.rows_per_shard,
-		row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
-		shards: [...base.shards, ...newManifest.shards],
-		counts: {
-			train: base.counts.train + (newManifest.counts.train ?? 0),
-			val: base.counts.val,
-			test: base.counts.test,
-		},
-		total_rows: base.total_rows + newManifest.total_rows,
-	}
-
-	// Stamp source on the legacy v0.3.0 shards too, so audit's shard.source path is the
-	// authoritative one. v0.3.0 shards mix sources; we use the first_source_id-prefix
-	// inference for them (audit.ts will re-derive on its own when shard.source is absent).
-	const combinedPath = join(options.outDir, `corpus-v${corpusVersion}`, "MANIFEST.json")
-	writeFileSync(combinedPath, JSON.stringify(combined, null, 2) + "\n", "utf8")
-	report?.(`wrote combined manifest → ${combinedPath}`)
-	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newManifest.total_rows})`)
-	report?.(`  shards=${combined.shards.length} (base=${base.shards.length}, added=${newManifest.shards.length})`)
-	report?.(`  compression=${SHARD_COMPRESSION}`)
-}

package/src/tools/shard-translit.ts

@@ -1,245 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Build per-script parquet shards from the DeepSeek-generated transliteration JSONL and emit the
- *   corpus-v0.4.0 MANIFEST that combines them with the existing kryptonite + v0.3.0 shards.
- *
- *   Sibling to `shard-kryptonite.ts`. The two modules share the same composition pattern: take a
- *   base MANIFEST, append new shards, write a combined MANIFEST. Differences specific to
- *   transliteration:
- *
- *   - One JSONL contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
- *       `source` and write one shard per script so `audit.ts` can attribute each shard to its
- *       synthetic source without relying on filename-prefix inference.
- *   - Each shard is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
- *       `part-0000.parquet`, which v0.4.0's first builder already produced).
- *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 shard paths are
- *       rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined MANIFEST so
- *       all paths share one container-friendly form.
- *
- *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for prompts, model, and the
- *   reproducibility contract.
- *
- *   Invoke via `mailwoman corpus shard translit \
- *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
- *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/MANIFEST.json \
- *   --out-dir /data/corpus/versioned/v0.4.0`
- */
-
-import { existsSync, readFileSync, writeFileSync } from "node:fs"
-import { mkdir, stat } from "node:fs/promises"
-import { join } from "node:path"
-
-import { parseJSONStrict } from "@mailwoman/core/objects"
-import { mailwomanDataRoot, sha256File } from "@mailwoman/core/utils"
-import { JSONSpliterator } from "spliterator"
-
-import type { CanonicalRow, LabeledRow } from "#types"
-import {
-	alignRow,
-	appendShape,
-	LABELED_ROW_SCHEMA,
-	PARQUET_COLUMNS,
-	ROW_GROUP_SIZE,
-	rowToParquet,
-	SHARD_COMPRESSION,
-} from "#utils"
-import type { ParquetRow, ShardDescriptor, ShardManifest } from "#utils"
-
-import { ParquetWriter } from "../parquet-wrapper/index.ts"
-
-export interface ShardTranslitOptions {
-	jsonl: string
-	baseManifest: string
-	outDir: string
-	/**
-	 * Default `"0.4.0"`.
-	 */
-	corpusVersion?: string
-	/**
-	 * Default `"/data/"`.
-	 */
-	canonicalPathPrefix?: string
-	/**
-	 * Prefix the base manifest's shard paths currently carry, to be rewritten to
-	 * {@link ShardTranslitOptions.canonicalPathPrefix}. Defaults to `mailwomanDataRoot()` with a trailing slash — the
-	 * root that WROTE those paths. Pass it explicitly when translating a manifest generated under a different
-	 * `$MAILWOMAN_DATA_ROOT` than the one you are running with.
-	 */
-	legacyPathPrefix?: string
-}
-
-function toCanonicalRow(raw: Record<string, unknown>, corpusVersion: string): CanonicalRow {
-	return {
-		raw: raw["raw"] as string,
-		components: raw["components"] as Record<string, string>,
-		country: (raw["country"] as string) ?? "US",
-		locale: (raw["locale"] as string) ?? undefined,
-		source: raw["source"] as string,
-		source_id: raw["source_id"] as string,
-		corpus_version: corpusVersion,
-		license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
-		synth: raw["synth"] as CanonicalRow["synth"],
-	}
-}
-
-/**
- * Write one shard for a single source slug. Returns the populated ShardDescriptor + a list of quarantine reasons for
- * rows that failed alignment.
- */
-async function writeOneShard(
-	rows: readonly LabeledRow[],
-	outPath: string,
-	source: string,
-	corpusVersion: string
-): Promise<ShardDescriptor> {
-	const writer = await ParquetWriter.openFile<ParquetRow>(LABELED_ROW_SCHEMA, outPath, {
-		rowGroupSize: ROW_GROUP_SIZE,
-	})
-
-	writer.setMetadata("mailwoman.corpus_version", corpusVersion)
-	writer.setMetadata("mailwoman.split", "train")
-	writer.setMetadata("mailwoman.shard_source", source)
-
-	let firstSourceID = ""
-	let lastSourceID = ""
-
-	for (const row of rows) {
-		const pq = rowToParquet(row)
-		await writer.appendRow(appendShape(pq))
-
-		if (firstSourceID === "") {
-			firstSourceID = row.source_id
-		}
-
-		lastSourceID = row.source_id
-	}
-
-	await writer.close()
-
-	const fileStat = await stat(outPath)
-	const sha256 = await sha256File(outPath)
-
-	return {
-		split: "train",
-		path: outPath,
-		format: "parquet",
-		compression: SHARD_COMPRESSION,
-		rows: rows.length,
-		bytes: fileStat.size,
-		sha256,
-		first_source_id: firstSourceID,
-		last_source_id: lastSourceID,
-		// Stamp source so audit.ts attributes the shard without falling back to filename-prefix inference.
-		source,
-	}
-}
-
-function canonicalizeShardPath(path: string, legacyPrefix: string, canonicalPrefix: string): string {
-	if (path.startsWith(legacyPrefix)) return canonicalPrefix + path.slice(legacyPrefix.length)
-
-	return path
-}
-
-export async function buildTranslitShard(
-	options: ShardTranslitOptions,
-	report?: (line: string) => void
-): Promise<void> {
-	const corpusVersion = options.corpusVersion ?? "0.4.0"
-	// This is a portable manifest namespace, not a host filesystem default.
-	const canonicalPathPrefix = options.canonicalPathPrefix ?? "/data/"
-	const legacyPathPrefix = options.legacyPathPrefix ?? `${mailwomanDataRoot()}/`
-
-	if (!existsSync(options.jsonl)) throw new Error(`jsonl not found: ${options.jsonl}`)
-
-	if (!existsSync(options.baseManifest)) throw new Error(`base-manifest not found: ${options.baseManifest}`)
-
-	const corpusDir = join(options.outDir, `corpus-v${corpusVersion}`)
-	const trainDir = join(corpusDir, "train")
-	await mkdir(trainDir, { recursive: true })
-
-	// Bucket canonical rows by source. Quarantined rows are logged.
-	const buckets = new Map<string, LabeledRow[]>()
-	const quarantine: string[] = []
-	let totalIn = 0
-
-	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(options.jsonl)) {
-		totalIn++
-		const canon = toCanonicalRow(raw, corpusVersion)
-		const result = alignRow(canon)
-
-		if (result.kind !== "labeled") {
-			quarantine.push(`${canon.source_id}\t${result.row.reason}`)
-
-			continue
-		}
-
-		const bucket = buckets.get(canon.source)
-
-		if (bucket) {
-			bucket.push(result.row)
-		} else {
-			buckets.set(canon.source, [result.row])
-		}
-	}
-
-	report?.(`read ${totalIn} rows; ${quarantine.length} quarantined; ${buckets.size} script buckets`)
-
-	const newShards: ShardDescriptor[] = []
-	const sortedKeys = [...buckets.keys()].toSorted()
-
-	for (const source of sortedKeys) {
-		const rows = buckets.get(source)!
-		const slug = source.startsWith("deepseek-translit-") ? source.slice("deepseek-translit-".length) : source
-		const outPath = join(trainDir, `part-translit-${slug}.parquet`)
-		const descriptor = await writeOneShard(rows, outPath, source, corpusVersion)
-		newShards.push(descriptor)
-		report?.(`  ${source}: ${descriptor.rows} rows → ${outPath} (${descriptor.bytes} bytes)`)
-	}
-
-	if (quarantine.length) {
-		const qPath = join(corpusDir, "quarantine-transliteration.tsv")
-		writeFileSync(qPath, quarantine.join("\n") + "\n", "utf8")
-		report?.(`quarantine log → ${qPath} (${quarantine.length} rows)`)
-	}
-
-	// Compose final MANIFEST: rewrite base.shards paths from /mnt/playpen/... → /data/... and append
-	// the new translit shards. Kryptonite shard already lives in the base manifest (it was written
-	// there by Thread B).
-	const base = parseJSONStrict<ShardManifest>(readFileSync(options.baseManifest, "utf8"))
-
-	const rewrittenBase = base.shards.map((sh) => ({
-		...sh,
-		path: canonicalizeShardPath(sh.path, legacyPathPrefix, canonicalPathPrefix),
-	}))
-
-	const newTrainRows = newShards.reduce((sum, sh) => sum + sh.rows, 0)
-
-	const combined: ShardManifest = {
-		corpus_version: corpusVersion,
-		schema: PARQUET_COLUMNS,
-		rows_per_shard: base.rows_per_shard,
-		row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
-		shards: [...rewrittenBase, ...newShards],
-		counts: {
-			train: base.counts.train + newTrainRows,
-			val: base.counts.val,
-			test: base.counts.test,
-		},
-		total_rows: base.total_rows + newTrainRows,
-	}
-
-	const combinedPath = join(corpusDir, "MANIFEST.json")
-	writeFileSync(combinedPath, JSON.stringify(combined, null, 2) + "\n", "utf8")
-	report?.(`wrote combined manifest → ${combinedPath}`)
-	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newTrainRows})`)
-	report?.(`  shards=${combined.shards.length} (base=${base.shards.length}, added=${newShards.length})`)
-	report?.(`  compression=${SHARD_COMPRESSION}`)
-	const pathFix = rewrittenBase.filter((s, i) => s.path !== base.shards[i]!.path).length
-
-	if (pathFix > 0) {
-		report?.(`  path-canonicalized base shards: ${pathFix} (legacy '${legacyPathPrefix}' → '${canonicalPathPrefix}')`)
-	}
-}

package/src/tools/sub-venue/harvest.ts

@@ -1,235 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   @file The harvest side of the sub-venue lexicon: what one row of a harvestable source looks like,
- *   how a file of them is read back, and what a pass over them extracts — attested surfaces plus the
- *   measured shape of each designator's identifier half.
- *
- *   {@link SubVenueHarvestRow} is SOURCE-NEUTRAL, and verified so: an Overture Places row from the
- *   `airport_terminal` category is `{ designatorID, name }` and fits unchanged. It is declared here
- *   rather than imported so the builder does not take a dependency on `@mailwoman/osm` (which
- *   `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
- *
- *   Two measurements shape {@link extractAttestedPhrases}, and both are the reason it is safe to run
- *   over raw OSM at all:
- *
- *   **A feature's `name` is usually the VENUE's name, not a sub-venue phrase.** On the Berlin extract
- *   (2,060 matched features) a `railway=platform` is named `Stendaler Straße` and an `amenity=university`
- *   is named `Hertie School`; across 250,116 named Great Britain features only 6,003 (2.40%) contain a
- *   designator token at all. So names are not harvested wholesale — a name contributes only when it
- *   CONTAINS a phrase already in the surface index, which is what makes `Terminal E (Untere Ebene)`
- *   evidence and `Otto Lilienthal Flughafen Berlin Tegel` not.
- *
- *   **A matched phrase belongs to the record the PHRASE names, not the record the ROW carries.**
- *   Attributing every hit to `row.designatorID` — the rule that matched the FEATURE — produced `west →
- *   platform`, `hall → platform` and `biggin → platform` on the GB extract, because a bus stop tagged
- *   `public_transport=platform` is named "Village Hall" or "West Kensington"; 108 of 133 OSM-derived
- *   surfaces named a different record than the one they pointed at. Attribution therefore runs through
- *   the index, and the row's own designator is kept as `context` — exactly the axis a confound board
- *   needs, since a `hall` seen on a platform is a confound and a `hall` seen on a terminal is evidence.
- */
-
-import { readFileSync } from "node:fs"
-
-import { parseJSONStrict } from "@mailwoman/core/objects"
-import { TextSpliterator } from "spliterator"
-
-import { nameContainsSurfaces, type SurfaceIndex } from "./surfaces.ts"
-import type { IdentifierShape, SubVenueSurface } from "./table.ts"
-
-/**
- * One row of a harvestable source, as read back off JSONL or out of a layer database.
- *
- * SOURCE-NEUTRAL by design, and verified so in wave 2: an Overture Places row from the `airport_terminal` category is
- * `{ designatorID, name }` and fits unchanged. What did NOT fit was the harvest function's hardcoded `osm:name` source
- * stamp — see `overture-subvenue.ts`'s docstring. Declared locally so the builder does not import `@mailwoman/osm`
- * (which `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
- */
-export interface SubVenueHarvestRow {
-	/**
-	 * The designator the SOURCE's rule assigned to the FEATURE. Not necessarily the record a matched phrase names — see
-	 * this file's header on phrase attribution. Carried into {@link SubVenueSurface.context}.
-	 */
-	designatorID: string
-	/**
-	 * `venue` (station, airport, campus) or `sub_venue`, when the source's rule assigns one. The harvest itself does not
-	 * read it; the sub-venue shard recipe fills its venue slot from it.
-	 */
-	tier?: string
-	name?: string | null
-	ref?: string | null
-	localizedNames?: Record<string, string>
-}
-
-/**
- * Classify an OSM `ref` into an {@link IdentifierShape} class.
- *
- * The classes are the ones Berlin's gates actually produced, plus the two aviation forms the corpus task names
- * (`Terminal 2F` is digit-letter, `Concourse B` is letter). `range` covers both separators OSM uses for a gate serving
- * more than one stand: `16-18` and `0/1`.
- */
-export function classifyIdentifier(ref: string): string {
-	const value = ref.trim()
-
-	if (/^[0-9]+$/.test(value)) return "digit"
-
-	if (/^[A-Za-z]$/.test(value)) return "letter"
-
-	if (/^[A-Za-z]+[0-9]+$/.test(value)) return "letter-digit"
-
-	if (/^[0-9]+[A-Za-z]+$/.test(value)) return "digit-letter"
-
-	if (/^[0-9A-Za-z]+\s*[-/]\s*[0-9A-Za-z]+$/.test(value)) return "range"
-
-	return "other"
-}
-
-/**
- * How many real `ref` values each {@link IdentifierShape} keeps.
- *
- * Eight, not "all" and not one. The field exists so a shard author can see what a class actually CONTAINS — GB's
- * `other` class turned out to be semicolon multi-values (`1;2;3`, `13;14`), which one example would have hidden and
- * which the class name does not say. Eight fits a terminal line and covers the variety inside every class the GB
- * extract produced. The COUNT lives in `observations`; this is a sample, not a census.
- */
-const IDENTIFIER_EXAMPLES_PER_SHAPE = 8
-
-/**
- * Options for one harvest pass — which source stamp its surfaces carry and which region they were attested in.
- *
- * Both default to the OSM/unknown-region values wave 1 hardcoded, so an existing caller is unchanged.
- */
-export interface HarvestOptions {
-	/**
-	 * Source family: `osm` yields `osm:name` / `osm:name:<lang>`, `overture` yields `overture:name`.
-	 */
-	source?: string
-	/**
-	 * ISO 3166-1 alpha-2 of the extract or partition. `""` when unknown.
-	 */
-	region?: string
-}
-
-/**
- * Harvest attested phrases and identifier shapes out of a source's rows.
- *
- * `index` gates the name harvest — a name contributes only when it CONTAINS a phrase already in the table. That filter
- * is the whole reason this function is safe to run over raw OSM: see this file's header for the Berlin measurement that
- * motivated it. The index also decides ATTRIBUTION: a hit is a surface of the record the PHRASE names, and the row's
- * own designator is recorded as `context`.
- *
- * Returns surfaces with real `observations` counts, so the lexicon can rank `terminal` above a phrase attested once.
- */
-export function extractAttestedPhrases(
-	rows: Iterable<SubVenueHarvestRow>,
-	index: SurfaceIndex,
-	options: HarvestOptions = {}
-): { surfaces: SubVenueSurface[]; identifierShapes: IdentifierShape[] } {
-	const source = options.source ?? "osm"
-	const region = options.region ?? ""
-	/**
-	 * `phrase\0lang\0source` → { count, context }.
-	 */
-	const surfaceCounts = new Map<string, { count: number; context: Map<string, number> }>()
-	/**
-	 * `designatorID\0shape` → { count, examples }.
-	 */
-	const shapes = new Map<string, { count: number; examples: Set<string> }>()
-
-	const note = (phrase: string, lang: string, sourceTag: string, context: string): void => {
-		const key = `${phrase}\0${lang}\0${sourceTag}`
-		const entry = surfaceCounts.get(key) ?? { count: 0, context: new Map<string, number>() }
-
-		entry.count++
-		entry.context.set(context, (entry.context.get(context) ?? 0) + 1)
-		surfaceCounts.set(key, entry)
-	}
-
-	for (const row of rows) {
-		if (row.name) {
-			for (const hit of nameContainsSurfaces(row.name, index)) {
-				// `und` — the default `name` tag carries no language. Overture's `name` is the same: a
-				// primary name in whatever language the place uses, untagged.
-				note(hit, "und", `${source}:name`, row.designatorID)
-			}
-		}
-
-		for (const [lang, localized] of Object.entries(row.localizedNames ?? {})) {
-			for (const hit of nameContainsSurfaces(localized, index)) {
-				note(hit, lang, `${source}:name:${lang}`, row.designatorID)
-			}
-		}
-
-		if (row.ref) {
-			const shape = classifyIdentifier(row.ref)
-			const key = `${row.designatorID}\0${shape}`
-			const entry = shapes.get(key) ?? { count: 0, examples: new Set<string>() }
-
-			entry.count++
-
-			if (entry.examples.size < IDENTIFIER_EXAMPLES_PER_SHAPE) {
-				entry.examples.add(row.ref.trim())
-			}
-
-			shapes.set(key, entry)
-		}
-	}
-
-	const surfaces: SubVenueSurface[] = [...surfaceCounts].map(([key, entry]) => {
-		const [phrase, lang, sourceTag] = key.split("\0") as [string, string, string]
-		const record = index.get(phrase)!
-
-		return {
-			phrase,
-			recordID: record.recordID,
-			recordKind: record.recordKind,
-			lang,
-			region,
-			source: sourceTag,
-			curated: false,
-			observations: entry.count,
-			context: Object.fromEntries([...entry.context].toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))),
-		}
-	})
-
-	const identifierShapes: IdentifierShape[] = [...shapes].map(([key, entry]) => {
-		const [designatorID, shape] = key.split("\0") as [string, string]
-
-		return {
-			designatorID,
-			region,
-			shape,
-			observations: entry.count,
-			examples: [...entry.examples].toSorted((a, b) => a.localeCompare(b)),
-		}
-	})
-
-	return { surfaces, identifierShapes }
-}
-
-/**
- * Read a JSONL file of {@link SubVenueHarvestRow}s. Blank lines and unparseable rows are skipped rather than fatal — an
- * extract is a build output, and one malformed line should not cost the whole lexicon.
- */
-export function readSubVenueJSONL(path: string): SubVenueHarvestRow[] {
-	const out: SubVenueHarvestRow[] = []
-
-	// `TextSpliterator` rather than `split("\n")` — a whole-country extract runs to 250,000 lines
-	// (52 MB for Great Britain), and materializing every segment before reading the first is exactly
-	// what the repo lint rule exists to prevent.
-	for (const line of TextSpliterator.from(readFileSync(path, "utf8"))) {
-		const trimmed = line.trim()
-
-		if (!trimmed) continue
-
-		try {
-			out.push(parseJSONStrict<SubVenueHarvestRow>(trimmed))
-		} catch {
-			continue
-		}
-	}
-
-	return out
-}