@mailwoman/corpus 9.1.0 → 9.3.0

package/src/tools/overture-subvenue.ts

@@ -1,215 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Overture Places source for the sub-venue lexicon (#35 wave 2) — read the sub-venue-bearing
- *   category slices of `poi.db` (spatial layer #1, `$MAILWOMAN_DATA_ROOT/poi/poi.db`) and yield
- *   {@link SubVenueHarvestRow}s the lexicon builder consumes exactly like an OSM extract.
- *
- *   ── WHY OVERTURE AT ALL, when the OSM extractor already exists ────────────────────────────────────
- *   Wave 1 measured `concourse` at 21 real Overture rows against 4 in the whole Great Britain OSM
- *   extract, 3 of which are a street called CONCOURSE WAY. Overture's `airport_terminal` category is
- *   curated venue-interior naming; OSM's `aeroway=terminal` is a building footprint that usually
- *   carries the AIRPORT's name. The two sources fail differently, so both are read.
- *
- *   ── THE CATEGORY SET IS MEASURED, NOT GUESSED ────────────────────────────────────────────────────
- *   A full scan of all 13,681,698 rows (2026-08-05, poi.db vintage 2026-05-20.0) counted, per
- *   category, how many named rows carry a designator token. The ranking is NOT what a category name
- *   predicts — `gas_station` leads the whole table with 12,996 hits, every one of them the token
- *   `station` inside "Holiday Station" / "Chevron Station Seward", and `shoe_store` contributes 708
- *   hits of `wing` because Red Wing sells boots. {@link OVERTURE_SUBVENUE_CATEGORIES} is the four
- *   categories whose hits survived reading the distribution; the rejects are listed below it so nobody
- *   re-proposes them.
- *
- *   ── poi.db IS FOUR COUNTRIES ─────────────────────────────────────────────────────────────────────
- *   Measured the same day: US 11,521,612 / CA 794,418 / FR 721,352 / MX 644,316, and nothing else. The
- *   shipped layer is not a world gazetteer, so Overture can attest en-US, en-CA, fr-FR and es-MX
- *   surfaces and NOTHING ELSE. Every non-Latin designator the corpus task asks for — `ターミナル`,
- *   `Halle`, `Flügel` — has to come from the OSM leg. Do not read a zero count here as evidence of
- *   absence in the world; it is evidence of absence in four countries.
- *
- *   ── The row shape fits; the PROVENANCE STAMP did not ─────────────────────────────────────────────
- *   Wave 1's `OSMSubVenueRow` was written to accept a non-OSM row, and it does: an Overture row is
- *   `{ designatorID, name }` with no `ref` and no `localizedNames`. What did NOT fit is
- *   `extractAttestedPhrases`, which hardcoded `osm:name` as the surface's `source`. Feeding Overture
- *   rows through it unchanged would have labelled every Overture surface as OSM-attested — a
- *   provenance lie, and under ODbL a consequential one. Hence the `source`/`region` options on that
- *   function and the source-neutral {@link SubVenueHarvestRow} name.
- */
-
-import { DatabaseSync } from "node:sqlite"
-
-import { DatabaseClient } from "@mailwoman/core/kysley/client"
-
-import type { SubVenueHarvestRow } from "./sub-venue-lexicon.ts"
-
-/**
- * Overture Places category → the designator its rows attest.
- *
- * Each entry carries the 2026-08-05 full-scan measurement in a comment: `rows` is the category's total, `hits` is how
- * many of its named rows contain any designator token, and the token list is the top of that distribution.
- *
- * REJECTED, with the number that rejected them — do not re-add without a fresh census:
- *
- * - `gas_station` (12,996 hits) — all `station` inside a brand name (Holiday Station, Chevron Station Seward).
- * - `fire_station` (10,377) — same, plus 330 `hall` from fire halls, which are venues rather than sub-venues.
- * - `town_hall` (5,528) — `City Hall` is a whole building, the `venue` tier, not interior structure.
- * - `building_supply_store` (4,030) — `building` inside "Allied Building Products".
- * - `jehovahs_witness_place_of_worship` (2,334) — every hit is "Kingdom Hall of Jehovah's Witnesses".
- * - `shoe_store` (909, of which 708 `wing`) — Red Wing. This one is a CONFOUND BOARD entry, not a source.
- * - `college_university` (3,697) — 2,082 `campus`, but the row names the whole institution; `campus_building` is the
- *   interior slice and is kept instead.
- * - `airport` (6,000 rows, 4,302 hits) — 4,071 of them are the token `airport` in the aerodrome's own name. Venue tier,
- *   already covered by OurAirports, and it drowns the interior signal.
- * - `transport_interchange` (1 row), `rail_facility_or_service` (81 rows, 2 hits) — too small to matter.
- * - `public_transit_facility_or_service` (2,274 rows, 185 hits) — 161 are `station` naming the station itself.
- */
-export const OVERTURE_SUBVENUE_CATEGORIES: Readonly<Record<string, string>> = {
-	// 1,022 rows / 669 hits — terminal 266, concourse 19, gate 13, hall 4. The densest interior naming
-	// in the layer and the reason this reader exists.
-	airport_terminal: "terminal",
-	// 7,366 rows / 3,382 hits — hall 2,134, building 1,019, campus 209. US academic halls and numbered
-	// campus buildings: "UAA Cuddy Hall", "UAA Science Building". The row is a building ON a campus, so
-	// `campus` is its CONTEXT designator; the phrase found inside the name decides the record it
-	// attests, which for these rows is mostly `hall`.
-	campus_building: "campus",
-	// 443 rows / 301 hits — pier 282, terminal 17.
-	pier: "pier",
-	// 388 rows / 66 hits — airport 36, terminal 20, concourse 5, gate 3. Small, but the hits are almost
-	// all genuine ("Delta Sky Club Concourse B").
-	airport_lounge: "terminal",
-}
-
-/**
- * The `poi` and `poi_category_codes` columns this reader touches, declared LOCALLY.
- *
- * `@mailwoman/resolver-wof-sqlite` owns the full `POIDatabase` interface, and `@mailwoman/corpus` does not depend on it
- * — the same dependency-direction call `sub-venue-lexicon.ts` makes for `@mailwoman/osm`'s row type. This is a
- * read-only projection of four columns; `overture-subvenue.test.ts` builds a fixture with exactly this DDL, so a column
- * rename upstream fails a test here rather than throwing at runtime against a 3.9 GB database nobody has in CI.
- */
-interface POIReadDatabase {
-	poi: {
-		category_id: number
-		name: string | null
-		country: string
-	}
-	poi_category_codes: {
-		id: number
-		category: string
-	}
-	layer_manifest: {
-		name: string
-		source_vintage: string
-	}
-}
-
-/**
- * The layer's `source_vintage` — the Overture release the rows came from (`2026-05-20.0`), for the lexicon's
- * `sources[]`.
- *
- * Read off the database rather than passed in, because a vintage a caller types is a vintage that goes stale silently.
- * The layer-contract tables are part of every layer database by construction — see `docs/engineering/reference/
- * layer-contract.mdx` — so there is no version of poi.db where this is absent.
- */
-export async function readOvertureLayerVintage(databasePath: string): Promise<string> {
-	using kdb = new DatabaseClient<POIReadDatabase>({
-		database: new DatabaseSync(databasePath, { readOnly: true }),
-	})
-
-	const row = await kdb
-		.selectFrom("layer_manifest")
-		.select("source_vintage")
-		.where("name", "=", "poi")
-		.executeTakeFirst()
-
-	return row?.source_vintage ?? ""
-}
-
-export interface ReadOvertureSubVenuesOptions {
-	/**
-	 * Path to `poi.db`. Typically `dataRootPath("poi", "poi.db")`.
-	 */
-	databasePath: string
-	/**
-	 * Category → designator map. Defaults to {@link OVERTURE_SUBVENUE_CATEGORIES}.
-	 */
-	categories?: Readonly<Record<string, string>>
-	/**
-	 * Restrict to these ISO 3166-1 alpha-2 countries. Omit for all four the layer carries.
-	 */
-	countries?: readonly string[]
-}
-
-/**
- * One Overture row plus the country it came from — {@link SubVenueHarvestRow} widened by the field the layer has and an
- * OSM extract does not (a Geofabrik extract's country is a property of the invocation, so OSM rows carry `country:
- * ""`).
- */
-export interface OvertureSubVenueRow extends SubVenueHarvestRow {
-	/**
-	 * ISO 3166-1 alpha-2, straight off the Overture partition.
-	 */
-	country: string
-	/**
-	 * The Overture category the row was read from, so a surface's provenance survives past the designator.
-	 */
-	category: string
-}
-
-/**
- * Read the sub-venue-bearing category slices of `poi.db`.
- *
- * Cold path, already async, no interface constraint — so Kysely, per the repo's inline-SQL rule.
- *
- * `category_id` is the SECOND component of poi.db's clustered `(h3_cell, category_id, …)` primary key, which reads like
- * a full 13.7M-row scan and is not one: SQLite skip-scans the leading `h3_cell` and the whole four-category read
- * returned 9,219 rows in **1.4 s** on the shipped layer (2026-08-05). A JS-side scan of every row to survey the same
- * question took 52 s, which is the number to remember if you are tempted to filter in JS instead.
- */
-export async function readOvertureSubVenues(options: ReadOvertureSubVenuesOptions): Promise<OvertureSubVenueRow[]> {
-	const categories = options.categories ?? OVERTURE_SUBVENUE_CATEGORIES
-
-	using kdb = new DatabaseClient<POIReadDatabase>({
-		database: new DatabaseSync(options.databasePath, { readOnly: true }),
-	})
-
-	const codes = await kdb
-		.selectFrom("poi_category_codes")
-		.select(["id", "category"])
-		.where("category", "in", Object.keys(categories))
-		.execute()
-
-	if (!codes.length) return []
-
-	const designatorByID = new Map<number, string>()
-	const categoryByID = new Map<number, string>()
-
-	for (const code of codes) {
-		const designator = categories[code.category]
-
-		if (!designator) continue
-		designatorByID.set(code.id, designator)
-		categoryByID.set(code.id, code.category)
-	}
-
-	let query = kdb
-		.selectFrom("poi")
-		.select(["category_id", "name", "country"])
-		.where("category_id", "in", [...designatorByID.keys()])
-		.where("name", "is not", null)
-
-	if (options.countries?.length) {
-		query = query.where("country", "in", [...options.countries])
-	}
-
-	const rows = await query.execute()
-
-	return rows.map((row) => ({
-		designatorID: designatorByID.get(row.category_id)!,
-		name: row.name,
-		country: row.country,
-		category: categoryByID.get(row.category_id)!,
-	}))
-}

package/src/tools/shard-kryptonite.ts

@@ -1,138 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Build a parquet shard from the DeepSeek-generated kryptonite JSONL and emit the corpus-v0.4.0
- *   MANIFEST. corpus-v0.4.0 is a pure adapter-addition revision: it points at every shard from
- *   v0.3.0 plus the new kryptonite shard(s). No v0.3.0 bytes are touched or re-shuffled.
- *
- *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for the why; that doc also pins the
- *   DeepSeek model version + prompt versions used to produce the JSONL.
- *
- *   Invoke via `mailwoman corpus shard kryptonite \
- *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
- *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/MANIFEST.json \
- *   --out-dir /data/corpus/versioned/v0.4.0`
- */
-
-import { existsSync, readFileSync, writeFileSync } from "node:fs"
-import { mkdir } from "node:fs/promises"
-import { join } from "node:path"
-
-import { parseJSONStrict } from "@mailwoman/core/objects"
-import type { CanonicalRow, LabeledRow } from "@mailwoman/corpus/types"
-import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, SHARD_COMPRESSION, writeShards } from "@mailwoman/corpus/utils"
-import type { ShardManifest } from "@mailwoman/corpus/utils"
-import { JSONSpliterator } from "spliterator"
-
-export interface ShardKryptoniteOptions {
-	jsonl: string
-	baseManifest: string
-	outDir: string
-	/**
-	 * Default `"0.4.0"`.
-	 */
-	corpusVersion?: string
-	/**
-	 * Default `"deepseek-kryptonite"`.
-	 */
-	source?: string
-}
-
-async function* canonicalRows(jsonl: string, corpusVersion: string): AsyncIterable<CanonicalRow> {
-	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(jsonl)) {
-		// Strip sidecar underscore-prefixed fields the generator left behind for debugging.
-		const components = raw["components"] as Record<string, string>
-
-		yield {
-			raw: raw["raw"] as string,
-			components,
-			country: (raw["country"] as string) ?? "US",
-			locale: (raw["locale"] as string) ?? undefined,
-			source: (raw["source"] as string) ?? "deepseek-kryptonite",
-			source_id: raw["source_id"] as string,
-			corpus_version: corpusVersion,
-			license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
-			synth: raw["synth"] as CanonicalRow["synth"],
-		}
-	}
-}
-
-async function* labeledRows(jsonl: string, corpusVersion: string, quarantineLog: string[]): AsyncIterable<LabeledRow> {
-	for await (const row of canonicalRows(jsonl, corpusVersion)) {
-		const result = alignRow(row)
-
-		if (result.kind === "labeled") {
-			yield result.row
-		} else {
-			quarantineLog.push(`${row.source_id}\t${result.row.reason}`)
-		}
-	}
-}
-
-export async function buildKryptoniteShard(
-	options: ShardKryptoniteOptions,
-	report?: (line: string) => void
-): Promise<void> {
-	const corpusVersion = options.corpusVersion ?? "0.4.0"
-	const source = options.source ?? "deepseek-kryptonite"
-
-	if (!existsSync(options.jsonl)) throw new Error(`jsonl not found: ${options.jsonl}`)
-
-	if (!existsSync(options.baseManifest)) throw new Error(`base-manifest not found: ${options.baseManifest}`)
-
-	await mkdir(options.outDir, { recursive: true })
-
-	const quarantine: string[] = []
-
-	const newManifest = await writeShards(
-		{ train: labeledRows(options.jsonl, corpusVersion, quarantine) },
-		{ outputDir: options.outDir, corpusVersion }
-	)
-
-	report?.(
-		`wrote ${newManifest.total_rows} rows into ${newManifest.shards.length} shard(s); ` +
-			`quarantined ${quarantine.length}`
-	)
-
-	if (quarantine.length) {
-		const qPath = join(options.outDir, `corpus-v${corpusVersion}`, "quarantine-kryptonite.tsv")
-		writeFileSync(qPath, quarantine.join("\n") + "\n", "utf8")
-		report?.(`quarantine log → ${qPath}`)
-	}
-
-	// Stamp the new shard's source field for audit.ts (which prefers shard.source over
-	// first_source_id-prefix inference). Without this, deepseek-kryptonite IDs would have
-	// to match a prefix in KNOWN_SOURCE_PREFIXES — we add it there too as a belt-and-braces.
-	for (const sh of newManifest.shards) {
-		;(sh as unknown as { source: string }).source = source
-	}
-
-	// Compose the final corpus-v0.4.0 manifest: every shard from base + the new shard(s).
-	const base = parseJSONStrict<ShardManifest>(readFileSync(options.baseManifest, "utf8"))
-
-	const combined: ShardManifest = {
-		corpus_version: corpusVersion,
-		schema: PARQUET_COLUMNS,
-		rows_per_shard: base.rows_per_shard,
-		row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
-		shards: [...base.shards, ...newManifest.shards],
-		counts: {
-			train: base.counts.train + (newManifest.counts.train ?? 0),
-			val: base.counts.val,
-			test: base.counts.test,
-		},
-		total_rows: base.total_rows + newManifest.total_rows,
-	}
-
-	// Stamp source on the legacy v0.3.0 shards too, so audit's shard.source path is the
-	// authoritative one. v0.3.0 shards mix sources; we use the first_source_id-prefix
-	// inference for them (audit.ts will re-derive on its own when shard.source is absent).
-	const combinedPath = join(options.outDir, `corpus-v${corpusVersion}`, "MANIFEST.json")
-	writeFileSync(combinedPath, JSON.stringify(combined, null, 2) + "\n", "utf8")
-	report?.(`wrote combined manifest → ${combinedPath}`)
-	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newManifest.total_rows})`)
-	report?.(`  shards=${combined.shards.length} (base=${base.shards.length}, added=${newManifest.shards.length})`)
-	report?.(`  compression=${SHARD_COMPRESSION}`)
-}

package/src/tools/shard-translit.ts

@@ -1,270 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Build per-script parquet shards from the DeepSeek-generated transliteration JSONL and emit the
- *   corpus-v0.4.0 MANIFEST that combines them with the existing kryptonite + v0.3.0 shards.
- *
- *   Sibling to `shard-kryptonite.ts`. The two modules share the same composition pattern: take a
- *   base MANIFEST, append new shards, write a combined MANIFEST. Differences specific to
- *   transliteration:
- *
- *   - One JSONL contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
- *       `source` and write one shard per script so `audit.ts` can attribute each shard to its
- *       synthetic source without relying on filename-prefix inference.
- *   - Each shard is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
- *       `part-0000.parquet`, which v0.4.0's first builder already produced).
- *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 shard paths are
- *       rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined MANIFEST so
- *       all paths share one container-friendly form.
- *
- *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for prompts, model, and the
- *   reproducibility contract.
- *
- *   Invoke via `mailwoman corpus shard translit \
- *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
- *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/MANIFEST.json \
- *   --out-dir /data/corpus/versioned/v0.4.0`
- */
-
-import { existsSync, readFileSync, writeFileSync } from "node:fs"
-import { mkdir, stat } from "node:fs/promises"
-import { join } from "node:path"
-
-import { parseJSONStrict } from "@mailwoman/core/objects"
-import { mailwomanDataRoot, sha256File } from "@mailwoman/core/utils"
-import type { CanonicalRow, LabeledRow } from "@mailwoman/corpus/types"
-import {
-	alignRow,
-	LABELED_ROW_SCHEMA,
-	PARQUET_COLUMNS,
-	ROW_GROUP_SIZE,
-	rowToParquet,
-	SHARD_COMPRESSION,
-} from "@mailwoman/corpus/utils"
-import type { ParquetRow, ShardDescriptor, ShardManifest } from "@mailwoman/corpus/utils"
-import { JSONSpliterator } from "spliterator"
-
-import { ParquetWriter } from "../parquet-wrapper/index.ts"
-
-export interface ShardTranslitOptions {
-	jsonl: string
-	baseManifest: string
-	outDir: string
-	/**
-	 * Default `"0.4.0"`.
-	 */
-	corpusVersion?: string
-	/**
-	 * Default `"/data/"`.
-	 */
-	canonicalPathPrefix?: string
-	/**
-	 * Prefix the base manifest's shard paths currently carry, to be rewritten to
-	 * {@link ShardTranslitOptions.canonicalPathPrefix}. Defaults to `mailwomanDataRoot()` with a trailing slash — the
-	 * root that WROTE those paths. Pass it explicitly when translating a manifest generated under a different
-	 * `$MAILWOMAN_DATA_ROOT` than the one you are running with.
-	 */
-	legacyPathPrefix?: string
-}
-
-function toCanonicalRow(raw: Record<string, unknown>, corpusVersion: string): CanonicalRow {
-	return {
-		raw: raw["raw"] as string,
-		components: raw["components"] as Record<string, string>,
-		country: (raw["country"] as string) ?? "US",
-		locale: (raw["locale"] as string) ?? undefined,
-		source: raw["source"] as string,
-		source_id: raw["source_id"] as string,
-		corpus_version: corpusVersion,
-		license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
-		synth: raw["synth"] as CanonicalRow["synth"],
-	}
-}
-
-function appendShape(row: ParquetRow): Record<string, unknown> {
-	const out: Record<string, unknown> = {
-		raw: row.raw,
-		tokens: row.tokens,
-		labels: row.labels,
-		country: row.country,
-		source: row.source,
-		source_id: row.source_id,
-		corpus_version: row.corpus_version,
-		license: row.license,
-	}
-
-	if (row.locale !== null) {
-		out.locale = row.locale
-	}
-
-	if (row.synth_method !== null) {
-		out.synth_method = row.synth_method
-	}
-
-	if (row.synth_base_id !== null) {
-		out.synth_base_id = row.synth_base_id
-	}
-
-	return out
-}
-
-/**
- * Write one shard for a single source slug. Returns the populated ShardDescriptor + a list of quarantine reasons for
- * rows that failed alignment.
- */
-async function writeOneShard(
-	rows: readonly LabeledRow[],
-	outPath: string,
-	source: string,
-	corpusVersion: string
-): Promise<ShardDescriptor> {
-	const writer = await ParquetWriter.openFile<ParquetRow>(LABELED_ROW_SCHEMA, outPath, {
-		rowGroupSize: ROW_GROUP_SIZE,
-	})
-
-	writer.setMetadata("mailwoman.corpus_version", corpusVersion)
-	writer.setMetadata("mailwoman.split", "train")
-	writer.setMetadata("mailwoman.shard_source", source)
-
-	let firstSourceID = ""
-	let lastSourceID = ""
-
-	for (const row of rows) {
-		const pq = rowToParquet(row)
-		await writer.appendRow(appendShape(pq) as unknown as ParquetRow)
-
-		if (firstSourceID === "") {
-			firstSourceID = row.source_id
-		}
-
-		lastSourceID = row.source_id
-	}
-
-	await writer.close()
-
-	const fileStat = await stat(outPath)
-	const sha256 = await sha256File(outPath)
-
-	return {
-		split: "train",
-		path: outPath,
-		format: "parquet",
-		compression: SHARD_COMPRESSION,
-		rows: rows.length,
-		bytes: fileStat.size,
-		sha256,
-		first_source_id: firstSourceID,
-		last_source_id: lastSourceID,
-		// Stamp source so audit.ts attributes the shard without falling back to filename-prefix
-		// inference. Cast widens ShardDescriptor; the field is read by audit.ts.
-		...({ source } as Record<string, string>),
-	}
-}
-
-function canonicalizeShardPath(path: string, legacyPrefix: string, canonicalPrefix: string): string {
-	if (path.startsWith(legacyPrefix)) return canonicalPrefix + path.slice(legacyPrefix.length)
-
-	return path
-}
-
-export async function buildTranslitShard(
-	options: ShardTranslitOptions,
-	report?: (line: string) => void
-): Promise<void> {
-	const corpusVersion = options.corpusVersion ?? "0.4.0"
-	const canonicalPathPrefix = options.canonicalPathPrefix ?? "/data/"
-	const legacyPathPrefix = options.legacyPathPrefix ?? `${mailwomanDataRoot()}/`
-
-	if (!existsSync(options.jsonl)) throw new Error(`jsonl not found: ${options.jsonl}`)
-
-	if (!existsSync(options.baseManifest)) throw new Error(`base-manifest not found: ${options.baseManifest}`)
-
-	const corpusDir = join(options.outDir, `corpus-v${corpusVersion}`)
-	const trainDir = join(corpusDir, "train")
-	await mkdir(trainDir, { recursive: true })
-
-	// Bucket canonical rows by source. Quarantined rows are logged.
-	const buckets = new Map<string, LabeledRow[]>()
-	const quarantine: string[] = []
-	let totalIn = 0
-
-	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(options.jsonl)) {
-		totalIn++
-		const canon = toCanonicalRow(raw, corpusVersion)
-		const result = alignRow(canon)
-
-		if (result.kind !== "labeled") {
-			quarantine.push(`${canon.source_id}\t${result.row.reason}`)
-
-			continue
-		}
-
-		const bucket = buckets.get(canon.source)
-
-		if (bucket) {
-			bucket.push(result.row)
-		} else {
-			buckets.set(canon.source, [result.row])
-		}
-	}
-
-	report?.(`read ${totalIn} rows; ${quarantine.length} quarantined; ${buckets.size} script buckets`)
-
-	const newShards: ShardDescriptor[] = []
-	const sortedKeys = [...buckets.keys()].toSorted()
-
-	for (const source of sortedKeys) {
-		const rows = buckets.get(source)!
-		const slug = source.startsWith("deepseek-translit-") ? source.slice("deepseek-translit-".length) : source
-		const outPath = join(trainDir, `part-translit-${slug}.parquet`)
-		const descriptor = await writeOneShard(rows, outPath, source, corpusVersion)
-		newShards.push(descriptor)
-		report?.(`  ${source}: ${descriptor.rows} rows → ${outPath} (${descriptor.bytes} bytes)`)
-	}
-
-	if (quarantine.length) {
-		const qPath = join(corpusDir, "quarantine-transliteration.tsv")
-		writeFileSync(qPath, quarantine.join("\n") + "\n", "utf8")
-		report?.(`quarantine log → ${qPath} (${quarantine.length} rows)`)
-	}
-
-	// Compose final MANIFEST: rewrite base.shards paths from /mnt/playpen/... → /data/... and append
-	// the new translit shards. Kryptonite shard already lives in the base manifest (it was written
-	// there by Thread B).
-	const base = parseJSONStrict<ShardManifest>(readFileSync(options.baseManifest, "utf8"))
-
-	const rewrittenBase = base.shards.map((sh) => ({
-		...sh,
-		path: canonicalizeShardPath(sh.path, legacyPathPrefix, canonicalPathPrefix),
-	}))
-
-	const newTrainRows = newShards.reduce((sum, sh) => sum + sh.rows, 0)
-
-	const combined: ShardManifest = {
-		corpus_version: corpusVersion,
-		schema: PARQUET_COLUMNS,
-		rows_per_shard: base.rows_per_shard,
-		row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
-		shards: [...rewrittenBase, ...newShards],
-		counts: {
-			train: base.counts.train + newTrainRows,
-			val: base.counts.val,
-			test: base.counts.test,
-		},
-		total_rows: base.total_rows + newTrainRows,
-	}
-
-	const combinedPath = join(corpusDir, "MANIFEST.json")
-	writeFileSync(combinedPath, JSON.stringify(combined, null, 2) + "\n", "utf8")
-	report?.(`wrote combined manifest → ${combinedPath}`)
-	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newTrainRows})`)
-	report?.(`  shards=${combined.shards.length} (base=${base.shards.length}, added=${newShards.length})`)
-	report?.(`  compression=${SHARD_COMPRESSION}`)
-	const pathFix = rewrittenBase.filter((s, i) => s.path !== base.shards[i]!.path).length
-
-	if (pathFix > 0) {
-		report?.(`  path-canonicalized base shards: ${pathFix} (legacy '${legacyPathPrefix}' → '${canonicalPathPrefix}')`)
-	}
-}