@mailwoman/corpus 9.2.0 → 9.4.0

package/src/utils/parquet.ts

@@ -1,391 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Final output sharder for the corpus pipeline.
- *
- *   Phase 1 (#9) shipped JSONL shards + a Python (PyArrow) converter as the path to binary Parquet —
- *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
- *   writer. The build pipeline no longer touches Python at all in its hot path; the only remaining
- *   Python is the one-shot `train_tokenizer.py` SentencePiece step.
- *
- *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`, but `@dsnp/parquetjs` 1.7.0 only
- *   supports UNCOMPRESSED / GZIP / SNAPPY / BROTLI (see `node_modules/@dsnp/parquetjs/dist/lib/
- *   compression.js`). SNAPPY is the standard ML-corpus default (PyArrow's default too) and is the
- *   closest substitute on speed; revisit if @dsnp/parquetjs gains zstd support. Documented in
- *   `DECISIONS.md`.
- *
- *   Layout under `<outputDir>`:
- *
- *   ```
- *   corpus-v<version>/
- *   MANIFEST.json
- *   train/
- *     part-0000.parquet
- *     part-0001.parquet
- *     ...
- *   val/
- *     part-0000.parquet
- *   test/
- *     part-0000.parquet
- * ```
- *
- *   Each shard caps at `rowsPerShard` (default 1_000_000); within a shard, parquetjs flushes row
- *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
- *   shard's path, row count, byte size, and SHA-256 (computed by re-reading the shard once after
- *   close — cheap relative to writing it).
- */
-
-import { mkdir, stat, writeFile } from "node:fs/promises"
-import { join } from "node:path"
-
-import { sha256File } from "@mailwoman/core/utils"
-
-import { ParquetWriter, type ParquetSchemaDefinition } from "#parquet-wrapper"
-import type { LabeledRow } from "#types"
-
-import type { SplitName } from "./split.ts"
-
-/**
- * Row groups flush at this many rows (parquetjs internal cadence within a shard).
- */
-export const ROW_GROUP_SIZE = 50_000
-
-/**
- * Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0.
- */
-export const SHARD_COMPRESSION = "SNAPPY" as const
-
-/**
- * A single Parquet-style row shape. The `[key: string]: unknown` index signature is required for compatibility with
- * `ParquetRecordLike` in the wrapper — parquetjs accepts any string key on rows.
- *
- * The three columns the schema marks `optional: true` are optional here too: `appendRow` receives the row with them
- * OMITTED rather than null (see {@link appendShape}), and a reader gets back whichever of null/absent parquetjs
- * surfaces.
- */
-export interface ParquetRow {
-	raw: string
-	tokens: readonly string[]
-	labels: readonly string[]
-	span_starts: readonly number[]
-	span_ends: readonly number[]
-	span_tags: readonly string[]
-	country: string
-	locale?: string | null
-	source: string
-	source_id: string
-	corpus_version: string
-	license: string
-	synth_method?: string | null
-	synth_base_id?: string | null
-	[key: string]: unknown
-}
-
-/**
- * Column names emitted into every shard. Matches `ParquetRow`.
- */
-export const PARQUET_COLUMNS = [
-	"raw",
-	"tokens",
-	"labels",
-	"span_starts",
-	"span_ends",
-	"span_tags",
-	"country",
-	"locale",
-	"source",
-	"source_id",
-	"corpus_version",
-	"license",
-	"synth_method",
-	"synth_base_id",
-] as const
-
-/* oxlint-disable unicorn/text-encoding-identifier-case -- `"UTF8"` below is a ParquetType enum member,
-   not a text-encoding identifier. Lowercasing it does not type-check against ParquetSchemaDefinition,
-   and the rule has no way to tell the two apart. */
-
-/**
- * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
- * tokens/labels arrays. Compression is per-column SNAPPY.
- */
-export const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow> = {
-	raw: { type: "UTF8", compression: SHARD_COMPRESSION },
-	tokens: { type: "UTF8", repeated: true, compression: SHARD_COMPRESSION },
-	labels: { type: "UTF8", repeated: true, compression: SHARD_COMPRESSION },
-	// v0.5.0 char-offset label spans (#519): parallel arrays over `raw` (UTF-16 code units,
-	// [start, end) exclusive-end, sorted, non-overlapping). INT32 — raw is a short address string,
-	// and INT32 round-trips as `number` where parquetjs INT64 would surface bigint.
-	span_starts: { type: "INT32", repeated: true, compression: SHARD_COMPRESSION },
-	span_ends: { type: "INT32", repeated: true, compression: SHARD_COMPRESSION },
-	span_tags: { type: "UTF8", repeated: true, compression: SHARD_COMPRESSION },
-	country: { type: "UTF8", compression: SHARD_COMPRESSION },
-	locale: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
-	source: { type: "UTF8", compression: SHARD_COMPRESSION },
-	source_id: { type: "UTF8", compression: SHARD_COMPRESSION },
-	corpus_version: { type: "UTF8", compression: SHARD_COMPRESSION },
-	license: { type: "UTF8", compression: SHARD_COMPRESSION },
-	synth_method: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
-	synth_base_id: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
-}
-
-/**
- * Per-shard metadata captured in `MANIFEST.json`.
- */
-export interface ShardDescriptor {
-	split: SplitName
-	path: string
-	format: "parquet"
-	compression: typeof SHARD_COMPRESSION
-	rows: number
-	bytes: number
-	sha256: string
-	first_source_id: string
-	last_source_id: string
-	/**
-	 * The shard's corpus source slug, when the writer knows it. `audit.ts` prefers this over inferring the source from
-	 * `first_source_id`'s prefix; `writeShards` itself writes multi-source shards and leaves it unset.
-	 */
-	source?: string
-}
-
-export interface ShardManifest {
-	corpus_version: string
-	schema: readonly string[]
-	rows_per_shard: number
-	row_group_size: number
-	shards: ShardDescriptor[]
-	counts: Record<SplitName, number>
-	total_rows: number
-}
-
-export interface WriteShardsOptions {
-	/**
-	 * Root output directory; corpus version dir is created beneath.
-	 */
-	outputDir: string
-
-	/**
-	 * Corpus version stamped onto rows + into the output directory name.
-	 */
-	corpusVersion: string
-
-	/**
-	 * Max rows per `.parquet` shard. Default 1_000_000 per the Phase 1 plan.
-	 */
-	rowsPerShard?: number
-}
-
-/**
- * Pre-partitioned labeled-row streams, one per split. Callers (`buildCorpus`) decide each row's split inline at align
- * time via `splitForRow` and route rows to the matching stream, eliminating the prior `Map<source_id, SplitName>` O(n)
- * lookup table.
- *
- * Splits with no rows can be omitted (or passed as an empty iterable); `writeShards` skips them.
- */
-export type PerSplitRows = Partial<Record<SplitName, AsyncIterable<LabeledRow>>>
-
-/**
- * Project a labeled row to the Parquet schema.
- *
- * The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
- * from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the shard (the "builders
- * before parquet = silent loss" hazard). Loud failure, naming the row, instead.
- */
-export function rowToParquet(row: LabeledRow): ParquetRow {
-	const { span_starts, span_ends, span_tags } = row
-
-	if (span_starts === undefined || span_ends === undefined || span_tags === undefined) {
-		throw new Error(
-			`rowToParquet: row is missing the char-offset span triple (#519) — ` +
-				`span_starts=${span_starts !== undefined} span_ends=${span_ends !== undefined} span_tags=${span_tags !== undefined} ` +
-				`(source=${row.source}, source_id=${row.source_id}). ` +
-				`Every parquet-bound row must carry span_starts/span_ends/span_tags; ` +
-				`producers that emit tokens/labels only have not migrated to the v0.5.0 format.`
-		)
-	}
-
-	if (span_starts.length !== span_ends.length || span_starts.length !== span_tags.length) {
-		throw new Error(
-			`rowToParquet: span triple arrays are not parallel — ` +
-				`starts=${span_starts.length} ends=${span_ends.length} tags=${span_tags.length} ` +
-				`(source=${row.source}, source_id=${row.source_id})`
-		)
-	}
-
-	return {
-		raw: row.raw,
-		tokens: row.tokens,
-		labels: row.labels,
-		span_starts,
-		span_ends,
-		span_tags,
-		country: row.country,
-		locale: row.locale ?? null,
-		source: row.source,
-		source_id: row.source_id,
-		corpus_version: row.corpus_version,
-		license: row.license,
-		synth_method: row.synth?.method ?? null,
-		synth_base_id: row.synth?.base_source_id ?? null,
-	}
-}
-
-/**
- * Project a `ParquetRow` for `appendRow`. parquetjs treats `null` as "skip" for `optional` columns; passing it
- * explicitly is fine, but cleaner to omit so the on-disk Definition Levels match what PyArrow / DuckDB / etc. produce
- * for the same logical row.
- *
- * EXPORTED because it was not, and a second producer hand-rolled its own copy that predates the v0.5.0 span triple
- * (#519). `LABELED_ROW_SCHEMA` declares those three columns `repeated` and NOT optional, so the copy wrote empty arrays
- * under a schema promising values. Every new column has to arrive here once, not once per writer.
- */
-export function appendShape(row: ParquetRow): ParquetRow {
-	const out: ParquetRow = {
-		raw: row.raw,
-		tokens: row.tokens,
-		labels: row.labels,
-		span_starts: row.span_starts,
-		span_ends: row.span_ends,
-		span_tags: row.span_tags,
-		country: row.country,
-		source: row.source,
-		source_id: row.source_id,
-		corpus_version: row.corpus_version,
-		license: row.license,
-	}
-
-	if (row.locale !== null) {
-		out.locale = row.locale
-	}
-
-	if (row.synth_method !== null) {
-		out.synth_method = row.synth_method
-	}
-
-	if (row.synth_base_id !== null) {
-		out.synth_base_id = row.synth_base_id
-	}
-
-	return out
-}
-
-/**
- * Stream labeled rows into `.parquet` shards, one set of shards per split. Splits are processed sequentially so that
- * only one shard writer is open at a time — memory cost is bounded by the parquetjs row-group buffer (~`ROW_GROUP_SIZE
- * × row_size`), not by the labeled-row count.
- *
- * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
- * because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
- * for the new wire-up.
- */
-export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptions): Promise<ShardManifest> {
-	const rowsPerShard = opts.rowsPerShard ?? 1_000_000
-	const corpusDir = join(opts.outputDir, `corpus-v${opts.corpusVersion}`)
-	await mkdir(corpusDir, { recursive: true })
-
-	const shards: ShardDescriptor[] = []
-	const counts: Record<SplitName, number> = { train: 0, val: 0, test: 0 }
-	let totalRows = 0
-
-	for (const split of ["train", "val", "test"] as const) {
-		const rows = perSplit[split]
-
-		if (!rows) continue
-
-		let shardIndex = 0
-		let writer: ParquetWriter<ParquetRow> | null = null
-		let path = ""
-		let shardRows = 0
-		let firstSourceID = ""
-		let lastSourceID = ""
-
-		const openShard = async (): Promise<void> => {
-			const splitDir = join(corpusDir, split)
-			await mkdir(splitDir, { recursive: true })
-			path = join(splitDir, `part-${String(shardIndex).padStart(4, "0")}.parquet`)
-
-			writer = await ParquetWriter.openFile<ParquetRow>(LABELED_ROW_SCHEMA, path, {
-				rowGroupSize: ROW_GROUP_SIZE,
-			})
-
-			writer.setMetadata("mailwoman.corpus_version", opts.corpusVersion)
-			writer.setMetadata("mailwoman.split", split)
-			writer.setMetadata("mailwoman.shard_index", String(shardIndex))
-			shardRows = 0
-			firstSourceID = ""
-			lastSourceID = ""
-		}
-
-		const closeShard = async (): Promise<void> => {
-			if (!writer) return
-			await writer.close()
-
-			if (shardRows > 0) {
-				const fileStat = await stat(path)
-				const sha256 = await sha256File(path)
-
-				shards.push({
-					split,
-					path,
-					format: "parquet",
-					compression: SHARD_COMPRESSION,
-					rows: shardRows,
-					bytes: fileStat.size,
-					sha256,
-					first_source_id: firstSourceID,
-					last_source_id: lastSourceID,
-				})
-			}
-
-			writer = null
-		}
-
-		for await (const row of rows) {
-			if (!writer) {
-				await openShard()
-			}
-
-			const pq = rowToParquet(row)
-			await writer!.appendRow(appendShape(pq))
-
-			if (shardRows === 0) {
-				firstSourceID = row.source_id
-			}
-
-			lastSourceID = row.source_id
-
-			shardRows++
-
-			counts[split]++
-
-			totalRows++
-
-			if (shardRows >= rowsPerShard) {
-				await closeShard()
-
-				shardIndex++
-			}
-		}
-
-		await closeShard()
-	}
-
-	shards.sort((a, b) => (a.split === b.split ? a.path.localeCompare(b.path) : a.split.localeCompare(b.split)))
-
-	const manifest: ShardManifest = {
-		corpus_version: opts.corpusVersion,
-		schema: PARQUET_COLUMNS,
-		rows_per_shard: rowsPerShard,
-		row_group_size: ROW_GROUP_SIZE,
-		shards,
-		counts,
-		total_rows: totalRows,
-	}
-
-	await writeFile(join(corpusDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
-
-	return manifest
-}

package/src/utils/split.ts

@@ -1,279 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Train / val / test split with **locality holdout** per the Phase 1 plan.
- *
- *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
- *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
- *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
- *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
- *   memorization, not by learning the underlying schema.
- *
- *   Phase 1 holdouts (chosen for low data density + administrative isolation):
- *
- *   - **US**: Vermont, Wyoming, North Dakota
- *   - **FR**: Corse, Lozère, Creuse
- *
- *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
- *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
- *   the locality boundary, not the exact split percentages.
- *
- *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
- *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
- */
-
-import { spawn } from "node:child_process"
-import { createHash } from "node:crypto"
-import { createWriteStream } from "node:fs"
-import { mkdir, unlink, writeFile } from "node:fs/promises"
-import { join } from "node:path"
-
-import { childEnv } from "@mailwoman/core/scripting/utils"
-import { JSONSpliterator } from "spliterator"
-
-import type { CanonicalRow, LabeledRow } from "#types"
-
-export type SplitName = "train" | "val" | "test"
-
-export interface SplitOptions {
-	/**
-	 * Region-name → holdout policy, keyed by ISO 3166-1 alpha-2 country. The values are the region-component strings the
-	 * splitter looks for in `row.components.region`. Override to change the holdout for an experiment; defaults to
-	 * `defaultHoldouts()`.
-	 */
-	holdouts?: Record<string, readonly string[]>
-}
-
-/**
- * Output manifest: source_id lists per split.
- */
-export interface SplitManifest {
-	train: string[]
-	val: string[]
-	test: string[]
-	/**
-	 * Echoes the holdouts used, so the manifest is self-describing.
-	 */
-	holdouts: Record<string, readonly string[]>
-	/**
-	 * Corpus version stamped onto the manifest. Read from the first row.
-	 */
-	corpus_version: string
-	/**
-	 * Counts for quick sanity checks.
-	 */
-	counts: { train: number; val: number; test: number; total: number }
-}
-
-/**
- * Phase 1 default holdouts (per plan).
- *
- * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
- * - FR: Corse, Lozère, Creuse (small departments / regions).
- * - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
- *   the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
- *   shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
- *   their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
- */
-export function defaultHoldouts(): Record<string, readonly string[]> {
-	return {
-		US: ["Vermont", "VT", "Wyoming", "WY", "North Dakota", "ND"],
-		FR: ["Corse", "Lozère", "Lozere", "Creuse"],
-		DE: ["Saarland", "SL", "Mecklenburg-Vorpommern", "MV"],
-	}
-}
-
-type SplitInputRow = Pick<CanonicalRow, "source_id" | "country" | "corpus_version" | "components">
-
-/**
- * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
- * (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
- * array-based path so the decision is stable regardless of caller.
- */
-export function splitForRow(
-	row: Pick<SplitInputRow, "source_id" | "country" | "components">,
-	holdouts: Record<string, readonly string[]> = defaultHoldouts()
-): SplitName {
-	const region = row.components.region
-	const countryHoldouts = holdouts[row.country] ?? []
-	const isHeldOut = region !== undefined && countryHoldouts.includes(region)
-
-	if (!isHeldOut) return "train"
-
-	// 50/50 deterministic by source_id hash. Same input always lands in the same split.
-	return hashBucket(row.source_id, 2) === 0 ? "val" : "test"
-}
-
-/**
- * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
- * `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
- *
- * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
- * streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
- * split membership in heap.
- */
-export function splitRows(rows: Iterable<SplitInputRow>, opts: SplitOptions = {}): SplitManifest {
-	const holdouts = opts.holdouts ?? defaultHoldouts()
-	const train: string[] = []
-	const val: string[] = []
-	const test: string[] = []
-	let corpus_version = ""
-
-	for (const row of rows) {
-		if (!corpus_version && row.corpus_version) {
-			corpus_version = row.corpus_version
-		}
-
-		const split = splitForRow(row, holdouts)
-
-		if (split === "train") {
-			train.push(row.source_id)
-		} else if (split === "val") {
-			val.push(row.source_id)
-		} else {
-			test.push(row.source_id)
-		}
-	}
-
-	const total = train.length + val.length + test.length
-
-	return {
-		train,
-		val,
-		test,
-		holdouts,
-		corpus_version,
-		counts: { train: train.length, val: val.length, test: test.length, total },
-	}
-}
-
-/**
- * Lightweight deterministic 0..(n-1) bucket from a string id.
- */
-/**
- * Deterministic bucket for a stable id.
- *
- * Stays on raw `createHash` rather than `sha256Hex` from `@mailwoman/core/utils`: it needs the digest BYTES, and the
- * shared helper returns hex. Re-parsing hex back into bytes to reach the same four octets would cost more than the one
- * line it saves.
- */
-export function hashBucket(id: string, n: number): number {
-	const digest = createHash("sha256").update(id).digest()
-	// Read 4 bytes as uint32 to avoid bigint overhead.
-	const u = digest[0]! * 0x01_00_00_00 + digest[1]! * 0x01_00_00 + digest[2]! * 0x01_00 + digest[3]!
-
-	return u % n
-}
-
-/**
- * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
- * id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
- * including holdouts + counts + corpus version.
- *
- * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
- */
-export async function writeSplitManifests(manifest: SplitManifest, outputDir: string): Promise<void> {
-	await mkdir(outputDir, { recursive: true })
-
-	for (const name of ["train", "val", "test"] as const) {
-		const sorted = [...manifest[name]].toSorted()
-		await writeFile(join(outputDir, `${name}.txt`), sorted.join("\n") + (sorted.length ? "\n" : ""), "utf8")
-	}
-
-	const summary = {
-		corpus_version: manifest.corpus_version,
-		holdouts: manifest.holdouts,
-		counts: manifest.counts,
-	}
-
-	await writeFile(join(outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8")
-}
-
-/**
- * Type re-export for callers that want to ingest LabeledRow specifically.
- */
-export type SplitInputLabeledRow = Pick<LabeledRow, "source_id" | "country" | "corpus_version" | "components">
-
-/**
- * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
- * streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
- * handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
- *
- * Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
- * via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
- */
-export async function writeSplitManifestsFromLabeledFiles(opts: {
-	labeledPaths: Record<SplitName, string>
-	outputDir: string
-	corpusVersion: string
-	counts: Record<SplitName, number>
-	holdouts?: Record<string, readonly string[]>
-}): Promise<SplitManifest["counts"]> {
-	await mkdir(opts.outputDir, { recursive: true })
-	const holdouts = opts.holdouts ?? defaultHoldouts()
-
-	for (const split of ["train", "val", "test"] as const) {
-		const labeledPath = opts.labeledPaths[split]
-		const outPath = join(opts.outputDir, `${split}.txt`)
-		await streamSortedSourceIDs(labeledPath, outPath)
-	}
-
-	const total = opts.counts.train + opts.counts.val + opts.counts.test
-
-	const summary = {
-		corpus_version: opts.corpusVersion,
-		holdouts,
-		counts: { ...opts.counts, total },
-	}
-
-	await writeFile(join(opts.outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8")
-
-	return summary.counts
-}
-
-/**
- * Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input → empty output file (not
- * absent). Uses `sort(1)` for disk-spilling external sort so peak memory stays O(1) regardless of labeled-row count.
- */
-async function streamSortedSourceIDs(labeledJsonlPath: string, outPath: string): Promise<void> {
-	const unsortedPath = `${outPath}.unsorted`
-	const out = createWriteStream(unsortedPath, { encoding: "utf8" })
-
-	const outClosed = new Promise<void>((resolve, reject) => {
-		out.on("close", () => resolve())
-		out.on("error", reject)
-	})
-
-	// JSONSpliterator parses each row (skipEmpty drops blank lines); a malformed row throws
-	// SyntaxError out of the loop, matching the prior `reject(err)` fail-loud behavior. `finally`
-	// always ends the write stream so `sort` reads a complete file even if the read throws.
-	try {
-		for await (const obj of JSONSpliterator.fromAsync<{ source_id?: string }>(labeledJsonlPath)) {
-			if (typeof obj.source_id === "string") {
-				out.write(`${obj.source_id}\n`)
-			}
-		}
-	} finally {
-		out.end()
-	}
-
-	await outClosed
-
-	await new Promise<void>((resolve, reject) => {
-		// LC_ALL=C: byte-sort, locale-independent → deterministic across hosts.
-		const proc = spawn("sort", [unsortedPath, "-o", outPath], { env: childEnv({ LC_ALL: "C" }) })
-		proc.on("error", reject)
-
-		proc.on("exit", (code) => {
-			if (code === 0) {
-				resolve()
-			} else {
-				reject(new Error(`sort exited with code ${code}`))
-			}
-		})
-	})
-
-	await unlink(unsortedPath).catch(() => {})
-}