@mailwoman/corpus 7.2.0 → 7.2.1

package/src/license.ts

@@ -0,0 +1,48 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Corpus licensing — the single source of truth for the training-data license policy (#26).
+ *
+ *   Posture (operator, 2026-06-19): **exclusion is a deliberate act, not a silent default.** The
+ *   build INCLUDES every row an adapter yields (stamping its `license`); a build that needs a clean
+ *   license set — e.g. the proprietary `@mailwoman/neural-weights-*` weights, which must not
+ *   inherit a share-alike obligation — PURPOSELY excludes kinds via `buildCorpus({ excludeLicenses
+ *   })` (CLI `--exclude-licenses` / `--exclude-share-alike`). Nothing is dropped on a license
+ *   string unless the operator named it. This avoids the trap of silently dropping allowed data
+ *   mis-stamped with a conservative license (e.g. BAN, which is dual-licensed Licence Ouverte OR
+ *   ODbL — we elect Licence Ouverte; a default-deny on the old `ODbL` stamp would have wrongly
+ *   dropped 48M allowed rows).
+ *
+ *   Tier reference (#26): A = PD/CC0 (allowed); B = CC-BY / Licence Ouverte (allowed WITH attribution
+ *   — the model card must carry it); C = share-alike (ODbL, CC-BY-SA, CC-SA) — exclude for a
+ *   proprietary-weights build via `--exclude-share-alike`.
+ */
+
+/**
+ * Licenses that require share-alike / create a copyleft obligation on derived works (Tier C). The
+ * `--exclude-share-alike` convenience expands to this; `allowShareAlike: false` adapters also use it.
+ */
+export const SHARE_ALIKE_PATTERN = /^ODbL|^Open Database License|^CC-BY-SA|^CC-SA/i
+
+/**
+ * Compile a `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored, case-insensitive prefix
+ * patterns. Each entry matches a license string that STARTS with it, so `CC-BY-SA` catches `CC-BY-SA-3.0`,
+ * `CC-BY-SA-4.0`, etc. Regex metacharacters are escaped — the spec is a literal license prefix, not a user-supplied
+ * regex.
+ */
+export function compileLicenseExcludes(spec: string): RegExp[] {
+	return spec
+		.split(",")
+		.map((s) => s.trim())
+		.filter(Boolean)
+		.map((s) => new RegExp("^" + s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"))
+}
+
+/** True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded). */
+export function licenseExcluded(license: string | undefined, patterns: readonly RegExp[]): boolean {
+	const l = license ?? ""
+
+	return patterns.some((p) => p.test(l))
+}

package/src/parquet-wrapper/index.ts

@@ -0,0 +1,9 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+export * from "./reader.ts"
+export * from "./schema.ts"
+export * from "./writer.ts"

package/src/parquet-wrapper/reader.ts

@@ -0,0 +1,67 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed wrapper around `@dsnp/parquetjs`'s `ParquetReader` that narrows the row-iterator generic to
+ *   a user-supplied record type and adds `AsyncDisposable` support so `await using` cleans up the
+ *   envelope reader without an explicit `close()`.
+ */
+
+import { ParquetReader as BaseParquetReader } from "@dsnp/parquetjs"
+import type { BufferReaderOptions } from "@dsnp/parquetjs/dist/lib/bufferReader.js"
+import { ParquetEnvelopeReader } from "@dsnp/parquetjs/dist/lib/reader.js"
+
+import { type ParquetRecordLike, ParquetSchema } from "./schema.ts"
+
+/** A typed Parquet reader, wrapping the base Parquet reader. */
+export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReader implements AsyncDisposable {
+	declare schema: ParquetSchema<T>
+
+	static override async openFile<T extends ParquetRecordLike>(
+		filePath: string | URL,
+		options?: BufferReaderOptions
+	): Promise<ParquetReader<T>> {
+		const envelopeReader = await ParquetEnvelopeReader.openFile(filePath.toString(), options)
+
+		return ParquetReader.openEnvelopeReader<T>(envelopeReader, options)
+	}
+
+	static override async openBuffer<T extends ParquetRecordLike>(buffer: Buffer, options?: BufferReaderOptions) {
+		const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer, options)
+
+		return this.openEnvelopeReader<T>(envelopeReader, options)
+	}
+
+	static override async openEnvelopeReader<T extends ParquetRecordLike>(
+		envelopeReader: ParquetEnvelopeReader,
+		opts?: BufferReaderOptions
+	) {
+		if (opts?.metadata) {
+			return new ParquetReader<T>(opts.metadata, envelopeReader, opts)
+		}
+
+		try {
+			await envelopeReader.readHeader()
+
+			const metadata = await envelopeReader.readFooter()
+
+			return new ParquetReader<T>(metadata, envelopeReader, opts)
+		} catch (err) {
+			await envelopeReader.close()
+			throw err
+		}
+	}
+
+	public override [Symbol.asyncIterator](): AsyncGenerator<T, void, unknown> {
+		return super[Symbol.asyncIterator]() as AsyncGenerator<T, void, unknown>
+	}
+
+	public async [Symbol.asyncDispose]() {
+		return this.close()
+	}
+
+	public async dispose() {
+		return this[Symbol.asyncDispose]()
+	}
+}

package/src/parquet-wrapper/schema.ts

@@ -0,0 +1,80 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed wrapper around `@dsnp/parquetjs`'s schema definition. Adds:
+ *
+ *   - `ParquetSchema<T>`: a generic class narrowing the base schema's `schema` property to a
+ *       field-by-field typed dict.
+ *   - `ParquetSchemaDefinitionCache`: an LRU lookup so hot paths that compute the same schema
+ *       repeatedly pay the cost once. Implements `Disposable` so `using` works.
+ *   - `createBloomFilters<T>`: helper that takes a schema and a list of columns and returns the
+ *       `@dsnp/parquetjs`-shaped bloom-filter spec array.
+ *
+ *   `Symbol.dispose` is sync (the original was async, but `Disposable`'s contract is sync — async
+ *   cleanup belongs on `AsyncDisposable`).
+ */
+
+import { ParquetSchema as BaseParquetSchema } from "@dsnp/parquetjs"
+import type { createSBBFParams as BloomFilterCreation } from "@dsnp/parquetjs/dist/lib/bloomFilterIO/bloomFilterWriter.js"
+import type { FieldDefinition } from "@dsnp/parquetjs/dist/lib/declare.js"
+import { LRUCache } from "lru-cache"
+
+/** A Parquet record-like object, i.e. a record with string keys and JSON-serializable values. */
+export type ParquetRecordLike = {
+	[key: string]: unknown | undefined
+}
+
+/** Typed Parquet schema definition. */
+export type ParquetSchemaDefinition<T = ParquetRecordLike> = {
+	[field in Extract<keyof T, string>]: FieldDefinition
+}
+
+/** Typed Parquet schema. */
+export class ParquetSchema<T> extends BaseParquetSchema {
+	declare schema: ParquetSchemaDefinition<T>
+}
+
+/** Given a Parquet schema and a list of columns, create a list of Bloom filters for those columns. */
+export function createBloomFilters<T>(
+	parquetSchemaDef: ParquetSchemaDefinition<T>,
+	columns: Extract<keyof T, string>[]
+) {
+	const bloomFilters: BloomFilterCreation[] = []
+
+	for (const column of columns) {
+		if (!parquetSchemaDef[column]) {
+			throw new Error(`Bloom filter column ${column} not found in Parquet schema`)
+		}
+
+		bloomFilters.push({ column })
+	}
+
+	return bloomFilters
+}
+
+export class ParquetSchemaDefinitionCache
+	extends LRUCache<ParquetSchemaDefinition<ParquetRecordLike>, ParquetSchema<ParquetRecordLike>>
+	implements Disposable
+{
+	constructor(max = 1000) {
+		super({ max })
+	}
+
+	public findOrCreateSchema<T extends ParquetRecordLike>(schemaDef: ParquetSchemaDefinition<T>): ParquetSchema<T> {
+		const key = schemaDef as ParquetSchemaDefinition<ParquetRecordLike>
+		let schema = this.get(key) as ParquetSchema<T> | undefined
+
+		if (!schema) {
+			schema = new ParquetSchema<T>(schemaDef)
+			this.set(key, schema as ParquetSchema<ParquetRecordLike>)
+		}
+
+		return schema
+	}
+
+	public [Symbol.dispose]() {
+		this.clear()
+	}
+}

package/src/parquet-wrapper/writer.ts

@@ -0,0 +1,102 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed Parquet writer. Two static constructors mirror the base class:
+ *
+ *   - `openStream`: wrap an existing writable stream.
+ *   - `openFile`: open a path on disk, ensuring the parent directory exists first.
+ *
+ *   Implements `AsyncDisposable` so `await using writer = await ParquetWriter.openFile(...)` flushes
+ *   and closes cleanly. `close()` internally serializes against any in-flight flush so back-to-back
+ *   dispose calls don't race.
+ */
+
+import * as fs from "node:fs/promises"
+import * as path from "node:path"
+
+import { ParquetWriter as BaseParquetWriter } from "@dsnp/parquetjs"
+import type { WriterOptions } from "@dsnp/parquetjs/dist/lib/declare.js"
+import { osopen, type WriteStreamMinimal } from "@dsnp/parquetjs/dist/lib/util.js"
+import { ParquetEnvelopeWriter } from "@dsnp/parquetjs/dist/lib/writer.js"
+
+import {
+	type ParquetRecordLike,
+	ParquetSchema,
+	type ParquetSchemaDefinition,
+	ParquetSchemaDefinitionCache,
+} from "./schema.ts"
+
+/**
+ * A typed Parquet writer, wrapping the base Parquet writer.
+ */
+export class ParquetWriter<T extends ParquetRecordLike> extends BaseParquetWriter implements AsyncDisposable {
+	declare schema: ParquetSchema<T>
+	protected static readonly SchemaDefinitionCache = new ParquetSchemaDefinitionCache()
+	#flushing: Promise<void> = Promise.resolve()
+
+	static override async openStream<T extends ParquetRecordLike>(
+		schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>,
+		outputStream: WriteStreamMinimal,
+		opts: WriterOptions = {}
+	): Promise<ParquetWriter<T>> {
+		const schema =
+			schemaLike instanceof ParquetSchema
+				? schemaLike
+				: ParquetWriter.SchemaDefinitionCache.findOrCreateSchema(schemaLike)
+
+		const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts)
+
+		return new ParquetWriter<T>(schema, envelopeWriter, opts)
+	}
+
+	/** Convenience method to create a new buffered parquet writer that writes to the specified file. */
+	static override async openFile<T extends ParquetRecordLike>(
+		schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>,
+		sourcePath: string | Buffer | URL,
+		opts?: WriterOptions
+	): Promise<ParquetWriter<T>> {
+		if (typeof sourcePath === "string") {
+			await fs.mkdir(path.dirname(sourcePath), { recursive: true })
+		}
+
+		const outputStream = await osopen(sourcePath, opts)
+
+		return ParquetWriter.openStream<T>(schemaLike, outputStream, opts)
+	}
+
+	// @note This fixes invalid Markdown in the base class JSDoc.
+	/** Set a metadata key-value pair on the writer. */
+	public override setMetadata(key: string, value: string): void {
+		return super.setMetadata(key, value)
+	}
+
+	/** Append a row to the buffer. If the buffer is full, the data will be written to disk. */
+	public override async appendRow(row: T): Promise<void> {
+		return super.appendRow(row)
+	}
+
+	/** Flush all buffered data to disk, close the file, and release resources. */
+	public override async close(): Promise<void> {
+		await this.#flushing
+
+		if (this.closed) return
+
+		const { promise, resolve, reject } = Promise.withResolvers<void>()
+
+		super.close().then(resolve, reject)
+
+		this.#flushing = promise
+
+		return this.#flushing
+	}
+
+	public async [Symbol.asyncDispose]() {
+		return this.close()
+	}
+
+	public async dispose() {
+		return this[Symbol.asyncDispose]()
+	}
+}

package/src/parquet.ts

@@ -0,0 +1,360 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Final output sharder for the corpus pipeline.
+ *
+ *   Phase 1 (#9) shipped JSONL shards + a Python (PyArrow) converter as the path to binary Parquet —
+ *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
+ *   writer. The build pipeline no longer touches Python at all in its hot path; the only remaining
+ *   Python is the one-shot `train_tokenizer.py` SentencePiece step.
+ *
+ *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`, but `@dsnp/parquetjs` 1.7.0 only
+ *   supports UNCOMPRESSED / GZIP / SNAPPY / BROTLI (see `node_modules/@dsnp/parquetjs/dist/lib/
+ *   compression.js`). SNAPPY is the standard ML-corpus default (PyArrow's default too) and is the
+ *   closest substitute on speed; revisit if @dsnp/parquetjs gains zstd support. Documented in
+ *   `DECISIONS.md`.
+ *
+ *   Layout under `<outputDir>`:
+ *
+ *   ```
+ *   corpus-v<version>/
+ *   MANIFEST.json
+ *   train/
+ *     part-0000.parquet
+ *     part-0001.parquet
+ *     ...
+ *   val/
+ *     part-0000.parquet
+ *   test/
+ *     part-0000.parquet
+ * ```
+ *
+ *   Each shard caps at `rowsPerShard` (default 1_000_000); within a shard, parquetjs flushes row
+ *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
+ *   shard's path, row count, byte size, and SHA-256 (computed by re-reading the shard once after
+ *   close — cheap relative to writing it).
+ */
+
+import { createHash } from "node:crypto"
+import { createReadStream } from "node:fs"
+import { mkdir, stat, writeFile } from "node:fs/promises"
+import { join } from "node:path"
+
+import { ParquetWriter, type ParquetSchemaDefinition } from "./parquet-wrapper/index.ts"
+import type { SplitName } from "./split.ts"
+import type { LabeledRow } from "./types.ts"
+
+/** Row groups flush at this many rows (parquetjs internal cadence within a shard). */
+export const ROW_GROUP_SIZE = 50_000
+
+/** Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0. */
+export const SHARD_COMPRESSION = "SNAPPY" as const
+
+/**
+ * A single Parquet-style row shape. The `[key: string]: unknown` index signature is required for compatibility with
+ * `ParquetRecordLike` in the wrapper — parquetjs accepts any string key on rows.
+ */
+export interface ParquetRow {
+	raw: string
+	tokens: readonly string[]
+	labels: readonly string[]
+	span_starts: readonly number[]
+	span_ends: readonly number[]
+	span_tags: readonly string[]
+	country: string
+	locale: string | null
+	source: string
+	source_id: string
+	corpus_version: string
+	license: string
+	synth_method: string | null
+	synth_base_id: string | null
+	[key: string]: unknown
+}
+
+/** Column names emitted into every shard. Matches `ParquetRow`. */
+export const PARQUET_COLUMNS = [
+	"raw",
+	"tokens",
+	"labels",
+	"span_starts",
+	"span_ends",
+	"span_tags",
+	"country",
+	"locale",
+	"source",
+	"source_id",
+	"corpus_version",
+	"license",
+	"synth_method",
+	"synth_base_id",
+] as const
+
+/**
+ * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
+ * tokens/labels arrays. Compression is per-column SNAPPY.
+ */
+export const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow> = {
+	raw: { type: "UTF8", compression: SHARD_COMPRESSION },
+	tokens: { type: "UTF8", repeated: true, compression: SHARD_COMPRESSION },
+	labels: { type: "UTF8", repeated: true, compression: SHARD_COMPRESSION },
+	// v0.5.0 char-offset label spans (#519): parallel arrays over `raw` (UTF-16 code units,
+	// [start, end) exclusive-end, sorted, non-overlapping). INT32 — raw is a short address string,
+	// and INT32 round-trips as `number` where parquetjs INT64 would surface bigint.
+	span_starts: { type: "INT32", repeated: true, compression: SHARD_COMPRESSION },
+	span_ends: { type: "INT32", repeated: true, compression: SHARD_COMPRESSION },
+	span_tags: { type: "UTF8", repeated: true, compression: SHARD_COMPRESSION },
+	country: { type: "UTF8", compression: SHARD_COMPRESSION },
+	locale: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
+	source: { type: "UTF8", compression: SHARD_COMPRESSION },
+	source_id: { type: "UTF8", compression: SHARD_COMPRESSION },
+	corpus_version: { type: "UTF8", compression: SHARD_COMPRESSION },
+	license: { type: "UTF8", compression: SHARD_COMPRESSION },
+	synth_method: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
+	synth_base_id: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
+}
+
+/** Per-shard metadata captured in `MANIFEST.json`. */
+export interface ShardDescriptor {
+	split: SplitName
+	path: string
+	format: "parquet"
+	compression: typeof SHARD_COMPRESSION
+	rows: number
+	bytes: number
+	sha256: string
+	first_source_id: string
+	last_source_id: string
+}
+
+export interface ShardManifest {
+	corpus_version: string
+	schema: readonly string[]
+	rows_per_shard: number
+	row_group_size: number
+	shards: ShardDescriptor[]
+	counts: Record<SplitName, number>
+	total_rows: number
+}
+
+export interface WriteShardsOptions {
+	/** Root output directory; corpus version dir is created beneath. */
+	outputDir: string
+
+	/** Corpus version stamped onto rows + into the output directory name. */
+	corpusVersion: string
+
+	/** Max rows per `.parquet` shard. Default 1_000_000 per the Phase 1 plan. */
+	rowsPerShard?: number
+}
+
+/**
+ * Pre-partitioned labeled-row streams, one per split. Callers (`buildCorpus`) decide each row's split inline at align
+ * time via `splitForRow` and route rows to the matching stream, eliminating the prior `Map<source_id, SplitName>` O(n)
+ * lookup table.
+ *
+ * Splits with no rows can be omitted (or passed as an empty iterable); `writeShards` skips them.
+ */
+export type PerSplitRows = Partial<Record<SplitName, AsyncIterable<LabeledRow>>>
+
+/**
+ * Project a labeled row to the Parquet schema.
+ *
+ * The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
+ * from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the shard (the "builders
+ * before parquet = silent loss" hazard). Loud failure, naming the row, instead.
+ */
+export function rowToParquet(row: LabeledRow): ParquetRow {
+	const { span_starts, span_ends, span_tags } = row
+
+	if (span_starts === undefined || span_ends === undefined || span_tags === undefined) {
+		throw new Error(
+			`rowToParquet: row is missing the char-offset span triple (#519) — ` +
+				`span_starts=${span_starts !== undefined} span_ends=${span_ends !== undefined} span_tags=${span_tags !== undefined} ` +
+				`(source=${row.source}, source_id=${row.source_id}). ` +
+				`Every parquet-bound row must carry span_starts/span_ends/span_tags; ` +
+				`producers that emit tokens/labels only have not migrated to the v0.5.0 format.`
+		)
+	}
+
+	if (span_starts.length !== span_ends.length || span_starts.length !== span_tags.length) {
+		throw new Error(
+			`rowToParquet: span triple arrays are not parallel — ` +
+				`starts=${span_starts.length} ends=${span_ends.length} tags=${span_tags.length} ` +
+				`(source=${row.source}, source_id=${row.source_id})`
+		)
+	}
+
+	return {
+		raw: row.raw,
+		tokens: row.tokens,
+		labels: row.labels,
+		span_starts,
+		span_ends,
+		span_tags,
+		country: row.country,
+		locale: row.locale ?? null,
+		source: row.source,
+		source_id: row.source_id,
+		corpus_version: row.corpus_version,
+		license: row.license,
+		synth_method: row.synth?.method ?? null,
+		synth_base_id: row.synth?.base_source_id ?? null,
+	}
+}
+
+/**
+ * Project a `ParquetRow` for `appendRow`. parquetjs treats `null` as "skip" for `optional` columns; passing it
+ * explicitly is fine, but cleaner to omit so the on-disk Definition Levels match what PyArrow / DuckDB / etc. produce
+ * for the same logical row.
+ */
+function appendShape(row: ParquetRow): Record<string, unknown> {
+	const out: Record<string, unknown> = {
+		raw: row.raw,
+		tokens: row.tokens,
+		labels: row.labels,
+		span_starts: row.span_starts,
+		span_ends: row.span_ends,
+		span_tags: row.span_tags,
+		country: row.country,
+		source: row.source,
+		source_id: row.source_id,
+		corpus_version: row.corpus_version,
+		license: row.license,
+	}
+
+	if (row.locale !== null) {
+		out.locale = row.locale
+	}
+
+	if (row.synth_method !== null) {
+		out.synth_method = row.synth_method
+	}
+
+	if (row.synth_base_id !== null) {
+		out.synth_base_id = row.synth_base_id
+	}
+
+	return out
+}
+
+/**
+ * Stream labeled rows into `.parquet` shards, one set of shards per split. Splits are processed sequentially so that
+ * only one shard writer is open at a time — memory cost is bounded by the parquetjs row-group buffer (~`ROW_GROUP_SIZE
+ * × row_size`), not by the labeled-row count.
+ *
+ * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
+ * because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
+ * for the new wire-up.
+ */
+export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptions): Promise<ShardManifest> {
+	const rowsPerShard = opts.rowsPerShard ?? 1_000_000
+	const corpusDir = join(opts.outputDir, `corpus-v${opts.corpusVersion}`)
+	await mkdir(corpusDir, { recursive: true })
+
+	const shards: ShardDescriptor[] = []
+	const counts: Record<SplitName, number> = { train: 0, val: 0, test: 0 }
+	let totalRows = 0
+
+	for (const split of ["train", "val", "test"] as const) {
+		const rows = perSplit[split]
+
+		if (!rows) continue
+
+		let shardIndex = 0
+		let writer: ParquetWriter<ParquetRow> | null = null
+		let path = ""
+		let shardRows = 0
+		let firstSourceID = ""
+		let lastSourceID = ""
+
+		const openShard = async (): Promise<void> => {
+			const splitDir = join(corpusDir, split)
+			await mkdir(splitDir, { recursive: true })
+			path = join(splitDir, `part-${String(shardIndex).padStart(4, "0")}.parquet`)
+			writer = await ParquetWriter.openFile<ParquetRow>(LABELED_ROW_SCHEMA, path, {
+				rowGroupSize: ROW_GROUP_SIZE,
+			})
+			writer.setMetadata("mailwoman.corpus_version", opts.corpusVersion)
+			writer.setMetadata("mailwoman.split", split)
+			writer.setMetadata("mailwoman.shard_index", String(shardIndex))
+			shardRows = 0
+			firstSourceID = ""
+			lastSourceID = ""
+		}
+
+		const closeShard = async (): Promise<void> => {
+			if (!writer) return
+			await writer.close()
+
+			if (shardRows > 0) {
+				const fileStat = await stat(path)
+				const sha256 = await hashFile(path)
+				shards.push({
+					split,
+					path,
+					format: "parquet",
+					compression: SHARD_COMPRESSION,
+					rows: shardRows,
+					bytes: fileStat.size,
+					sha256,
+					first_source_id: firstSourceID,
+					last_source_id: lastSourceID,
+				})
+			}
+			writer = null
+		}
+
+		for await (const row of rows) {
+			if (!writer) {
+				await openShard()
+			}
+			const pq = rowToParquet(row)
+			await writer!.appendRow(appendShape(pq) as unknown as ParquetRow)
+
+			if (shardRows === 0) {
+				firstSourceID = row.source_id
+			}
+			lastSourceID = row.source_id
+			shardRows++
+			counts[split]++
+			totalRows++
+
+			if (shardRows >= rowsPerShard) {
+				await closeShard()
+				shardIndex++
+			}
+		}
+
+		await closeShard()
+	}
+
+	shards.sort((a, b) => (a.split === b.split ? a.path.localeCompare(b.path) : a.split.localeCompare(b.split)))
+
+	const manifest: ShardManifest = {
+		corpus_version: opts.corpusVersion,
+		schema: PARQUET_COLUMNS,
+		rows_per_shard: rowsPerShard,
+		row_group_size: ROW_GROUP_SIZE,
+		shards,
+		counts,
+		total_rows: totalRows,
+	}
+	await writeFile(join(corpusDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
+
+	return manifest
+}
+
+/** Single-pass SHA-256 over the file at `path`. Cheap relative to Parquet write throughput. */
+async function hashFile(path: string): Promise<string> {
+	const hash = createHash("sha256")
+	const stream = createReadStream(path)
+
+	for await (const chunk of stream) {
+		hash.update(chunk as Buffer)
+	}
+
+	return hash.digest("hex")
+}