@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/build.ts

@@ -2,377 +2,189 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
+ * @file The corpus build, as the four phases it runs in order: adapters, align, split, parquet.
  *
- *   End-to-end corpus build (Phase 1 task #10 in the plan).
- *
- *   `buildCorpus(opts)` orchestrates every stage of the pipeline:
- *
- *   1. **Adapter runs** — drives every adapter in turn (via `runAdapter`), writing
- *        `<intermediate>/<adapter.id>/canonical.jsonl` slices.
- *   2. **Synthesis** — optional. For each canonical row, every applicable augmentation in the row's
- *        country-default policy emits an augmented row alongside the original.
- *   3. **Alignment** — every row (original + augmented) is aligned via `alignRow`. Successes go to
- *        `labeled.jsonl`; quarantines are appended to `quarantine.jsonl` with reasons.
- *   4. **Splits** — `splitRows` partitions labeled `source_id`s into train/val/test by locality holdout.
- *        Manifest written to `splits/SPLIT_MANIFEST.json` + per-split `train.txt` / `val.txt` /
- *        `test.txt`.
- *   5. **Parquet slices** — `writeSlices` streams labeled rows into 1M-row `.parquet` slices per split
- *        under `corpus-v<version>/{train,val,test}/part-NNNN.parquet` (SNAPPY-compressed, 50k-row
- *        row groups), with per-slice checksums + per-stage manifest in
- *        `corpus-v<version>/MANIFEST.json`.
- *   6. **Top-level manifest** — `<outputDir>/MANIFEST.json` ties every per-stage manifest together with
- *        a top-level corpus_version, built_at, and aggregate counts.
- *
- *   Output layout:
- *
- *   ```
- *   <outputDir>/
- *   MANIFEST.json
- *   intermediate/
- *     <adapter.id>/canonical.jsonl   # one per adapter
- *     labeled.jsonl                  # post-alignment, pre-slice
- *     quarantine.jsonl               # rows that failed alignment
- *   splits/
- *     SPLIT_MANIFEST.json
- *     train.txt / val.txt / test.txt
- *   corpus-v<version>/
- *     MANIFEST.json
- *     train/part-NNNN.parquet
- *     val/part-NNNN.parquet
- *     test/part-NNNN.parquet
- * ```
- *
- *   The intermediate files live alongside the final slices for reproducibility + debugging. Operators
- *   can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
- *   self-contained.
+ *   Each phase is its own module. This module defines their sequence and shared paths.
+ *   It assembles the manifest from the phase reports.
  */
 
-import { pathExists, readLocalJSONFile } from "@mailwoman/core/fs/readers"
-import { openWriteStream, type WriteStream } from "@mailwoman/core/fs/streams"
-import { writeLocalJSONFile, makeDirectories } from "@mailwoman/core/fs/writers"
-import { join } from "path-ts"
-import { JSONSpliterator } from "spliterator"
-
-import { defaultAdapterRegistry } from "#adapters/utils"
+import { makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
+import { heapLimitBytes } from "@mailwoman/core/utils/system"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { defaultAdapterRegistry } from "#adapters/registry"
+import { runAdapterPhase } from "#build/adapters"
+import { runAlignPhase } from "#build/align"
+import { readRegisterDecisions, readSourceEligibility } from "#build/eligibility"
+import { streamJSONL } from "#build/rows"
+import { BuildProfile, TRAINING_MANIFEST_FILE, type BuildCorpusManifest, type BuildCorpusOptions } from "#build/types"
 import { $public } from "#env"
-import { once, runAdapter, type AdapterRunManifest } from "#runner"
-import { defaultAugmentationsForCountry, synthesizeRow } from "#synthesizers/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "#types"
-import { alignRow } from "#utils/align"
-import { licenseExcluded } from "#utils/license"
-import { writeSlices, type SliceManifest } from "#utils/parquet"
-import {
-	defaultHoldouts,
-	splitForRow,
-	writeSplitManifestsFromLabeledFiles,
-	type SplitManifest,
-	type SplitName,
-} from "#utils/split"
-
-/**
- * Stage tags surfaced to `onProgress`.
- */
-export type BuildStage = "adapter-run" | "align" | "split" | "slice" | "manifest"
-
-/**
- * Per-invocation options for `buildCorpus`.
- */
-export interface BuildCorpusOptions {
-	/**
-	 * Root output directory. All build artifacts land beneath it.
-	 */
-	outputDir: string
-
-	/**
-	 * Corpus version (e.g. `"0.1.0"`). Stamped onto every row + into the output dir name.
-	 */
-	corpusVersion: string
-
-	/**
-	 * Adapters to drive, in order. Defaults to `defaultAdapterRegistry.list()`. Pass an explicit list to filter (e.g.
-	 * `[wofAdminAdapter]` for a smoke run).
-	 */
-	adapters?: readonly CorpusAdapter[]
-
-	/**
-	 * Per-adapter `AdapterOptions` — looked up by adapter id. Adapters whose id is missing from this map are skipped (and
-	 * noted in the manifest).
-	 */
-	adapterInputs: Record<string, AdapterOptions>
-
-	/**
-	 * Enable synthesis pass. Default `true`. Set `false` for fixture-driven smoke tests.
-	 */
-	synthesize?: boolean
-
-	/**
-	 * Forwarded to `writeSlices`. Default 1_000_000.
-	 */
-	rowsPerSlice?: number
-
-	/**
-	 * Progress hook. Errors thrown abort the build.
-	 */
-	onProgress?: (stage: BuildStage, message: string) => void
-
-	/**
-	 * License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see `compileLicenseExcludes` /
-	 * `SHARE_ALIKE_PATTERN` in `license.ts`); a row whose `license` matches any is dropped at ingest. Default (omitted)
-	 * includes EVERYTHING — exclusion is a deliberate act, not a silent default. A proprietary-weights build passes the
-	 * share-alike set (`--exclude-share-alike`).
-	 */
-	excludeLicenses?: readonly RegExp[]
-}
+import { ROWS_PER_FILE } from "#parquet/schema"
+import { DEFAULT_SHUFFLE_SEED, DEFAULT_SHUFFLE_WINDOW, shuffleWithinWindow } from "#parquet/shuffle"
+import { writeParquetSplits } from "#parquet/writers"
+import { freezeTrainingManifest } from "#source-register/training-manifest"
+import type { LabeledRow } from "#types"
+import { LicensePolicy } from "#utils/license"
+import { defaultHoldouts, writeSplitManifestsFromLabeledFiles, type SplitName } from "#utils/split"
 
-/**
- * Top-level manifest tying every stage together.
- */
-export interface BuildCorpusManifest {
-	corpus_version: string
-	built_at: string
-	adapters: AdapterRunManifest[]
-	skipped_adapters: string[]
-	splits: { counts: SplitManifest["counts"]; holdouts: SplitManifest["holdouts"] }
-	slices: { counts: SliceManifest["counts"]; total_rows: number }
-	quarantine_count: number
-	total_aligned_rows: number
-	/**
-	 * Resolved license set across all INCLUDED rows (license string → row count), + the count dropped by
-	 * `excludeLicenses` (#26). The model card derives its data-attribution table from `licenses`.
-	 */
-	licenses: Record<string, number>
-	excluded_by_license: number
-}
+export * from "#build/heap"
+export * from "#build/types"
 
 /**
  * Drive the full corpus build to completion.
- *
- * Memory profile: the function maintains an in-memory `Map<source_id, SplitName>` to bridge the align → slice hand-off.
- * For Phase 1 fixture-scale runs (≤ 10⁴ rows) this is trivial. For real 5M+ runs, the map fits comfortably in a few
- * hundred MB; the canonical.jsonl and labeled.jsonl payloads stream and never sit in memory.
  */
 export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpusManifest> {
 	const adapters = opts.adapters ?? defaultAdapterRegistry.list()
 	const synthesize = opts.synthesize ?? true
-	const rowsPerSlice = opts.rowsPerSlice ?? 1_000_000
+	const rowsPerFile = opts.rowsPerFile ?? ROWS_PER_FILE
+	const shuffleWindow = opts.shuffleWindow ?? DEFAULT_SHUFFLE_WINDOW
+	const shuffleSeed = opts.shuffleSeed ?? DEFAULT_SHUFFLE_SEED
+	const profile = opts.profile ?? BuildProfile.Exploratory
+	const licensePolicy = opts.licensePolicy ?? LicensePolicy.All
 	const built_at = new Date().toISOString()
 
-	await makeDirectories(opts.outputDir)
-	const intermediateDir = join(opts.outputDir, "intermediate")
-	await makeDirectories(intermediateDir)
-
-	// 1. Adapter runs.
-	const adapterRuns: AdapterRunManifest[] = []
-	const skipped: string[] = []
-
-	for (const adapter of adapters) {
-		const adapterOptions = opts.adapterInputs[adapter.id]
-
-		if (!adapterOptions) {
-			skipped.push(adapter.id)
-			opts.onProgress?.("adapter-run", `skipped ${adapter.id} (no input configured)`)
-
-			continue
-		}
+	// The build reports heap state when it aborts with SIGABRT.
+	// These aborts arrive twelve minutes into the adapter phase.
+	const heapLimit = heapLimitBytes()
+	const configured = adapters.filter((adapter) => opts.adapterInputs[adapter.id]).map((adapter) => adapter.id)
 
-		// Opt-in resume (MAILWOMAN_RESUME=1): if a complete per-adapter canonical.jsonl + MANIFEST.json
-		// already exist, reuse them instead of re-emitting. The MANIFEST is written only after the
-		// canonical is fully flushed, so its presence guarantees completeness; row order is identical,
-		// so downstream holdout-split determinism is preserved. Recovers an align-phase crash without
-		// redoing the (expensive) emit phase. Default (unset) re-emits, preserving correctness. (2026-06-12.)
-		const adapterDir = join(intermediateDir, adapter.id)
-		const cachedManifest = join(adapterDir, "MANIFEST.json")
-
-		if (
-			$public.MAILWOMAN_RESUME === "1" &&
-			(await pathExists(cachedManifest)) &&
-			(await pathExists(join(adapterDir, "canonical.jsonl")))
-		) {
-			const cached = await readLocalJSONFile<AdapterRunManifest>(cachedManifest)
-			opts.onProgress?.("adapter-run", `resumed ${adapter.id} (reused ${cached.yielded} canonical rows)`)
-			adapterRuns.push(cached)
-
-			continue
-		}
-
-		opts.onProgress?.("adapter-run", `running ${adapter.id}`)
+	opts.onProgress?.(
+		"adapter-run",
+		`heap limit ${(heapLimit / 1024 / 1024 / 1024).toFixed(1)} GiB, ${configured.length} adapters configured`
+	)
 
-		const m = await runAdapter({
-			adapter,
-			adapterOptions,
-			outputDir: intermediateDir,
-			corpusVersion: opts.corpusVersion,
-		})
+	const outputDir = PathBuilder.from(opts.outputDir)
+	const intermediateDir = outputDir("intermediate")
 
-		adapterRuns.push(m)
-	}
+	await makeDirectories(outputDir, intermediateDir)
 
-	// 2 + 3. Synthesis + alignment: stream every canonical.jsonl, optionally augment, align,
-	// and route each labeled row directly to its split-specific JSONL (`labeled-{train,val,test}.
-	// jsonl`). Memory cost is O(1) — the prior in-memory `splitInputs` array + `splitByIDMap`
-	// + `SplitManifest.{train,val,test}` arrays are gone; per-row split is decided inline via
-	// `splitForRow` (a pure function of source_id + region + holdout policy).
-	const labeledPaths: Record<SplitName, string> = {
-		train: join(intermediateDir, "labeled-train.jsonl"),
-		val: join(intermediateDir, "labeled-val.jsonl"),
-		test: join(intermediateDir, "labeled-test.jsonl"),
-	}
+	const { runs: adapterRuns, skipped } = await runAdapterPhase({
+		adapters,
+		adapterInputs: opts.adapterInputs,
+		corpusVersion: opts.corpusVersion,
+		intermediateDir,
+		onProgress: opts.onProgress,
+	})
 
-	const labeledStreams: Record<SplitName, WriteStream> = {
-		train: openWriteStream(labeledPaths.train, { encoding: "utf8" }),
-		val: openWriteStream(labeledPaths.val, { encoding: "utf8" }),
-		test: openWriteStream(labeledPaths.test, { encoding: "utf8" }),
+	const labeledPaths: Record<SplitName, PathBuilder> = {
+		train: intermediateDir("labeled-train.jsonl"),
+		val: intermediateDir("labeled-val.jsonl"),
+		test: intermediateDir("labeled-test.jsonl"),
 	}
 
-	const quarantinePath = join(intermediateDir, "quarantine.jsonl")
-	const quarantineStream = openWriteStream(quarantinePath, { encoding: "utf8" })
-
-	let aligned = 0
-	let quarantined = 0
-	const counts: Record<SplitName, number> = { train: 0, val: 0, test: 0 }
 	const holdouts = defaultHoldouts()
-	// License accounting + the deliberate exclusion filter (#26). `licenseCounts` is the resolved
-	// license set (→ manifest + model-card attribution); `excludeLicenses` (empty by default → include
-	// everything) is the operator's PURPOSEFUL exclusion, never a silent drop.
-	const excludeLicenses = opts.excludeLicenses ?? []
-	const licenseCounts = new Map<string, number>()
-	let excludedByLicense = 0
-
-	const writeQuarantine = (row: CanonicalRow, reason: string): void => {
-		quarantineStream.write(`${JSON.stringify({ row, reason })}\n`)
-	}
-
-	for (const adapterRun of adapterRuns) {
-		opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`)
-
-		for await (const row of streamJSONL<CanonicalRow>(adapterRun.jsonl_path)) {
-			licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1)
-
-			// Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
-			// kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
-			// purposeful act, not a silent default. Counted BEFORE the drop so the manifest's license
-			// set reflects what the corpus actually CONTAINED, and `excluded_by_license` what was removed.
-			if (licenseExcluded(row.license, excludeLicenses)) {
-				excludedByLicense++
-
-				continue
-			}
-
-			const fanned: CanonicalRow[] = [row]
+	const registerDecisions = await readRegisterDecisions()
 
-			if (synthesize) {
-				for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
-					fanned.push(aug)
-				}
-			}
-
-			for (const r of fanned) {
-				let result: ReturnType<typeof alignRow>
-
-				try {
-					result = alignRow(r)
-				} catch (error) {
-					// Last-resort robustness (2026-06-12): no single row may crash a multi-hour build.
-					// alignRow's targeted paths normalize/quarantine known issues with specific reasons;
-					// this catches any UNKNOWN throw (e.g. assertSpanInvariants on an unforeseen span
-					// shape) → quarantine + continue. A spike in `align-threw` reasons is a finding.
-					writeQuarantine(r, `align-threw:${(error as Error).message.slice(0, 160)}`)
-
-					quarantined++
-
-					continue
-				}
-
-				if (result.kind === "labeled") {
-					const split = splitForRow(result.row, holdouts)
-					labeledStreams[split].write(`${JSON.stringify(result.row)}\n`)
-
-					counts[split]++
-
-					aligned++
-				} else {
-					writeQuarantine(r, result.row.reason)
-
-					quarantined++
-				}
-			}
-		}
-	}
-
-	for (const s of Object.values(labeledStreams)) {
-		s.end()
-	}
-
-	quarantineStream.end()
-	await Promise.all([...Object.values(labeledStreams).map((s) => once(s, "close")), once(quarantineStream, "close")])
+	const align = await runAlignPhase({
+		adapterRuns,
+		corpusVersion: opts.corpusVersion,
+		synthesize,
+		profile,
+		licensePolicy,
+		excludeLicenses: opts.excludeLicenses ?? [],
+		holdouts,
+		eligibility: profile === BuildProfile.ReleaseEligible ? await readSourceEligibility() : null,
+		labeledPaths,
+		quarantinePath: intermediateDir("quarantine.jsonl"),
+		checkpointPath: intermediateDir("align-checkpoint.json"),
+		onProgress: opts.onProgress,
+	})
 
-	// 4. Splits — manifest derived by streaming the per-split labeled files; no in-memory
-	// source-id arrays. `sort(1)` from coreutils produces the deterministic per-split .txt
-	// manifests with disk spill for splits that exceed in-memory thresholds.
-	opts.onProgress?.("split", `splitting ${aligned} aligned rows`)
-	const splitsDir = join(opts.outputDir, "splits")
+	opts.onProgress?.("split", `splitting ${align.aligned} aligned rows`)
 
 	const splitCounts = await writeSplitManifestsFromLabeledFiles({
 		labeledPaths,
-		outputDir: splitsDir,
+		outputDir: outputDir("splits"),
 		corpusVersion: opts.corpusVersion,
-		counts,
+		counts: align.counts,
 		holdouts,
 	})
 
-	// 5. Parquet slices — per-split labeled JSONL streams in, sliced `.parquet` out. The prior
-	// `splitFor(source_id)` callback (and the `Map<source_id, SplitName>` behind it) is gone.
-	opts.onProgress?.("slice", "writing parquet slices")
+	opts.onProgress?.("parquet", "writing parquet files")
+
+	// Each split gets its own generator seeded from one base, so a split's row order does
+	// not depend on how many rows the earlier splits happened to contain.
+	const shuffled = (path: PathBuilderLike, salt: number) =>
+		shuffleWithinWindow(streamJSONL<LabeledRow>(path), makeMulberry32(shuffleSeed + salt), shuffleWindow)
 
-	const sliceManifest = await writeSlices(
+	opts.onProgress?.(
+		"parquet",
+		shuffleWindow > 1
+			? `shuffling each split within a ${shuffleWindow.toLocaleString("en-US")}-row window, seed ${shuffleSeed}`
+			: "writing rows in arrival order (shuffle window disabled)"
+	)
+
+	const parquetManifest = await writeParquetSplits(
 		{
-			train: streamJSONL<LabeledRow>(labeledPaths.train),
-			val: streamJSONL<LabeledRow>(labeledPaths.val),
-			test: streamJSONL<LabeledRow>(labeledPaths.test),
+			train: shuffled(labeledPaths.train, 0),
+			val: shuffled(labeledPaths.val, 1),
+			test: shuffled(labeledPaths.test, 2),
 		},
 		{
-			outputDir: opts.outputDir,
+			outputDir,
 			corpusVersion: opts.corpusVersion,
-			rowsPerSlice,
+			rowsPerFile,
+			resume: $public.MAILWOMAN_RESUME === "1",
 		}
 	)
 
-	// License-set visibility (#26): loudly report the resolved license set so a build is an obvious
-	// deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
-	const licenseSummary = [...licenseCounts.entries()].toSorted((a, b) => b[1] - a[1])
+	const licenseSummary = [...align.licenseCounts.entries()].toSorted((a, b) => b[1] - a[1])
+	const refusedValues = align.licenses.refusedValues()
+	const refusalTally = [...align.refusedByKind.entries()].map(([kind, rows]) => `${kind}=${rows}`).join(", ")
 
 	opts.onProgress?.(
 		"manifest",
 		`license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
-			(excludedByLicense > 0
-				? ` | EXCLUDED ${excludedByLicense} rows by --exclude-licenses`
-				: " | NO license exclusion applied (all rows kept)")
+			` | policy ${licensePolicy}` +
+			(align.excludedByLicense > 0
+				? ` | refused ${align.excludedByLicense} rows over ${refusedValues.size} values (${refusalTally})`
+				: ` | refused 0 rows`) +
+			` | admitted ${align.admittedUnresolved} rows whose license resolves to no expression`
 	)
 
-	// 6. Top-level manifest.
 	opts.onProgress?.("manifest", "writing top-level MANIFEST.json")
 
+	// Frozen from what this build observed rather than re-derived later, because the register moves
+	// as terms are reviewed and a re-derived record would describe a build that never happened.
+	const trainingManifest = freezeTrainingManifest({
+		corpusVersion: opts.corpusVersion,
+		builtAt: built_at,
+		profile,
+		rowsBySource: align.rowsBySource,
+		decisionsByLicense: registerDecisions,
+		refused: align.ineligibleSources,
+	})
+
+	await writeLocalJSONFile(trainingManifest, outputDir(TRAINING_MANIFEST_FILE))
+
 	const manifest: BuildCorpusManifest = {
 		corpus_version: opts.corpusVersion,
 		built_at,
 		adapters: adapterRuns,
 		skipped_adapters: skipped,
 		splits: { counts: splitCounts, holdouts },
-		slices: { counts: sliceManifest.counts, total_rows: sliceManifest.total_rows },
-		quarantine_count: quarantined,
-		total_aligned_rows: aligned,
+		slices: { counts: parquetManifest.counts, total_rows: parquetManifest.total_rows },
+		quarantine_count: align.quarantined,
+		total_aligned_rows: align.aligned,
 		licenses: Object.fromEntries(licenseSummary),
-		excluded_by_license: excludedByLicense,
+		licenses_cover:
+			"the rows this build aligned from its adapters. An overlay parquet assembled afterwards carries its " +
+			"own licenses, and the overlay corpus's own MANIFEST.json states no license set at all, so a reader " +
+			"attributing an overlay corpus reads this set plus each overlay's own record rather than this set alone.",
+		excluded_by_license: align.excludedByLicense,
+		license_policy: licensePolicy,
+		refused_by_license_kind: Object.fromEntries(align.refusedByKind),
+		refused_license_values: Object.fromEntries(refusedValues),
+		refused_rows_by_source: Object.fromEntries(align.refusedBySource),
+		admitted_unresolved_license_rows: align.admittedUnresolved,
+		profile,
+		excluded_by_eligibility: align.excludedByEligibility,
+		ineligible_sources: Object.fromEntries(align.ineligibleSources),
+		training_manifest_digest: trainingManifest.contentDigest,
 	}
 
-	await writeLocalJSONFile(manifest, opts.outputDir, "MANIFEST.json")
+	await writeLocalJSONFile(manifest, outputDir("MANIFEST.json"))
 
 	return manifest
 }
-
-async function* streamJSONL<T>(path: string): AsyncIterable<T> {
-	// JSONSpliterator yields already-parsed rows (skipEmpty is on by default, so blank
-	// lines are dropped at the row level) and throws SyntaxError on a malformed row —
-	// same fail-loud behavior as the prior readline + bare `JSON.parse`.
-	yield* JSONSpliterator.fromAsync<T>(path)
-}