@mailwoman/corpus 9.2.0 → 9.4.0

package/src/adapters/wof-json-rows.ts

@@ -1,79 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- * @file Deterministic canonical-row emission shared by WOF GeoJSON adapters.
- */
-
-import type { ComponentTag } from "@mailwoman/core/types"
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
-
-import type { AdapterOptions, CanonicalRow } from "#types"
-import type { WOFRecord } from "#utils"
-
-export interface WOFVariantSpec {
-	suffix: string
-	components: Partial<Record<ComponentTag, string>>
-}
-
-interface EmitWOFJSONRowsOptions {
-	records: ReadonlyMap<number, WOFRecord>
-	ancestry: ReadonlyMap<number, WOFRecord[]>
-	adapterOptions: AdapterOptions
-	adapterID: string
-	localeByCountry: Readonly<Record<string, string>>
-	shouldEmit?: (record: WOFRecord) => boolean
-	nameSlotsFor: (record: WOFRecord) => Array<{ key: string; value: string }>
-	variantsFor: (record: WOFRecord, ancestry: WOFRecord[], selfName: string) => WOFVariantSpec[]
-}
-
-/**
- * Emit aligned rows in WOF-id order, enforcing the adapter limit across all name and hierarchy variants.
- */
-export function* emitWOFJSONRows(options: EmitWOFJSONRowsOptions): Generator<CanonicalRow> {
-	const {
-		records,
-		ancestry,
-		adapterOptions,
-		adapterID,
-		localeByCountry,
-		shouldEmit = () => true,
-		nameSlotsFor,
-		variantsFor,
-	} = options
-
-	let emitted = 0
-
-	for (const id of [...records.keys()].toSorted((a, b) => a - b)) {
-		if (adapterOptions.signal?.aborted) return
-		const record = records.get(id)!
-
-		if (!shouldEmit(record)) continue
-
-		for (const slot of nameSlotsFor(record)) {
-			for (const variant of variantsFor(record, ancestry.get(id) ?? [], slot.value)) {
-				if (adapterOptions.limit !== undefined && emitted >= adapterOptions.limit) return
-
-				const raw = formatAddress(variant.components, record.country, { separator: ", " })
-
-				if (!raw) continue
-				const components = reconcileComponents(variant.components, raw)
-
-				if (!Object.keys(components).length) continue
-
-				yield {
-					raw,
-					components,
-					country: record.country,
-					locale: localeByCountry[record.country],
-					source: adapterID,
-					source_id: `${adapterID}-${record.id}-${slot.key}-${variant.suffix}`,
-					corpus_version: "",
-					license: "CC0-1.0",
-				}
-
-				emitted++
-			}
-		}
-	}
-}

package/src/adapters/wof-postalcode-json/adapter.ts

@@ -1,183 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `wof-postalcode`: Who's On First postalcode GeoJSON-bundle adapter.
- *
- *   **Phase 1.5.1 pivot.** Replaces the previous SpatiaLite-backed implementation (formerly at
- *   `packages/corpus/src/adapters/wof-postalcode/`, removed in this same change). The rationale is
- *   in `wof-admin-json/adapter.ts` and in `DECISIONS.md` — short version: the SQLite distribution
- *   mirror is dead, the live distro tags every postcode row `mz:is_current = -1` which the old
- *   `is_current = 1` predicate excluded, and localized `name:*` variants don't ship in the SQLite
- *   export at all.
- *
- *   Input: a directory containing one or more cloned `whosonfirst-data-postalcode-<cc>` repos plus
- *   the relevant `whosonfirst-data-admin-<cc>` repos (postcode records reference admin ancestry by
- *   `wof:parent_id`, so the locality / region / country records must be in the same walk for the
- *   ancestry chain to resolve). The corpus pipeline clones all four repos under
- *   `/data/corpus/sources/wof/repos/` and points the adapter at that root.
- *
- *   Per live postalcode record, the adapter emits one row per `(name-variant, hierarchy-variant)`
- *   pair:
- *
- *   - **Name variants**: canonical `wof:name` (slot key `default`, typically the postcode digits
- *       themselves) plus any `name:*` variants on the postcode feature. In practice WOF postcode
- *       records rarely carry localized name variants, so this expansion is usually a no-op — but
- *       the code path stays symmetric with the admin adapter for consistency.
- *   - **Hierarchy variants** (unchanged from the SQLite adapter): self, +locality, +locality+region,
- *       +locality+region+country.
- *
- *   `source_id` is `wof-postalcode-<wof_id>-<name-slot>-<hierarchy-variant>`. Ancestor names always
- *   come from the ancestor's canonical `wof:name`; this adapter does NOT iterate ancestor name
- *   variants (e.g. it does not emit `"75008 Париж"` even when Paris has a `name:rus_x_preferred`).
- *   That cross-product belongs to a future synthesis pass; emitting it here would multiply row
- *   counts ~10× without a clear training-value story.
- *
- *   License: CC0.
- */
-
-import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
-import type { ComponentTag } from "@mailwoman/core/types"
-
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "#utils"
-
-import { emitWOFJSONRows, type WOFVariantSpec } from "../wof-json-rows.ts"
-
-const COUNTRY_DISPLAY_NAME: Record<string, string> = {
-	US: "United States of America",
-	FR: "France",
-}
-
-const LOCALE_BY_COUNTRY: Record<string, string> = {
-	US: "en-US",
-	FR: "fr-FR",
-}
-
-function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag | undefined {
-	switch (placetype) {
-		case "country":
-		case "nation":
-			return "country"
-		case "macroregion":
-		case "region":
-			return "region"
-		case "locality":
-			return "locality"
-		case "postalcode":
-			return "postcode"
-		default:
-			return undefined
-	}
-}
-
-/**
- * Compute hierarchy variants for a postcode record. `selfName` is the postcode surface form (canonical `wof:name` for
- * the `default` slot, a `name:*` localized variant otherwise).
- */
-export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[] {
-	if (placetypeToTag(row.placetype) !== "postcode") return []
-
-	const locality = ancestry.find((a) => placetypeToTag(a.placetype) === "locality")
-	const region = ancestry.find((a) => placetypeToTag(a.placetype) === "region")
-	const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country")
-	const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country
-
-	const variants: WOFVariantSpec[] = [{ suffix: "self", components: { postcode: selfName } }]
-
-	if (locality) {
-		variants.push({
-			suffix: "with-locality",
-			components: { postcode: selfName, locality: locality.name },
-		})
-	}
-
-	if (locality && region) {
-		variants.push({
-			suffix: "with-locality-region",
-			components: { postcode: selfName, locality: locality.name, region: region.name },
-		})
-	}
-
-	if (locality && region && country) {
-		variants.push({
-			suffix: "with-locality-region-country",
-			components: {
-				postcode: selfName,
-				locality: locality.name,
-				region: region.name,
-				country: countryDisplay,
-			},
-		})
-	}
-
-	return variants
-}
-
-/**
- * Build the per-record name-slot list. The `default` slot uses `wof:name` verbatim (postcode digits); subsequent slots
- * come from `name:*` variants dedup'd against the default.
- */
-export function nameSlotsFor(rec: WOFRecord): Array<{ key: string; value: string }> {
-	const seen = new Set<string>([rec.name])
-	const slots: Array<{ key: string; value: string }> = [{ key: "default", value: rec.name }]
-
-	for (const [rawKey, value] of rec.nameVariants) {
-		if (seen.has(value)) continue
-		seen.add(value)
-		slots.push({ key: normalizeNameKey(rawKey), value })
-	}
-
-	return slots
-}
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const WOF_POSTALCODE_ADAPTER_ID = "wof-postalcode"
-
-export function createWOFPostalcodeAdapter(): CorpusAdapter {
-	return {
-		id: WOF_POSTALCODE_ADAPTER_ID,
-		defaultLicense: "CC0-1.0",
-		description:
-			"Who's On First postalcode GeoJSON bundles (postcode → locality/region pairs). Ancestor names from sibling admin repos.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			// Pass 1: full walk. We keep every record whose placetype maps to a ComponentTag — the
-			// postcode adapter needs locality / region / country admin records in the index so it
-			// can resolve postcode ancestry, even though it only emits rows for postcode records.
-			const byID = new Map<number, WOFRecord>()
-
-			for await (const rec of walkFeatures(opts.inputPath, { signal: opts.signal })) {
-				if (opts.signal?.aborted) return
-
-				if (opts.country && rec.country !== opts.country) continue
-
-				if (!placetypeToTag(rec.placetype)) continue
-				byID.set(rec.id, rec)
-			}
-
-			const ancestry = buildAncestryIndex(byID)
-
-			// Pass 2: emit postcode rows only, sorted by id for determinism.
-			yield* emitWOFJSONRows({
-				records: byID,
-				ancestry,
-				adapterOptions: opts,
-				adapterID: WOF_POSTALCODE_ADAPTER_ID,
-				localeByCountry: LOCALE_BY_COUNTRY,
-				shouldEmit: (record) => placetypeToTag(record.placetype) === "postcode",
-				nameSlotsFor,
-				variantsFor: postcodeVariantsFor,
-			})
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const wofPostalcodeAdapter = createWOFPostalcodeAdapter()

package/src/build.ts

@@ -1,386 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   End-to-end corpus build (Phase 1 task #10 in the plan).
- *
- *   `buildCorpus(opts)` orchestrates every stage of the pipeline:
- *
- *   1. **Adapter runs** — drives every adapter in turn (via `runAdapter`), writing
- *        `<intermediate>/<adapter.id>/canonical.jsonl` shards.
- *   2. **Synthesis** — optional. For each canonical row, every applicable augmentation in the row's
- *        country-default policy emits an augmented row alongside the original.
- *   3. **Alignment** — every row (original + augmented) is aligned via `alignRow`. Successes go to
- *        `labeled.jsonl`; quarantines are appended to `quarantine.jsonl` with reasons.
- *   4. **Splits** — `splitRows` partitions labeled `source_id`s into train/val/test by locality holdout.
- *        Manifest written to `splits/SPLIT_MANIFEST.json` + per-split `train.txt` / `val.txt` /
- *        `test.txt`.
- *   5. **Parquet shards** — `writeShards` streams labeled rows into 1M-row `.parquet` shards per split
- *        under `corpus-v<version>/{train,val,test}/part-NNNN.parquet` (SNAPPY-compressed, 50k-row
- *        row groups), with per-shard checksums + per-stage manifest in
- *        `corpus-v<version>/MANIFEST.json`.
- *   6. **Top-level manifest** — `<outputDir>/MANIFEST.json` ties every per-stage manifest together with
- *        a top-level corpus_version, built_at, and aggregate counts.
- *
- *   Output layout:
- *
- *   ```
- *   <outputDir>/
- *   MANIFEST.json
- *   intermediate/
- *     <adapter.id>/canonical.jsonl   # one per adapter
- *     labeled.jsonl                  # post-alignment, pre-shard
- *     quarantine.jsonl               # rows that failed alignment
- *   splits/
- *     SPLIT_MANIFEST.json
- *     train.txt / val.txt / test.txt
- *   corpus-v<version>/
- *     MANIFEST.json
- *     train/part-NNNN.parquet
- *     val/part-NNNN.parquet
- *     test/part-NNNN.parquet
- * ```
- *
- *   The intermediate files live alongside the final shards for reproducibility + debugging. Operators
- *   can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
- *   self-contained.
- */
-
-import { createWriteStream, existsSync, readFileSync, type WriteStream } from "node:fs"
-import { mkdir, writeFile } from "node:fs/promises"
-import { join } from "node:path"
-
-import { $public } from "@mailwoman/core/env"
-import { parseJSONStrict } from "@mailwoman/core/objects"
-import { JSONSpliterator } from "spliterator"
-
-import { defaultAdapterRegistry } from "#adapters/utils"
-import { runAdapter, type AdapterRunManifest } from "#runner"
-import { defaultAugmentationsForCountry, synthesizeRow } from "#synthesizers/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "#types"
-import { alignRow } from "#utils/align"
-import { licenseExcluded } from "#utils/license"
-import { writeShards, type ShardManifest } from "#utils/parquet"
-import {
-	defaultHoldouts,
-	splitForRow,
-	writeSplitManifestsFromLabeledFiles,
-	type SplitManifest,
-	type SplitName,
-} from "#utils/split"
-
-/**
- * Stage tags surfaced to `onProgress`.
- */
-export type BuildStage = "adapter-run" | "align" | "split" | "shard" | "manifest"
-
-/**
- * Per-invocation options for `buildCorpus`.
- */
-export interface BuildCorpusOptions {
-	/**
-	 * Root output directory. All build artifacts land beneath it.
-	 */
-	outputDir: string
-
-	/**
-	 * Corpus version (e.g. `"0.1.0"`). Stamped onto every row + into the output dir name.
-	 */
-	corpusVersion: string
-
-	/**
-	 * Adapters to drive, in order. Defaults to `defaultAdapterRegistry.list()`. Pass an explicit list to filter (e.g.
-	 * `[wofAdminAdapter]` for a smoke run).
-	 */
-	adapters?: readonly CorpusAdapter[]
-
-	/**
-	 * Per-adapter `AdapterOptions` — looked up by adapter id. Adapters whose id is missing from this map are skipped (and
-	 * noted in the manifest).
-	 */
-	adapterInputs: Record<string, AdapterOptions>
-
-	/**
-	 * Enable synthesis pass. Default `true`. Set `false` for fixture-driven smoke tests.
-	 */
-	synthesize?: boolean
-
-	/**
-	 * Forwarded to `writeShards`. Default 1_000_000.
-	 */
-	rowsPerShard?: number
-
-	/**
-	 * Progress hook. Errors thrown abort the build.
-	 */
-	onProgress?: (stage: BuildStage, message: string) => void
-
-	/**
-	 * License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see `compileLicenseExcludes` /
-	 * `SHARE_ALIKE_PATTERN` in `license.ts`); a row whose `license` matches any is dropped at ingest. Default (omitted)
-	 * includes EVERYTHING — exclusion is a deliberate act, not a silent default. A proprietary-weights build passes the
-	 * share-alike set (`--exclude-share-alike`).
-	 */
-	excludeLicenses?: readonly RegExp[]
-}
-
-/**
- * Top-level manifest tying every stage together.
- */
-export interface BuildCorpusManifest {
-	corpus_version: string
-	built_at: string
-	adapters: AdapterRunManifest[]
-	skipped_adapters: string[]
-	splits: { counts: SplitManifest["counts"]; holdouts: SplitManifest["holdouts"] }
-	shards: { counts: ShardManifest["counts"]; total_rows: number }
-	quarantine_count: number
-	total_aligned_rows: number
-	/**
-	 * Resolved license set across all INCLUDED rows (license string → row count), + the count dropped by
-	 * `excludeLicenses` (#26). The model card derives its data-attribution table from `licenses`.
-	 */
-	licenses: Record<string, number>
-	excluded_by_license: number
-}
-
-/**
- * Drive the full corpus build to completion.
- *
- * Memory profile: the function maintains an in-memory `Map<source_id, SplitName>` to bridge the align → shard hand-off.
- * For Phase 1 fixture-scale runs (≤ 10⁴ rows) this is trivial. For real 5M+ runs, the map fits comfortably in a few
- * hundred MB; the canonical.jsonl and labeled.jsonl payloads stream and never sit in memory.
- */
-export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpusManifest> {
-	const adapters = opts.adapters ?? defaultAdapterRegistry.list()
-	const synthesize = opts.synthesize ?? true
-	const rowsPerShard = opts.rowsPerShard ?? 1_000_000
-	const built_at = new Date().toISOString()
-
-	await mkdir(opts.outputDir, { recursive: true })
-	const intermediateDir = join(opts.outputDir, "intermediate")
-	await mkdir(intermediateDir, { recursive: true })
-
-	// 1. Adapter runs.
-	const adapterRuns: AdapterRunManifest[] = []
-	const skipped: string[] = []
-
-	for (const adapter of adapters) {
-		const adapterOptions = opts.adapterInputs[adapter.id]
-
-		if (!adapterOptions) {
-			skipped.push(adapter.id)
-			opts.onProgress?.("adapter-run", `skipped ${adapter.id} (no input configured)`)
-
-			continue
-		}
-
-		// Opt-in resume (MAILWOMAN_RESUME=1): if a complete per-adapter canonical.jsonl + MANIFEST.json
-		// already exist, reuse them instead of re-emitting. The MANIFEST is written only after the
-		// canonical is fully flushed, so its presence guarantees completeness; row order is identical,
-		// so downstream holdout-split determinism is preserved. Recovers an align-phase crash without
-		// redoing the (expensive) emit phase. Default (unset) re-emits, preserving correctness. (2026-06-12.)
-		const adapterDir = join(intermediateDir, adapter.id)
-		const cachedManifest = join(adapterDir, "MANIFEST.json")
-
-		if (
-			$public.MAILWOMAN_RESUME === "1" &&
-			existsSync(cachedManifest) &&
-			existsSync(join(adapterDir, "canonical.jsonl"))
-		) {
-			const cached = parseJSONStrict<AdapterRunManifest>(readFileSync(cachedManifest, "utf8"))
-			opts.onProgress?.("adapter-run", `resumed ${adapter.id} (reused ${cached.yielded} canonical rows)`)
-			adapterRuns.push(cached)
-
-			continue
-		}
-
-		opts.onProgress?.("adapter-run", `running ${adapter.id}`)
-
-		const m = await runAdapter({
-			adapter,
-			adapterOptions,
-			outputDir: intermediateDir,
-			corpusVersion: opts.corpusVersion,
-		})
-
-		adapterRuns.push(m)
-	}
-
-	// 2 + 3. Synthesis + alignment: stream every canonical.jsonl, optionally augment, align,
-	// and route each labeled row directly to its split-specific JSONL (`labeled-{train,val,test}.
-	// jsonl`). Memory cost is O(1) — the prior in-memory `splitInputs` array + `splitByIDMap`
-	// + `SplitManifest.{train,val,test}` arrays are gone; per-row split is decided inline via
-	// `splitForRow` (a pure function of source_id + region + holdout policy).
-	const labeledPaths: Record<SplitName, string> = {
-		train: join(intermediateDir, "labeled-train.jsonl"),
-		val: join(intermediateDir, "labeled-val.jsonl"),
-		test: join(intermediateDir, "labeled-test.jsonl"),
-	}
-
-	const labeledStreams: Record<SplitName, WriteStream> = {
-		train: createWriteStream(labeledPaths.train, { encoding: "utf8" }),
-		val: createWriteStream(labeledPaths.val, { encoding: "utf8" }),
-		test: createWriteStream(labeledPaths.test, { encoding: "utf8" }),
-	}
-
-	const quarantinePath = join(intermediateDir, "quarantine.jsonl")
-	const quarantineStream = createWriteStream(quarantinePath, { encoding: "utf8" })
-
-	let aligned = 0
-	let quarantined = 0
-	const counts: Record<SplitName, number> = { train: 0, val: 0, test: 0 }
-	const holdouts = defaultHoldouts()
-	// License accounting + the deliberate exclusion filter (#26). `licenseCounts` is the resolved
-	// license set (→ manifest + model-card attribution); `excludeLicenses` (empty by default → include
-	// everything) is the operator's PURPOSEFUL exclusion, never a silent drop.
-	const excludeLicenses = opts.excludeLicenses ?? []
-	const licenseCounts = new Map<string, number>()
-	let excludedByLicense = 0
-
-	const writeQuarantine = (row: CanonicalRow, reason: string): void => {
-		quarantineStream.write(`${JSON.stringify({ row, reason })}\n`)
-	}
-
-	for (const adapterRun of adapterRuns) {
-		opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`)
-
-		for await (const row of streamJSONL<CanonicalRow>(adapterRun.jsonl_path)) {
-			licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1)
-
-			// Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
-			// kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
-			// purposeful act, not a silent default. Counted BEFORE the drop so the manifest's license
-			// set reflects what the corpus actually CONTAINED, and `excluded_by_license` what was cut.
-			if (licenseExcluded(row.license, excludeLicenses)) {
-				excludedByLicense++
-
-				continue
-			}
-
-			const fanned: CanonicalRow[] = [row]
-
-			if (synthesize) {
-				for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
-					fanned.push(aug)
-				}
-			}
-
-			for (const r of fanned) {
-				let result: ReturnType<typeof alignRow>
-
-				try {
-					result = alignRow(r)
-				} catch (error) {
-					// Last-resort robustness (2026-06-12): no single row may crash a multi-hour build.
-					// alignRow's targeted paths normalize/quarantine known issues with specific reasons;
-					// this catches any UNKNOWN throw (e.g. assertSpanInvariants on an unforeseen span
-					// shape) → quarantine + continue. A spike in `align-threw` reasons is a finding.
-					writeQuarantine(r, `align-threw:${(error as Error).message.slice(0, 160)}`)
-
-					quarantined++
-
-					continue
-				}
-
-				if (result.kind === "labeled") {
-					const split = splitForRow(result.row, holdouts)
-					labeledStreams[split].write(`${JSON.stringify(result.row)}\n`)
-
-					counts[split]++
-
-					aligned++
-				} else {
-					writeQuarantine(r, result.row.reason)
-
-					quarantined++
-				}
-			}
-		}
-	}
-
-	for (const s of Object.values(labeledStreams)) {
-		s.end()
-	}
-
-	quarantineStream.end()
-	await Promise.all([...Object.values(labeledStreams).map(streamEnd), streamEnd(quarantineStream)])
-
-	// 4. Splits — manifest derived by streaming the per-split labeled files; no in-memory
-	// source-id arrays. `sort(1)` from coreutils produces the deterministic per-split .txt
-	// manifests with disk spill for splits that exceed in-memory thresholds.
-	opts.onProgress?.("split", `splitting ${aligned} aligned rows`)
-	const splitsDir = join(opts.outputDir, "splits")
-
-	const splitCounts = await writeSplitManifestsFromLabeledFiles({
-		labeledPaths,
-		outputDir: splitsDir,
-		corpusVersion: opts.corpusVersion,
-		counts,
-		holdouts,
-	})
-
-	// 5. Parquet shards — per-split labeled JSONL streams in, sharded `.parquet` out. The prior
-	// `splitFor(source_id)` callback (and the `Map<source_id, SplitName>` behind it) is gone.
-	opts.onProgress?.("shard", "writing parquet shards")
-
-	const shardManifest = await writeShards(
-		{
-			train: streamJSONL<LabeledRow>(labeledPaths.train),
-			val: streamJSONL<LabeledRow>(labeledPaths.val),
-			test: streamJSONL<LabeledRow>(labeledPaths.test),
-		},
-		{
-			outputDir: opts.outputDir,
-			corpusVersion: opts.corpusVersion,
-			rowsPerShard,
-		}
-	)
-
-	// License-set visibility (#26): loudly report the resolved license set so a build is an obvious
-	// deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
-	const licenseSummary = [...licenseCounts.entries()].toSorted((a, b) => b[1] - a[1])
-
-	opts.onProgress?.(
-		"manifest",
-		`license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
-			(excludedByLicense > 0
-				? ` | EXCLUDED ${excludedByLicense} rows by --exclude-licenses`
-				: " | NO license exclusion applied (all rows kept)")
-	)
-
-	// 6. Top-level manifest.
-	opts.onProgress?.("manifest", "writing top-level MANIFEST.json")
-
-	const manifest: BuildCorpusManifest = {
-		corpus_version: opts.corpusVersion,
-		built_at,
-		adapters: adapterRuns,
-		skipped_adapters: skipped,
-		splits: { counts: splitCounts, holdouts },
-		shards: { counts: shardManifest.counts, total_rows: shardManifest.total_rows },
-		quarantine_count: quarantined,
-		total_aligned_rows: aligned,
-		licenses: Object.fromEntries(licenseSummary),
-		excluded_by_license: excludedByLicense,
-	}
-
-	await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
-
-	return manifest
-}
-
-async function* streamJSONL<T>(path: string): AsyncIterable<T> {
-	// JSONSpliterator yields already-parsed rows (skipEmpty is on by default, so blank
-	// lines are dropped at the row level) and throws SyntaxError on a malformed row —
-	// same fail-loud behavior as the prior readline + bare `JSON.parse`.
-	yield* JSONSpliterator.fromAsync<T>(path)
-}
-
-function streamEnd(s: WriteStream): Promise<void> {
-	return new Promise((resolve, reject) => {
-		s.once("close", resolve)
-		s.once("error", reject)
-	})
-}

package/src/index.ts

@@ -1,14 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- */
-
-export * from "./adapters/utils/index.ts"
-export * from "./adapters/gnaf/assemble.ts"
-export * from "./adapters/index.ts"
-export * from "./build.ts"
-export * from "./runner.ts"
-export * from "./parquet-wrapper/index.ts"
-export * from "./shard-recipes/index.ts"
-export * from "#types"

package/src/name-prone-us-suffixes.ts

@@ -1,12 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- * Re-export shim: the name-prone set moved to `@mailwoman/codex/us` (data in
- * `codex/us/street-suffix.json`, 2026-08-10) so the golden relabel flags, the #1569 shard
- * recipe, AND the Python relabel pass (via the `gazetteer affix-relabel` v2 lexicon) all read
- * one record. Import from `@mailwoman/codex/us` in new code.
- */
-
-export { NAME_PRONE_US_SUFFIXES } from "@mailwoman/codex/us"

package/src/parquet-wrapper/index.ts

@@ -1,9 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- */
-
-export * from "./reader.ts"
-export * from "./schema.ts"
-export * from "./writer.ts"