@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/build/inputs.ts

@@ -0,0 +1,81 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The committed record of which adapter read which input for one base corpus version.
+ *
+ *   Before this record existed, the only statement of a build's inputs was a JSON string in the launch
+ *   command, kept afterwards in a build-log directory under the data root. The predecessor of the
+ *   `v0.7.0-de-holdout` record names paths under `/mnt/playpen/mailwoman-data`. Those paths stopped resolving
+ *   when the data root moved on 2026-09-23. A corpus `MANIFEST.json` records `skipped_adapters` without any
+ *   adapter's input path, so a rebuild began by re-deriving the adapter set from that skip list.
+ *
+ *   Each `inputPath` is written relative to `$MAILWOMAN_DATA_ROOT` and resolved at launch, so the record
+ *   survives a data-root move. A path that has to sit outside the data root is written absolute and used as
+ *   written.
+ */
+
+import { dataRootPath } from "@mailwoman/core/data-root"
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
+import { resolveModulePath } from "@mailwoman/core/module/resolvers"
+import { isAbsolute, type PathBuilderLike } from "path-ts"
+
+import type { AdapterOptions } from "#types"
+
+/**
+ * One adapter's input as the committed record states it.
+ *
+ * `inputPath` is data-root-relative.
+ * The remaining fields match {@linkcode AdapterOptions} and exist so a per-country
+ * or truncated run can be recorded as it ran.
+ */
+export interface BuildInputRecord {
+	inputPath: string
+	country?: string
+	limit?: number
+}
+
+/**
+ * The adapter inputs of one base corpus build.
+ */
+export interface BuildInputsRecord {
+	corpusVersion: string
+	builtAt?: string
+	adapters: Record<string, BuildInputRecord>
+}
+
+/**
+ * Resolve the committed inputs record for a corpus version through the package's `./data/*` export.
+ *
+ * Resolution happens on call, matching `addressSourceRegisterPath`.
+ */
+export function buildInputsPath(corpusVersion: string): string {
+	return resolveModulePath(`@mailwoman/corpus/data/builds/${corpusVersion}/inputs.json`)
+}
+
+/**
+ * Read a committed inputs record and resolve every `inputPath` against the data root.
+ *
+ * @throws When the record names no adapter, because a build with no inputs
+ * writes an empty corpus and reports success.
+ */
+export async function readBuildInputs(path: PathBuilderLike): Promise<Record<string, AdapterOptions>> {
+	const record = await readLocalJSONFile<BuildInputsRecord>(path)
+	const entries = Object.entries(record.adapters ?? {})
+
+	if (!entries.length) {
+		throw new Error(`${path.toString()} names no adapter under \`adapters\`, so the build would read nothing.`)
+	}
+
+	return Object.fromEntries(
+		entries.map(([id, input]) => {
+			if (typeof input?.inputPath !== "string" || !input.inputPath) {
+				throw new Error(`${path.toString()}: adapter ${id} has no \`inputPath\`.`)
+			}
+
+			const resolved = isAbsolute(input.inputPath) ? input.inputPath : dataRootPath(input.inputPath).toString()
+
+			return [id, { ...input, inputPath: resolved } satisfies AdapterOptions]
+		})
+	)
+}

package/lib/build/rows.ts

@@ -0,0 +1,21 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Reading a build's own JSONL back, the one reader every phase after the adapters uses.
+ */
+
+import { delimitedSource, preferCompressed } from "@mailwoman/core/fs/delimited"
+import type { PathBuilderLike } from "path-ts"
+import { JSONSpliterator } from "spliterator"
+
+/**
+ * Stream a JSONL file's rows.
+ *
+ * `JSONSpliterator` yields parsed rows and throws `SyntaxError` on a malformed row.
+ * `preferCompressed` and `delimitedSource` are called inline because each yields
+ * a source that may only be consumed once.
+ */
+export async function* streamJSONL<T>(path: PathBuilderLike): AsyncIterable<T> {
+	yield* JSONSpliterator.fromAsync<T>(delimitedSource(await preferCompressed(path)))
+}

package/lib/build/types.ts

@@ -0,0 +1,234 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Defines the `buildCorpus` options and the profile that determines source eligibility.
+ *   It also defines the top-level manifest for each build stage.
+ *
+ *   These sit apart from `buildCorpus` because every phase module reads them and the orchestrator imports
+ *   each phase. These types live separately because placing them beside the orchestrator would make phases import it.
+ */
+
+import type { PathBuilderLike } from "path-ts"
+
+import type { ParquetManifest } from "#parquet/writers"
+import type { AdapterRunManifest } from "#runner"
+import type { AdapterOptions, CorpusAdapter } from "#types"
+import type { LicensePolicy, LicenseRefusalKind } from "#utils/license"
+import type { SplitManifest } from "#utils/split"
+
+/**
+ * Stage tags surfaced to `onProgress`.
+ */
+export type BuildStage = "adapter-run" | "align" | "split" | "parquet" | "manifest"
+
+/**
+ * The frozen source record's filename, beside the build's own `MANIFEST.json`.
+ */
+export const TRAINING_MANIFEST_FILE = "TRAINING_SOURCES.json"
+
+/**
+ * Defines the purpose of a corpus build.
+ *
+ * The profile determines whether a source must be eligible before its rows enter.
+ *
+ * `excludeLicenses` is orthogonal to this: it is a refusal rather than positive eligibility.
+ */
+export const BuildProfile = {
+	/**
+	 * Include every row an adapter yields.
+	 * A measurement runs under this profile.
+	 */
+	Exploratory: "exploratory",
+	/**
+	 * Include a row only when the register marks its source eligible for ingest.
+	 * A source whose terms remain unread is refused here.
+	 */
+	ReleaseEligible: "release-eligible",
+} as const
+
+export type BuildProfile = (typeof BuildProfile)[keyof typeof BuildProfile]
+
+/**
+ * Per-invocation options for `buildCorpus`.
+ */
+export interface BuildCorpusOptions {
+	/**
+	 * Root output directory.
+	 * Every build artifact lands beneath it.
+	 */
+	outputDir: PathBuilderLike
+
+	/**
+	 * Corpus version (e.g. `"0.1.0"`), stamped onto every row and into the output directory name.
+	 */
+	corpusVersion: string
+
+	/**
+	 * Adapters to drive, in order.
+	 *
+	 * @defaultValue `defaultAdapterRegistry.list()`, and an explicit list filters the run.
+	 */
+	adapters?: readonly CorpusAdapter[]
+
+	/**
+	 * Per-adapter `AdapterOptions` looked up by adapter id.
+	 *
+	 * An adapter whose id is missing is skipped and recorded in the manifest.
+	 */
+	adapterInputs: Record<string, AdapterOptions>
+
+	/**
+	 * Enable the synthesis pass.
+	 *
+	 * @defaultValue `true`, and a fixture-driven smoke test passes `false`.
+	 */
+	synthesize?: boolean
+
+	/**
+	 * Max rows per `.parquet` file, forwarded to `writeParquetSplits`.
+	 *
+	 * @defaultValue `1_000_000`
+	 */
+	rowsPerFile?: number
+
+	/**
+	 * Rows held in memory while shuffling each split before it is written to parquet.
+	 *
+	 * Rows arrive in adapter order.
+	 * Within one source, that order groups rows by country.
+	 *
+	 * An unshuffled row group therefore holds rows from one to eleven countries in its source.
+	 * A bounded epoch draw reads only those countries.
+	 *
+	 * @defaultValue `DEFAULT_SHUFFLE_WINDOW`, where `0` or `1` writes arrival
+	 * order unchanged and consumes no random draw.
+	 */
+	shuffleWindow?: number
+
+	/**
+	 * Seed for {@linkcode BuildCorpusOptions.shuffleWindow}'s draw.
+	 *
+	 * It is fixed by default, so two builds of one input write the same row order.
+	 */
+	shuffleSeed?: number
+
+	/**
+	 * Reports each build stage.
+	 * An error thrown from it aborts the build.
+	 */
+	onProgress?: (stage: BuildStage, message: string) => void
+
+	/**
+	 * License prefixes supplied in `--exclude-licenses`.
+	 *
+	 * A row whose `license` starts with one is dropped at ingest.
+	 * The default excludes no prefix.
+	 */
+	excludeLicenses?: readonly RegExp[]
+
+	/**
+	 * Which rows this build admits on the evidence of their license obligations.
+	 *
+	 * @defaultValue `LicensePolicy.All`, admitting every row an adapter yields.
+	 *
+	 * A weights build that must have no share-alike obligation passes `LicensePolicy.ShareAlikeFree`.
+	 */
+	licensePolicy?: LicensePolicy
+
+	/**
+	 * What this corpus is being built for.
+	 *
+	 * @defaultValue {@linkcode BuildProfile.Exploratory}
+	 */
+	profile?: BuildProfile
+}
+
+/**
+ * Top-level manifest tying every stage together.
+ */
+export interface BuildCorpusManifest {
+	corpus_version: string
+	built_at: string
+	adapters: AdapterRunManifest[]
+	skipped_adapters: string[]
+	splits: { counts: SplitManifest["counts"]; holdouts: SplitManifest["holdouts"] }
+	slices: { counts: ParquetManifest["counts"]; total_rows: number }
+	quarantine_count: number
+	total_aligned_rows: number
+	/**
+	 * Resolved license set across all included rows (license string → row count).
+	 *
+	 * The model card derives its data-attribution table from it.
+	 *
+	 * It covers the rows this build aligned.
+	 * An overlay parquet added afterwards by `corpus overlay-manifest` contains
+	 * its own rows and its own licenses.
+	 *
+	 * The overlay corpus's `MANIFEST.json` copies this set forward without them.
+	 * {@linkcode licenses_cover} records this in the artifact, so a reader deriving an attribution
+	 * table from a corpus with overlays knows the set is incomplete rather than complete.
+	 */
+	licenses: Record<string, number>
+	/**
+	 * Which rows the `licenses` set was measured over.
+	 *
+	 * A base build writes `adapter-rows`, because no overlay had been assembled when it ran.
+	 */
+	licenses_cover: string
+	excluded_by_license: number
+	/**
+	 * The policy this build admitted rows under, so a consumer can tell a corpus that
+	 * was never filtered from one filtered and found clean.
+	 */
+	license_policy: LicensePolicy
+	/**
+	 * Rows refused per `LicenseRefusalKind`, summing to `excluded_by_license`.
+	 *
+	 * The classes rest on different evidence and a reader deciding what to do needs them apart.
+	 */
+	refused_by_license_kind: Record<string, number>
+	/**
+	 * Every distinct license value this build refused, with the class it was refused under.
+	 *
+	 * A build that dropped rows records the values it dropped them for rather than only the count.
+	 */
+	refused_license_values: Record<string, LicenseRefusalKind>
+	/**
+	 * Rows the license policy refused, counted per `source`.
+	 *
+	 * A source listed here contributed no row to this corpus, whatever `adapters[].written` reports
+	 * for it: that field records what an adapter emitted, and the license filter runs after it.
+	 * Model provenance is generated from what survived every filter, so a source present in corpus
+	 * provenance may be absent from model provenance, and this field is where the difference reads.
+	 */
+	refused_rows_by_source: Record<string, number>
+	/**
+	 * Admitted rows whose license resolves to no SPDX expression.
+	 * Their obligations are unknown rather than empty.
+	 *
+	 * Under `LicensePolicy.ResolvedOnly` this reads zero because those rows are refused instead.
+	 */
+	admitted_unresolved_license_rows: number
+	/**
+	 * The profile this build ran under, so a consumer can tell whether the corpus's sources were checked.
+	 */
+	profile: BuildProfile
+	/**
+	 * Rows dropped because the register does not call their source eligible.
+	 *
+	 * This reads zero under {@linkcode BuildProfile.Exploratory}, applying no eligibility check.
+	 */
+	excluded_by_eligibility: number
+	/**
+	 * Every source refused under {@linkcode BuildProfile.ReleaseEligible}, with the reasons the
+	 * register gave, so a blocked build reports what to fix along with the fact that it stopped.
+	 */
+	ineligible_sources: Record<string, readonly string[]>
+	/**
+	 * The `contentDigest` of the `TRAINING_SOURCES.json` written beside this manifest.
+	 *
+	 * That file is the frozen record of which sources contributed and under which terms.
+	 */
+	training_manifest_digest: string
+}