@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/utils/cooccurrence-stats.ts

@@ -0,0 +1,101 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Token/label co-occurrence accounting shared by the corpus-stats builder (`tools/corpus-stats.ts`)
+ *   and the recipe-output linter (`tools/lint/recipe-output/index.ts`). The two sides must agree on the bigram key
+ *   format: the linter looks its recipe output's bigrams up in the stats file's `bigrams` table. A key built with
+ *   a different separator never matches — which silently blanks the bigram-collision check.
+ */
+
+import { openParquetRowStream } from "#parquet/streams"
+
+/**
+ * Separator inside a bigram key (`tok1␟tok2`) and a label-bigram value (`lab1␟lab2`):
+ * U+001F unit separator, a character no address token contains.
+ *
+ * Render a key for humans with `key.split(COOCCURRENCE_KEY_SEP).join(" ")`.
+ */
+export const COOCCURRENCE_KEY_SEP = ""
+
+/**
+ * Nested tally: outer key → (inner key → count).
+ *
+ * Token → label counts, or bigram key → label-bigram counts.
+ */
+type CooccurrenceTable = Map<string, Map<string, number>>
+
+/**
+ * The token + bigram co-occurrence tables one pass over labeled rows accumulates.
+ */
+export interface CooccurrenceStats {
+	tokens: CooccurrenceTable
+	bigrams: CooccurrenceTable
+}
+
+/**
+ * An empty pair of tables for {@link accumulateCooccurrences}.
+ */
+export function createCooccurrenceStats(): CooccurrenceStats {
+	return { tokens: new Map(), bigrams: new Map() }
+}
+
+function bump(table: CooccurrenceTable, key: string, sub: string): void {
+	let counter = table.get(key)
+
+	if (!counter) {
+		counter = new Map()
+		table.set(key, counter)
+	}
+
+	counter.set(sub, (counter.get(sub) ?? 0) + 1)
+}
+
+/**
+ * Fold one row's parallel `tokens`/`labels` into the tables.
+ *
+ * The caller has already checked the arrays are the same length.
+ */
+export function accumulateCooccurrences(
+	stats: CooccurrenceStats,
+	tokens: readonly string[],
+	labels: readonly string[]
+): void {
+	for (let i = 0; i < tokens.length; i++) {
+		const tk = tokens[i]!
+		const lb = labels[i]!
+
+		bump(stats.tokens, tk, lb)
+
+		if (i + 1 < tokens.length) {
+			bump(stats.bigrams, tk + COOCCURRENCE_KEY_SEP + tokens[i + 1]!, lb + COOCCURRENCE_KEY_SEP + labels[i + 1]!)
+		}
+	}
+}
+
+/**
+ * A parquet row projected to the two columns the co-occurrence pass reads.
+ */
+export interface TokenLabelRow {
+	tokens: string[]
+	labels: string[]
+	[key: string]: unknown
+}
+
+/**
+ * Stream a parquet file's `tokens`/`labels` columns.
+ *
+ * Projected rather than read whole: parquet is columnar, so the unused columns are never touched.
+ * `limit` stops the iteration rather than filtering afterwards, so a capped
+ * run reads only the row groups it needs.
+ */
+export async function* streamTokenLabelRows(parquetPath: string, limit?: number): AsyncIterable<TokenLabelRow> {
+	let emitted = 0
+
+	for await (const row of openParquetRowStream<TokenLabelRow>(parquetPath, { columns: ["tokens", "labels"], limit })) {
+		yield row
+
+		emitted++
+	}
+}

package/lib/utils/golden.ts

@@ -3,36 +3,25 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Golden eval-set validator (Phase 1 task #9 in the plan).
- *
- *   The golden set is hand-labeled ground truth for the neural classifier. Each entry must carry
- *   components whose surface forms actually occur in `raw` — otherwise the entry will silently rot
- *   the eval signal. This module:
- *
- *   - Defines `GoldenEntry` (schema check).
- *   - Loads `.jsonl` files (one entry per line).
- *   - Validates every entry: schema shape, ComponentTag membership, reachability of each component in
- *       `raw` via the same `reconcileComponents` helper alignment uses.
- *   - Returns a structured report of per-entry errors so the CLI / CI surface can act on it.
- *
- *   The 1000-entry target (500 US + 500 FR) is a human task. This module catches the regressions that
- *   creep in over time as new entries land.
+ *   Validates the hand-labeled golden eval set, where every labeled component must occur in `raw`.
  */
 
-import { readDirectory } from "@mailwoman/core/fs/readers"
-import { parseJSONStrict } from "@mailwoman/core/json"
-import { COMPONENT_TAGS, type ComponentTag } from "@mailwoman/core/types"
-import { reconcileComponents } from "@mailwoman/formatter"
-import { join } from "path-ts"
+import { componentsPresentIn } from "@mailwoman/codex/address/format"
+import { COMPONENT_TAGS, type ComponentTag } from "@mailwoman/codex/component"
+import { isAlpha2CodeShape } from "@mailwoman/codex/country"
+import { parseJSONStrict, stringifyJSON } from "@mailwoman/core/json"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
 import { TextSpliterator } from "spliterator"
+import { Globerator } from "spliterator/node/fs"
 
 const TAG_SET = new Set<string>(COMPONENT_TAGS as readonly string[])
 
 /**
  * A golden-set candidate row, as `golden-expand` writes and `golden-promote` reads.
  *
- * `source` names the producer (`expand-golden:<provider>`); the seed/provenance fields trace the candidate back to the
- * corpus row and LLM call that produced it. A committed golden entry is the {@link GoldenEntry} narrowing.
+ * `source` identifies the producer, such as `expand-golden:<provider>`.
+ * The seed and provenance fields trace the candidate to its corpus row and LLM call.
+ * {@link GoldenEntry} narrows this type to a committed entry.
  */
 export interface GoldenCandidateEntry {
 	raw: string
@@ -54,7 +43,7 @@ export interface GoldenEntry extends GoldenCandidateEntry {
 }
 
 /**
- * Per-entry validation failure.
+ * A validation failure for one line of a golden file.
  */
 export interface GoldenIssue {
 	file: string
@@ -63,7 +52,7 @@ export interface GoldenIssue {
 }
 
 /**
- * Aggregate report from `validateGoldenDir`.
+ * The aggregate report that `validateGoldenDir` returns.
  */
 export interface GoldenReport {
 	entries: number
@@ -72,23 +61,25 @@ export interface GoldenReport {
 }
 
 /**
- * Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations.
+ * Parses one JSONL line into a `GoldenEntry`.
+ *
+ * `validateGoldenFile` records the thrown message against the line number, so this parse must stay strict.
+ *
+ * @throws On schema violations.
  */
 export function parseGoldenLine(line: string): GoldenEntry {
-	// The throw IS the result: `validateGoldenFile` catches it and records the message against the line
-	// number, so a tolerant parse would report a corrupt row as valid.
 	const obj = parseJSONStrict<Partial<GoldenEntry> & Record<string, unknown>>(line)
 
 	if (typeof obj.raw !== "string" || !obj.raw.length) {
 		throw new Error("missing/empty raw")
 	}
 
-	if (typeof obj.country !== "string" || !/^[A-Z]{2}$/u.test(obj.country)) {
-		throw new Error(`country must be ISO 3166-1 alpha-2 (got ${JSON.stringify(obj.country)})`)
+	if (!isAlpha2CodeShape(obj.country)) {
+		throw new Error(`country must be ISO 3166-1 alpha-2 (got ${stringifyJSON(obj.country)})`)
 	}
 
 	if (obj.source !== "golden") {
-		throw new Error(`source must be "golden" (got ${JSON.stringify(obj.source)})`)
+		throw new Error(`source must be "golden" (got ${stringifyJSON(obj.source)})`)
 	}
 
 	const components = (obj.components ?? {}) as Record<string, unknown>
@@ -111,14 +102,17 @@ export function parseGoldenLine(line: string): GoldenEntry {
 }
 
 /**
- * Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent).
+ * Returns the tags in `entry` whose values do not occur in `entry.raw`.
+ *
+ * A golden `raw` is hand-written, so this check uses containment instead of
+ * reconciling against a rendered layout.
  */
 export function unreachableComponents(entry: GoldenEntry): ComponentTag[] {
-	const reconciled = reconcileComponents(entry.components, entry.raw)
+	const present = componentsPresentIn(entry.components, entry.raw)
 	const missing: ComponentTag[] = []
 
 	for (const tag of Object.keys(entry.components) as ComponentTag[]) {
-		if (!(tag in reconciled)) {
+		if (!(tag in present)) {
 			missing.push(tag)
 		}
 	}
@@ -127,21 +121,19 @@ export function unreachableComponents(entry: GoldenEntry): ComponentTag[] {
 }
 
 /**
- * Validate one `.jsonl` file end-to-end, returning a list of issues.
+ * Validates one `.jsonl` file and returns its issues.
  *
- * Parses line by line over `TextSpliterator` rather than `JSONSpliterator`: every issue this returns carries the LINE
- * NUMBER it was found on, and a malformed line has to be REPORTED rather than thrown. `JSONSpliterator` parses each row
- * for you and throws on the first bad one — correct for consumers that want the rows, wrong for the validator whose
- * whole job is locating the bad ones.
+ * This reads lines with `TextSpliterator` because `JSONSpliterator` throws on the first malformed row.
+ * The validator must instead report every bad row with its line number.
  */
-export async function validateGoldenFile(path: string): Promise<GoldenIssue[]> {
+export async function validateGoldenFile(source: PathBuilderLike): Promise<GoldenIssue[]> {
+	const path = source.toString()
 	const issues: GoldenIssue[] = []
-	// Counted over EVERY row including blanks, so the number matches what an editor shows.
+	// Blank lines count too, so the number matches the line an editor shows.
 	let lineNumber = 0
 
-	for await (const raw of TextSpliterator.fromAsync(path, { skipEmpty: false })) {
+	for await (const line of TextSpliterator.fromAsync(path, { skipEmpty: false })) {
 		lineNumber++
-		const line = raw.trim()
 
 		if (!line) continue
 		const i = lineNumber - 1
@@ -166,23 +158,20 @@ export async function validateGoldenFile(path: string): Promise<GoldenIssue[]> {
 }
 
 /**
- * Validate every `.jsonl` in a golden directory.
+ * Validates every `.jsonl` file directly inside a golden directory.
  */
-export async function validateGoldenDir(dir: string): Promise<GoldenReport> {
-	const files = (await readDirectory(dir)).filter((n) => n.endsWith(".jsonl")).toSorted()
+export async function validateGoldenDir(dir: PathBuilderLike): Promise<GoldenReport> {
+	const root = PathBuilder.from(dir)
+	const files = await Globerator.files("jsonl", { cwd: root, absolute: false, recursive: false }).toSorted()
 	const issues: GoldenIssue[] = []
 	let entries = 0
 
 	for (const name of files) {
-		const fullPath = join(dir, name)
+		const fullPath = root(name)
 		const fileIssues = await validateGoldenFile(fullPath)
 		issues.push(...fileIssues)
 
-		for await (const line of TextSpliterator.fromAsync(fullPath)) {
-			if (line.trim()) {
-				entries++
-			}
-		}
+		entries += await TextSpliterator.countAsync(fullPath)
 	}
 
 	return { entries, files: files.length, issues }

package/lib/utils/license.ts

@@ -3,48 +3,260 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Corpus licensing — the single source of truth for the training-data license policy (#26).
+ *   Corpus licensing — the training-data license policy, read from a row's obligations rather than from
+ *   the shape of its license string.
  *
- *   Posture (operator, 2026-06-19): **exclusion is a deliberate act, not a silent default.** The
- *   build INCLUDES every row an adapter yields (stamping its `license`); a build that needs a clean
- *   license set — e.g. the proprietary `@mailwoman/neural-weights-*` weights, which must not
- *   inherit a share-alike obligation — PURPOSELY excludes kinds via `buildCorpus({ excludeLicenses
- *   })` (CLI `--exclude-licenses` / `--exclude-share-alike`). Nothing is dropped on a license
- *   string unless the operator named it. This avoids the trap of silently dropping allowed data
- *   mis-stamped with a conservative license (e.g. BAN, which is dual-licensed Licence Ouverte OR
- *   ODbL — we elect Licence Ouverte; a default-deny on the old `ODbL` stamp would have wrongly
- *   dropped 48M allowed rows).
+ *   Exclusion is a deliberate act rather than a silent default: the build includes every row an adapter
+ *   yields (stamping its `license`), and a build that needs a license set carrying no share-alike
+ *   obligation asks for {@linkcode LicensePolicy.ShareAlikeFree}, so no source is dropped unless the
+ *   operator selected the policy.
  *
- *   Tier reference (#26): A = PD/CC0 (allowed); B = CC-BY / Licence Ouverte (allowed WITH attribution
- *   — the model card must carry it); C = share-alike (ODbL, CC-BY-SA, CC-SA) — exclude for a
- *   proprietary-weights build via `--exclude-share-alike`.
+ *   The refusal reads `readLicenseRecord`, which resolves a value to an SPDX expression and looks up
+ *   that expression's obligations. An anchored prefix match over the raw column cannot do this. Measured
+ *   over `v0.7.0-de-holdout` on 2026-09-26, the column holds 71 distinct values across 703,835,753 rows,
+ *   of which 4 are SPDX identifiers and 66 are prose. A row whose license reads "OpenStreetMap venue +
+ *   sub-venue names (ODbL, © OpenStreetMap contributors) …" is share-alike-derived and starts with
+ *   neither `ODbL` nor `CC-BY-SA`, so `/^ODbL/` admitted it.
+ *
+ *   Three refusal classes exist because they are different claims. A value that resolves to an
+ *   expression carrying share-alike states the obligation. A value whose prose mentions a share-alike
+ *   license while resolving to no expression is share-alike-derived without stating its own grant. A
+ *   value that resolves to no expression has unknown obligations. Unknown obligations differ from an empty
+ *   obligation set. Of these rows, 628,203,119 contain prose this repository writes about its own renderings. The
+ *   third class is therefore reported rather than refused under {@linkcode LicensePolicy.ShareAlikeFree},
+ *   and {@linkcode LicensePolicy.ResolvedOnly} refuses it for a caller that needs every grant stated.
+ */
+
+import { carriesShareAlike, mentionsShareAlike, readLicenseRecord } from "@mailwoman/core/license/record"
+import { extractDelimited } from "@mailwoman/core/scripting/arguments"
+import { escapeRegExp } from "@mailwoman/core/strings/regexp"
+
+/**
+ * Which rows a corpus build admits on the evidence of their license values.
+ */
+export const LicensePolicy = {
+	/**
+	 * Admit every row an adapter yields, whatever its license value states.
+	 * The policy a measurement runs under.
+	 */
+	All: "all",
+	/**
+	 * Refuse a row whose license imposes a share-alike obligation.
+	 *
+	 * Also refuse a row whose license text mentions a share-alike license while stating no grant of its own.
+	 *
+	 * A row whose license resolves to no expression and mentions no share-alike license is admitted
+	 * and counted, because its obligations are unknown rather than known to include share-alike.
+	 */
+	ShareAlikeFree: "share-alike-free",
+	/**
+	 * Apply all refusals from {@linkcode LicensePolicy.ShareAlikeFree}.
+	 * Also refuse a row whose license resolves to no expression.
+	 *
+	 * Measured over `v0.7.0-de-holdout`, this refuses 628,203,119 of 703,835,753 rows,
+	 * so a caller asking for it is asking for the subset whose grant is stated as an identifier.
+	 */
+	ResolvedOnly: "resolved-only",
+} as const
+
+/**
+ * One of the {@linkcode LicensePolicy} values.
  */
+export type LicensePolicy = (typeof LicensePolicy)[keyof typeof LicensePolicy]
 
 /**
- * Licenses that require share-alike / create a copyleft obligation on derived works (Tier C). The
- * `--exclude-share-alike` convenience expands to this; `allowShareAlike: false` adapters also use it.
+ * Why a row's license value kept it out of the corpus, counted separately per class
+ * because the classes rest on different evidence.
  */
-export const SHARE_ALIKE_PATTERN = /^ODbL|^Open Database License|^CC-BY-SA|^CC-SA/i
+export const LicenseRefusalKind = {
+	/**
+	 * The operator supplied this license prefix in `--exclude-licenses`.
+	 */
+	OperatorExcluded: "operator-excluded",
+	/**
+	 * The value resolves to an expression whose recorded obligations include share-alike.
+	 */
+	ShareAlikeCarried: "share-alike-carried",
+	/**
+	 * The value's text mentions a share-alike license while resolving to no expression of its own.
+	 */
+	ShareAlikeMentioned: "share-alike-mentioned",
+	/**
+	 * The value resolves to no expression, so its obligations are unknown.
+	 */
+	Unresolved: "unresolved",
+} as const
 
 /**
- * Compile a `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored, case-insensitive prefix
- * patterns. Each entry matches a license string that STARTS with it, so `CC-BY-SA` catches `CC-BY-SA-3.0`,
- * `CC-BY-SA-4.0`, etc. Regex metacharacters are escaped — the spec is a literal license prefix, not a user-supplied
- * regex.
+ * One of the {@linkcode LicenseRefusalKind} values.
+ */
+export type LicenseRefusalKind = (typeof LicenseRefusalKind)[keyof typeof LicenseRefusalKind]
+
+/**
+ * What a license value means for a build, decided once per distinct value.
+ */
+export interface LicenseVerdict {
+	/**
+	 * Why the row was refused, or `null` when the policy admits it.
+	 */
+	refusal: LicenseRefusalKind | null
+	/**
+	 * Whether the value resolves to an expression whose obligations are recorded.
+	 *
+	 * An admitted row reading `false` here entered with unknown obligations.
+	 */
+	resolved: boolean
+	/**
+	 * Whether the value's text mentions a share-alike license, whether or not it was refused for it.
+	 */
+	mentionsShareAlike: boolean
+}
+
+/**
+ * Compile an `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored,
+ * case-insensitive prefix patterns, so `CC-BY-SA` catches `CC-BY-SA-3.0`.
+ *
+ * The spec is a literal license prefix the operator typed.
+ * Regex metacharacters are escaped.
+ *
+ * This is the one place a prefix match is the right reading, because the operator
+ * is naming a spelling rather than asking about an obligation.
  */
 export function compileLicenseExcludes(spec: string): RegExp[] {
-	return spec
-		.split(",")
-		.map((s) => s.trim())
-		.filter((entry) => entry !== "")
-		.map((s) => new RegExp("^" + s.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"))
+	return extractDelimited(spec).map((s) => new RegExp("^" + escapeRegExp(s), "i"))
 }
 
 /**
- * True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded).
+ * True when `license` matches any of the operator's exclude `patterns`; empty patterns never exclude.
  */
 export function licenseExcluded(license: string | undefined, patterns: readonly RegExp[]): boolean {
 	const l = license ?? ""
 
 	return patterns.some((p) => p.test(l))
 }
+
+/**
+ * Reads one license value against a policy.
+ *
+ * This resolves an expression and runs the mention patterns on every call.
+ * A corpus build asks about hundreds of millions of rows holding tens of distinct values,
+ * so a row loop calls {@linkcode createLicenseVerdictCache} instead.
+ */
+export function licenseVerdict(
+	license: string | undefined,
+	policy: LicensePolicy,
+	excluded: readonly RegExp[] = []
+): LicenseVerdict {
+	const record = readLicenseRecord(license)
+	const resolved = record.expression !== null
+	const mentions = mentionsShareAlike(record)
+
+	const verdict = (refusal: LicenseRefusalKind | null): LicenseVerdict => ({
+		refusal,
+		resolved,
+		mentionsShareAlike: mentions || carriesShareAlike(record),
+	})
+
+	if (licenseExcluded(license, excluded)) return verdict(LicenseRefusalKind.OperatorExcluded)
+
+	if (policy === LicensePolicy.All) return verdict(null)
+
+	if (carriesShareAlike(record)) return verdict(LicenseRefusalKind.ShareAlikeCarried)
+
+	if (mentions) return verdict(LicenseRefusalKind.ShareAlikeMentioned)
+
+	if (!resolved && policy === LicensePolicy.ResolvedOnly) return verdict(LicenseRefusalKind.Unresolved)
+
+	return verdict(null)
+}
+
+/**
+ * One license value in a built corpus that a share-alike-free reading refuses, with its row count.
+ */
+export interface ShareAlikeFinding {
+	/**
+	 * The value exactly as the corpus stores it.
+	 */
+	license: string
+	/**
+	 * Rows the built manifest counts under this value.
+	 */
+	rows: number
+	kind: LicenseRefusalKind
+}
+
+/**
+ * Every license value in a built corpus's license set that imposes or mentions share-alike.
+ *
+ * Takes the `licenses` map recorded by `BuildCorpusManifest`.
+ * The map counts the rows adapters yielded.
+ *
+ * A caller about to move or publish a corpus reads this rather than its own list of source ids,
+ * because the obligation is a property of the license value on the row.
+ *
+ * Ordered by row count, most rows first.
+ */
+export function shareAlikeFindings(licenses: Readonly<Record<string, number>>): ShareAlikeFinding[] {
+	const findings: ShareAlikeFinding[] = []
+
+	for (const [license, rows] of Object.entries(licenses)) {
+		const { refusal } = licenseVerdict(license, LicensePolicy.ShareAlikeFree)
+
+		if (refusal) {
+			findings.push({ license, rows, kind: refusal })
+		}
+	}
+
+	return findings.toSorted((a, b) => b.rows - a.rows)
+}
+
+/**
+ * A license reading memoized per distinct raw value, for a loop over corpus rows.
+ */
+export interface LicenseVerdictCache {
+	/**
+	 * The verdict for this value under the policy this cache was built with.
+	 */
+	read(license: string | undefined): LicenseVerdict
+	/**
+	 * Every distinct value refused so far, with the class it was refused under,
+	 * so a build that dropped rows reports which values caused each drop.
+	 */
+	refusedValues(): Map<string, LicenseRefusalKind>
+}
+
+/**
+ * Builds a cache that reads each distinct license value once.
+ *
+ * The corpus holds tens of distinct values over hundreds of millions of rows,
+ * so the memo turns one expression resolution per row into one per value.
+ */
+export function createLicenseVerdictCache(
+	policy: LicensePolicy,
+	excluded: readonly RegExp[] = []
+): LicenseVerdictCache {
+	const verdicts = new Map<string, LicenseVerdict>()
+	const refused = new Map<string, LicenseRefusalKind>()
+
+	return {
+		read(license) {
+			const key = license ?? ""
+			const cached = verdicts.get(key)
+
+			if (cached) return cached
+
+			const verdict = licenseVerdict(license, policy, excluded)
+
+			verdicts.set(key, verdict)
+
+			if (verdict.refusal) {
+				refused.set(key, verdict.refusal)
+			}
+
+			return verdict
+		},
+		refusedValues() {
+			return new Map(refused)
+		},
+	}
+}