@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/tools/lint/recipe-output.ts

@@ -0,0 +1,482 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Corpus linter. Compares a new recipe output (one parquet file) against pre-computed corpus statistics and
+ * flags token-label distribution outliers, label-vacuum tokens, bigram-label collisions, anti-pattern rule
+ * matches and basic sanity problems.
+ *
+ * The markdown report goes to stdout, with optional markdown and JSON sidecars. The command exits 0 when no
+ * errors are found and 1 when any are, because warnings do not refuse. A flagged recipe output should include
+ * `lint_acknowledged: true` in its manifest before training consumes it.
+ */
+
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
+import { writeLocalFile, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
+import { resolvePackagePath } from "@mailwoman/core/module/resolvers"
+
+import {
+	accumulateCooccurrences,
+	createCooccurrenceStats,
+	COOCCURRENCE_KEY_SEP as SEP,
+	streamTokenLabelRows,
+	type TokenLabelRow as RecipeOutputRow,
+} from "#utils/cooccurrence-stats"
+
+/**
+ * Occurrences of a forbidden label before it is reported: one or two are noise, five is a pattern.
+ */
+const FORBIDDEN_LABEL_REPORT_THRESHOLD = 5
+
+/**
+ * Examples printed per finding before the list is truncated.
+ */
+const MAX_LISTED_EXAMPLES = 20
+
+/**
+ * Calibrated thresholds, tunable if new failure modes surface that the current numbers miss.
+ */
+const CORPUS_CONFIDENCE_FLOOR = 0.66
+const OUTPUT_MIN_COUNT = 50
+const CORPUS_MIN_COUNT = 200
+const VACUUM_OUTPUT_MIN_COUNT = 20
+const VACUUM_CORPUS_MIN_COUNT = 100
+const BIGRAM_MIN_COUNT = 10
+const ALL_O_RATIO_CEILING = 0.9
+
+/**
+ * Default `lint-rules.json` path — the rules ship beside this module in the source tree.
+ *
+ * Tsc does not emit readFileSync'd JSON into `out/`, so the compiled tree falls back
+ * to the source-tree copy (corpus/out/src/tools/ → corpus/src/tools/).
+ *
+ * In-repo the `node` exports condition loads this module from source anyway,
+ * so the sibling URL is the common path.
+ */
+function defaultRulesPath(): string {
+	return resolvePackagePath("@mailwoman/corpus", "lib", "tools", "lint-rules.json")
+}
+
+/**
+ * Options for {@linkcode lintRecipeOutput}.
+ */
+export interface LintRecipeOutputOptions {
+	/**
+	 * The new recipe output parquet to lint.
+	 */
+	recipeOutputPath: string
+	/**
+	 * Pre-computed corpus stats JSON (see `corpus-stats.ts`).
+	 */
+	statsPath: string
+	/**
+	 * Anti-pattern rules JSON.
+	 *
+	 * Default: the `lint-rules.json` beside this module.
+	 */
+	rulesPath?: string
+	/**
+	 * Write the markdown report here as well as stdout.
+	 */
+	outMd?: string
+	/**
+	 * Write a JSON sidecar of the flags + summary here.
+	 */
+	outJSON?: string
+}
+
+interface CorpusStats {
+	row_count: number
+	slice_paths: string[]
+	tokens: Record<string, Record<string, number>>
+	bigrams: Record<string, Record<string, number>>
+}
+
+interface LintRule {
+	id: string
+	pattern: string
+	pattern_case_sensitive: boolean
+	forbidden_labels: string[]
+	message: string
+	severity: "error" | "warn"
+}
+
+interface LintRulesFile {
+	rules: LintRule[]
+}
+
+interface RecipeOutputStats {
+	rowCount: number
+	tokens: Map<string, Map<string, number>>
+	bigrams: Map<string, Map<string, number>>
+	truncatedRows: number
+	allORows: number
+}
+
+async function statsFromRecipeOutput(rows: AsyncIterable<RecipeOutputRow>): Promise<RecipeOutputStats> {
+	const co = createCooccurrenceStats()
+
+	const out: RecipeOutputStats = {
+		rowCount: 0,
+		tokens: co.tokens,
+		bigrams: co.bigrams,
+		truncatedRows: 0,
+		allORows: 0,
+	}
+
+	for await (const row of rows) {
+		out.rowCount++
+
+		if (row.tokens.length !== row.labels.length) {
+			out.truncatedRows++
+
+			continue
+		}
+
+		if (row.labels.every((l) => l === "O")) {
+			out.allORows++
+		}
+
+		accumulateCooccurrences(co, row.tokens, row.labels)
+	}
+
+	return out
+}
+
+function majorityLabel(distribution: Map<string, number> | Record<string, number>): {
+	label: string
+	count: number
+	total: number
+	confidence: number
+} {
+	const entries = distribution instanceof Map ? [...distribution.entries()] : Object.entries(distribution)
+	let bestLabel = ""
+	let bestCount = 0
+	let total = 0
+
+	for (const [label, count] of entries) {
+		total += count
+
+		if (count > bestCount) {
+			bestCount = count
+			bestLabel = label
+		}
+	}
+
+	return { label: bestLabel, count: bestCount, total, confidence: total === 0 ? 0 : bestCount / total }
+}
+
+/**
+ * One lint flag emitted by a check.
+ */
+export interface LintFlag {
+	check: string
+	severity: "error" | "warn"
+	token?: string
+	bigram?: string
+	outputLabel?: string
+	corpusLabel?: string
+	outputCount?: number
+	corpusCount?: number
+	detail: string
+	ruleID?: string
+}
+
+/**
+ * Findings summary returned by {@linkcode lintRecipeOutput}.
+ */
+export interface LintRecipeOutputSummary {
+	errors: number
+	warnings: number
+	findings: LintFlag[]
+	/**
+	 * The rendered markdown report (also printed to stdout).
+	 */
+	report: string
+}
+
+function checkDistributionOutliers(output: RecipeOutputStats, corpus: CorpusStats): LintFlag[] {
+	const flags: LintFlag[] = []
+
+	for (const [token, outputLabelMap] of output.tokens) {
+		const corpusLabelMap = corpus.tokens[token]
+
+		if (!corpusLabelMap) continue
+		const outputMaj = majorityLabel(outputLabelMap)
+		const corpusMaj = majorityLabel(corpusLabelMap)
+
+		if (
+			corpusMaj.confidence >= CORPUS_CONFIDENCE_FLOOR &&
+			outputMaj.label !== corpusMaj.label &&
+			outputMaj.count >= OUTPUT_MIN_COUNT &&
+			corpusMaj.total >= CORPUS_MIN_COUNT
+		) {
+			flags.push({
+				check: "distribution-outlier",
+				severity: "error",
+				token,
+				outputLabel: outputMaj.label,
+				corpusLabel: corpusMaj.label,
+				outputCount: outputMaj.count,
+				corpusCount: corpusMaj.count,
+				detail: `Token "${token}": recipe-output majority is ${outputMaj.label} (${outputMaj.count}/${outputMaj.total}, ${(outputMaj.confidence * 100).toFixed(0)}%), corpus majority is ${corpusMaj.label} (${corpusMaj.count}/${corpusMaj.total}, ${(corpusMaj.confidence * 100).toFixed(0)}%).`,
+			})
+		}
+	}
+
+	return flags
+}
+
+function checkLabelVacuum(output: RecipeOutputStats, corpus: CorpusStats): LintFlag[] {
+	const flags: LintFlag[] = []
+
+	for (const [token, outputLabelMap] of output.tokens) {
+		const corpusLabelMap = corpus.tokens[token]
+
+		if (!corpusLabelMap) continue
+		const corpusTotal = Object.values(corpusLabelMap).reduce((a, b) => a + b, 0)
+
+		if (corpusTotal < VACUUM_CORPUS_MIN_COUNT) continue
+
+		for (const [label, outputCount] of outputLabelMap) {
+			if (outputCount < VACUUM_OUTPUT_MIN_COUNT) continue
+
+			if (corpusLabelMap[label] === undefined || corpusLabelMap[label] === 0) {
+				flags.push({
+					check: "label-vacuum",
+					severity: "error",
+					token,
+					outputLabel: label,
+					outputCount,
+					corpusCount: corpusTotal,
+					detail: `Token "${token}": the recipe output labels it ${label} ${outputCount} times, but the corpus (${corpusTotal} instances of this token) has ZERO instances of this label.`,
+				})
+			}
+		}
+	}
+
+	return flags
+}
+
+function checkBigramCollisions(output: RecipeOutputStats, corpus: CorpusStats): LintFlag[] {
+	const flags: LintFlag[] = []
+
+	for (const [bigram, outputLabelMap] of output.bigrams) {
+		const corpusLabelMap = corpus.bigrams[bigram]
+
+		if (!corpusLabelMap) continue
+		const outputMaj = majorityLabel(outputLabelMap)
+		const corpusMaj = majorityLabel(corpusLabelMap)
+
+		if (
+			outputMaj.label !== corpusMaj.label &&
+			outputMaj.count >= BIGRAM_MIN_COUNT &&
+			corpusMaj.count >= BIGRAM_MIN_COUNT
+		) {
+			const renderBigram = bigram.split(SEP).join(" ")
+			const renderOutputLabel = outputMaj.label.split(SEP).join(" → ")
+			const renderCorpusLabel = corpusMaj.label.split(SEP).join(" → ")
+
+			flags.push({
+				check: "bigram-collision",
+				severity: "error",
+				bigram: renderBigram,
+				outputLabel: renderOutputLabel,
+				corpusLabel: renderCorpusLabel,
+				outputCount: outputMaj.count,
+				corpusCount: corpusMaj.count,
+				detail: `Bigram "${renderBigram}": recipe-output label-bigram is [${renderOutputLabel}] (${outputMaj.count}×), corpus label-bigram is [${renderCorpusLabel}] (${corpusMaj.count}×). Same surface text, different structural reading.`,
+			})
+		}
+	}
+
+	return flags
+}
+
+function checkRules(output: RecipeOutputStats, rulesFile: LintRulesFile): LintFlag[] {
+	const flags: LintFlag[] = []
+
+	const compiled = rulesFile.rules.map((r) => ({
+		rule: r,
+		regex: new RegExp(r.pattern, r.pattern_case_sensitive ? "" : "i"),
+	}))
+
+	for (const [token, labelMap] of output.tokens) {
+		for (const { rule, regex } of compiled) {
+			if (!regex.test(token)) continue
+
+			for (const [label, count] of labelMap) {
+				if (rule.forbidden_labels.includes(label) && count >= FORBIDDEN_LABEL_REPORT_THRESHOLD) {
+					flags.push({
+						check: "anti-pattern-rule",
+						severity: rule.severity,
+						ruleID: rule.id,
+						token,
+						outputLabel: label,
+						outputCount: count,
+						detail: `Token "${token}" matched rule ${rule.id} and is labeled ${label} ${count} time(s). Rule message: ${rule.message}`,
+					})
+				}
+			}
+		}
+	}
+
+	return flags
+}
+
+function checkSanity(output: RecipeOutputStats): LintFlag[] {
+	const flags: LintFlag[] = []
+
+	if (output.truncatedRows > 0) {
+		flags.push({
+			check: "truncated-rows",
+			severity: "error",
+			detail: `${output.truncatedRows} row(s) have tokens.length !== labels.length. Pipeline alignment bug.`,
+		})
+	}
+
+	const allORatio = output.allORows / Math.max(1, output.rowCount)
+
+	if (allORatio >= ALL_O_RATIO_CEILING) {
+		flags.push({
+			check: "all-O-output",
+			severity: "warn",
+			detail: `${output.allORows}/${output.rowCount} rows (${(allORatio * 100).toFixed(0)}%) are entirely O-labeled. The recipe output contributes no signal.`,
+		})
+	}
+
+	return flags
+}
+
+function renderReport(
+	opts: { outputPath: string; statsPath: string; rulesPath: string },
+	output: RecipeOutputStats,
+	flags: LintFlag[]
+): string {
+	const errors = flags.filter((f) => f.severity === "error")
+	const warns = flags.filter((f) => f.severity === "warn")
+	const verdict = !errors.length ? "**PASS** ✓" : "**FLAGGED** ⚠"
+
+	const lines: string[] = [
+		`# Corpus Lint: ${verdict}`,
+		"",
+		`- **Recipe output:** \`${opts.outputPath}\``,
+		`- **Corpus stats:** \`${opts.statsPath}\``,
+		`- **Rules:** \`${opts.rulesPath}\``,
+		`- **Recipe output rows:** ${output.rowCount}`,
+		`- **Unique tokens:** ${output.tokens.size}`,
+		`- **Unique bigrams:** ${output.bigrams.size}`,
+		"",
+		`**Errors:** ${errors.length} (refuses the recipe output's inclusion unless MANIFEST sets \`lint_acknowledged: true\`)`,
+		`**Warnings:** ${warns.length} (advisory)`,
+		"",
+	]
+
+	if (!flags.length) {
+		lines.push("No anomalies detected.")
+
+		return lines.join("\n")
+	}
+
+	const byCheck = new Map<string, LintFlag[]>()
+
+	for (const f of flags) {
+		const arr = byCheck.get(f.check) ?? []
+		arr.push(f)
+		byCheck.set(f.check, arr)
+	}
+
+	for (const [check, list] of byCheck) {
+		lines.push(`## ${check} (${list.length})`)
+		lines.push("")
+		// Highest-volume issues first.
+		list.sort((a, b) => (b.outputCount ?? 0) - (a.outputCount ?? 0))
+
+		for (const f of list.slice(0, 20)) {
+			lines.push(`- **[${f.severity.toUpperCase()}]** ${f.detail}`)
+		}
+
+		if (list.length > MAX_LISTED_EXAMPLES) {
+			lines.push(`- ... and ${list.length - 20} more`)
+		}
+
+		lines.push("")
+	}
+
+	return lines.join("\n")
+}
+
+/**
+ * Lint a recipe output against corpus stats and the anti-pattern rules,
+ * printing the markdown report to stdout.
+ */
+export async function lintRecipeOutput(
+	options: LintRecipeOutputOptions,
+	report?: (line: string) => void
+): Promise<LintRecipeOutputSummary> {
+	const rulesPath = options.rulesPath ?? defaultRulesPath()
+	report?.(`Reading corpus stats from ${options.statsPath}...`)
+	const corpus = await readLocalJSONFile<CorpusStats>(options.statsPath)
+
+	report?.(
+		`  ${corpus.row_count} rows from ${corpus.slice_paths.length} parquet file(s); ${Object.keys(corpus.tokens).length} tokens, ${Object.keys(corpus.bigrams).length} bigrams`
+	)
+
+	report?.(`Reading recipe output from ${options.recipeOutputPath}...`)
+
+	const output = await statsFromRecipeOutput(streamTokenLabelRows(options.recipeOutputPath))
+
+	report?.(`  ${output.rowCount} rows`)
+
+	report?.(`Loading rules from ${rulesPath}...`)
+	const rulesFile = await readLocalJSONFile<LintRulesFile>(rulesPath)
+
+	report?.(`Running checks...`)
+
+	const flags: LintFlag[] = [
+		...checkDistributionOutliers(output, corpus),
+		...checkLabelVacuum(output, corpus),
+		...checkBigramCollisions(output, corpus),
+		...checkRules(output, rulesFile),
+		...checkSanity(output),
+	]
+
+	const rendered = renderReport(
+		{ outputPath: options.recipeOutputPath, statsPath: options.statsPath, rulesPath },
+		output,
+		flags
+	)
+
+	console.log(rendered)
+
+	if (options.outMd) {
+		await writeLocalFile(rendered, options.outMd)
+	}
+
+	if (options.outJSON) {
+		await writeLocalJSONFile(
+			{
+				recipe_output: options.recipeOutputPath,
+				stats: options.statsPath,
+				flags,
+				summary: {
+					errors: flags.filter((f) => f.severity === "error").length,
+					warnings: flags.filter((f) => f.severity === "warn").length,
+				},
+			},
+			options.outJSON
+		)
+	}
+
+	const errorCount = flags.filter((f) => f.severity === "error").length
+	const warningCount = flags.filter((f) => f.severity === "warn").length
+
+	if (errorCount > 0) {
+		report?.(`LINT FAILED: ${errorCount} error(s).`)
+	} else {
+		report?.("LINT PASSED.")
+	}
+
+	return { errors: errorCount, warnings: warningCount, findings: flags, report: rendered }
+}

package/lib/tools/migrate/overlay-parquet.ts

@@ -0,0 +1,173 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Convert an overlay parquet to the current row schema, without its recipe output.
+ *   `mailwoman-derived-tuples` wherever the upstream was not recorded.
+ */
+
+import { openWriteStream } from "@mailwoman/core/fs/streams"
+import { temporaryDirectory } from "@mailwoman/core/fs/temporary"
+import { stringifyJSON } from "@mailwoman/core/json"
+import { once } from "@mailwoman/core/utils/events"
+import type { PathBuilderLike } from "path-ts"
+
+import { openParquetRowStream } from "#parquet/streams"
+import { retiredSourceName } from "#recipes/sources"
+import { SourceRegister } from "#registers"
+import { jsonlToParquet } from "#tools/jsonl-to-parquet"
+import { surfaceForSource } from "#tools/migrate/recipe-output"
+
+/**
+ * The register each overlay source's rows came from, or `mailwoman-derived-tuples`
+ * when the upstream was not recorded.
+ */
+export const OVERLAY_REGISTERS: Record<string, string | null> = {
+	// The recipe names its publication.
+	"synth-gb": SourceRegister.LandRegistryPricePaid,
+	"synth-german": SourceRegister.OpenAddresses,
+	"synth-nz": SourceRegister.OpenAddresses,
+	"synth-es": SourceRegister.OpenAddresses,
+	"synth-unit": SourceRegister.OpenAddresses,
+	"synth-unit-v30": SourceRegister.OpenAddresses,
+	"synth-country": SourceRegister.OpenAddresses,
+	"synth-fr-order": SourceRegister.BaseAdresseNationale,
+	"synth-fr-admin-split": SourceRegister.BaseAdresseNationale,
+	"synth-fr-bare-street": SourceRegister.BaseAdresseNationale,
+	"synth-fr-fragment": SourceRegister.BaseAdresseNationale,
+	"synth-fr-lieudit": SourceRegister.BaseAdresseNationale,
+	overture: SourceRegister.Overture,
+	"overture-latam": SourceRegister.Overture,
+	"synth-sg-register": SourceRegister.Overture,
+	gnaf: SourceRegister.GNAF,
+	osm: SourceRegister.OpenStreetMap,
+	"synth-pk-register": SourceRegister.OpenStreetMap,
+	"synth-bd-register": SourceRegister.OpenStreetMap,
+	"synth-bare-country": SourceRegister.Codex,
+	"synth-bare-postcode": SourceRegister.Codex,
+	"synth-reviewed-postcode-tail": SourceRegister.ReviewedByHand,
+	// The row names no published record at all.
+	"synth-po-box-cedex": null,
+	"synth-po-box-military": null,
+	"synth-intersection": null,
+	"synth-house-venue": null,
+	"deepseek-kryptonite": null,
+	"synth-anchor-absorption": null,
+	// The tuple extraction behind these was not recorded.
+	"synth-affix": SourceRegister.DerivedTuples,
+	"synth-suffix-boundary": SourceRegister.DerivedTuples,
+	"synth-cz-pcfirst-preposition": SourceRegister.DerivedTuples,
+	"synth-cz-pcfirst-preposition-v21": SourceRegister.DerivedTuples,
+	"synth-fr-bare-street-v22": SourceRegister.DerivedTuples,
+	"synth-si-bare-village": SourceRegister.DerivedTuples,
+	"synth-no-street-led": SourceRegister.DerivedTuples,
+	"synth-no-fragment": SourceRegister.DerivedTuples,
+	"synth-fragment": SourceRegister.DerivedTuples,
+	"synth-nl-postcode": SourceRegister.DerivedTuples,
+	"synth-sub-venue": SourceRegister.DerivedTuples,
+	"synth-trailing-region": SourceRegister.DerivedTuples,
+	"synth-trailing-region-us": SourceRegister.DerivedTuples,
+	"synth-trailing-region-ca-bare": SourceRegister.DerivedTuples,
+	"synth-trailing-region-structured": SourceRegister.DerivedTuples,
+	"synth-trailing-region-es-v28": SourceRegister.DerivedTuples,
+}
+
+/**
+ * Read an INT64 span offset back as a plain number.
+ *
+ * DuckDB returns INT64 values as `BigInt`.
+ * `stringifyJSON` refuses `BigInt` values.
+ *
+ * The current schema declares the span triple INT32 because `raw` is a short
+ * address string, so every in-range value is lossless.
+ *
+ * A value outside the safe-integer range is a corrupt offset, so this raises instead of truncating.
+ */
+function narrowBigInts(row: Record<string, unknown>): Record<string, unknown> {
+	const narrow = (value: unknown): unknown => {
+		if (typeof value === "bigint") {
+			if (value > BigInt(Number.MAX_SAFE_INTEGER) || value < BigInt(Number.MIN_SAFE_INTEGER)) {
+				throw new Error(`span offset ${value} is outside the safe-integer range, so it is a corrupt offset`)
+			}
+
+			return Number(value)
+		}
+
+		if (Array.isArray(value)) return value.map(narrow)
+
+		return value
+	}
+
+	return Object.fromEntries(Object.entries(row).map(([key, value]) => [key, narrow(value)]))
+}
+
+/**
+ * What a parquet migration observed.
+ */
+export interface OverlayMigrationSummary {
+	rows: number
+	bySource: Record<string, number>
+}
+
+/**
+ * The register for one overlay source, refusing a source this map does not name.
+ */
+export function registerForSource(source: string): string | null {
+	// The map keys on the retired spelling.
+	// A corpus that uses the current spelling needs the retired key for this lookup.
+	// Otherwise each row fails as though it had no register.
+	// That failure misstates a renamed source as an unrecorded one.
+	const key = retiredSourceName(source)
+
+	if (!(key in OVERLAY_REGISTERS)) {
+		throw new Error(
+			`No register recorded for overlay source ${stringifyJSON(source)}. Read the recipe that writes it and ` +
+				`add it to OVERLAY_REGISTERS: the publication it renders, or null when the row names no record, or ` +
+				`mailwoman-derived-tuples when its tuple extraction was not recorded.`
+		)
+	}
+
+	return OVERLAY_REGISTERS[key] ?? null
+}
+
+/**
+ * Rewrite one overlay parquet onto the current row schema, staged through
+ * {@linkcode jsonlToParquet} so it has the same column list as every other overlay.
+ */
+export async function migrateOverlayParquet(
+	input: PathBuilderLike,
+	output: PathBuilderLike
+): Promise<OverlayMigrationSummary> {
+	const summary: OverlayMigrationSummary = { rows: 0, bySource: {} }
+
+	await using staging = await temporaryDirectory("mw-migrate-overlay-")
+	const stagePath = staging.path("rows.jsonl")
+	const sink = staging.use(openWriteStream(stagePath, { encoding: "utf8" }))
+
+	for await (const row of openParquetRowStream<Record<string, unknown>>(input)) {
+		const { synth_method: recipe, synth_base_id: baseSourceID, ...rest } = narrowBigInts(row)
+		const source = String(row["source"] ?? "")
+
+		summary.rows++
+		summary.bySource[source] = (summary.bySource[source] ?? 0) + 1
+
+		const migrated = {
+			...rest,
+			recipe: recipe ?? null,
+			base_source_id: baseSourceID ?? null,
+			register: registerForSource(source),
+			surface: surfaceForSource(source),
+		}
+
+		if (!sink.write(`${stringifyJSON(migrated)}\n`)) {
+			await once(sink, "drain")
+		}
+	}
+
+	sink.end()
+	await once(sink, "finish")
+
+	await jsonlToParquet({ input: stagePath, output })
+
+	return summary
+}