@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/tools/migrate/recipe-output.ts

@@ -0,0 +1,153 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Convert a recipe output written before `recipe`/`register`/`surface` to the current row schema.
+ *
+ * `recipe` and `base_source_id` are renames of `synth_method` and `synth_base_id`, so they assert no new claim.
+ * `register` is `mailwoman-derived-tuples` for every row, because naming a publisher would record inference as fact.
+ */
+
+import { openWriteStream } from "@mailwoman/core/fs/streams"
+import { parseJSONStrict, stringifyJSON } from "@mailwoman/core/json"
+import { once } from "@mailwoman/core/utils/events"
+import type { PathBuilderLike } from "path-ts"
+import { TextSpliterator } from "spliterator"
+
+import { retiredSourceName } from "#recipes/sources"
+import { SourceRegister } from "#registers"
+import { SurfaceOrigin } from "#types"
+
+/**
+ * How each source id's recipe produced its surface.
+ *
+ * A source absent from this table refuses rather than taking a default.
+ */
+export const RECIPE_SURFACES: Record<string, SurfaceOrigin> = {
+	// Assembled from template tables.
+	// No register asserts the address exists.
+	"synth-intersection": SurfaceOrigin.Invented,
+	"synth-po-box-cedex": SurfaceOrigin.Invented,
+	"synth-house-venue": SurfaceOrigin.Invented,
+	"deepseek-kryptonite": SurfaceOrigin.Invented,
+	// A real record's fields, written in an order the recipe chose.
+	"synth-affix": SurfaceOrigin.Composed,
+	"synth-fr-order": SurfaceOrigin.Composed,
+	"synth-fr-bare-street": SurfaceOrigin.Composed,
+	"synth-fr-lieudit": SurfaceOrigin.Composed,
+	"synth-suffix-boundary": SurfaceOrigin.Composed,
+	"synth-unit": SurfaceOrigin.Composed,
+	"synth-unit-v30": SurfaceOrigin.Composed,
+	"synth-trailing-region": SurfaceOrigin.Composed,
+	"synth-trailing-region-us": SurfaceOrigin.Composed,
+	"synth-trailing-region-structured": SurfaceOrigin.Composed,
+	"synth-trailing-region-ca-bare": SurfaceOrigin.Composed,
+	"synth-trailing-region-es-v28": SurfaceOrigin.Composed,
+	"synth-cz-pcfirst-preposition": SurfaceOrigin.Composed,
+	"synth-cz-pcfirst-preposition-v21": SurfaceOrigin.Composed,
+	"synth-fr-bare-street-v22": SurfaceOrigin.Composed,
+	"synth-si-bare-village": SurfaceOrigin.Composed,
+	"synth-no-street-led": SurfaceOrigin.Composed,
+	"synth-no-fragment": SurfaceOrigin.Composed,
+	"synth-fr-fragment": SurfaceOrigin.Composed,
+	"synth-fragment": SurfaceOrigin.Composed,
+	"synth-fr-admin-split": SurfaceOrigin.Composed,
+	"synth-nl-postcode": SurfaceOrigin.Composed,
+	"synth-sub-venue": SurfaceOrigin.Composed,
+	"synth-german": SurfaceOrigin.Composed,
+	"synth-nz": SurfaceOrigin.Composed,
+	"synth-es": SurfaceOrigin.Composed,
+	"synth-gb": SurfaceOrigin.Composed,
+	"synth-sg-register": SurfaceOrigin.Composed,
+	"synth-pk-register": SurfaceOrigin.Composed,
+	"synth-bd-register": SurfaceOrigin.Composed,
+	"synth-reviewed-postcode-tail": SurfaceOrigin.Composed,
+	// The publisher's own string, preserved by an adapter rather than a recipe.
+	overture: SurfaceOrigin.Attested,
+	"overture-latam": SurfaceOrigin.Attested,
+	gnaf: SurfaceOrigin.Attested,
+	osm: SurfaceOrigin.Attested,
+	// Written from this repository's own tables.
+	// They publish the string verbatim.
+	"synth-bare-country": SurfaceOrigin.Attested,
+	"synth-bare-postcode": SurfaceOrigin.Attested,
+	// Drawn rather than read: the box number, the anchor and the venue are the recipe's.
+	"synth-po-box-military": SurfaceOrigin.Invented,
+	"synth-anchor-absorption": SurfaceOrigin.Invented,
+}
+
+/**
+ * What a migration pass observed.
+ */
+export interface MigrationSummary {
+	rows: number
+	bySource: Record<string, number>
+	alreadyMigrated: number
+}
+
+/**
+ * The surface for one source id, refusing a source the table does not name.
+ */
+export function surfaceForSource(source: string): SurfaceOrigin {
+	// The table keys on the retired spelling.
+	// A corpus that uses the current spelling needs the retired key for this lookup.
+	// Otherwise each row fails as though it had no surface.
+	// That failure misstates a renamed source as an unrecorded one.
+	const surface = RECIPE_SURFACES[retiredSourceName(source)]
+
+	if (!surface) {
+		throw new Error(
+			`No surface recorded for source ${stringifyJSON(source)}. Read the recipe that writes it, ` +
+				`add it to RECIPE_SURFACES, and say there whether it renders a real record or assembles one.`
+		)
+	}
+
+	return surface
+}
+
+/**
+ * Rewrite one recipe output onto the current row schema, passing an already-migrated
+ * row through untouched so a half-migrated file converges.
+ */
+export async function migrateRecipeOutput(input: PathBuilderLike, output: PathBuilderLike): Promise<MigrationSummary> {
+	const summary: MigrationSummary = { rows: 0, bySource: {}, alreadyMigrated: 0 }
+	const sink = openWriteStream(output, { encoding: "utf8" })
+
+	for await (const line of TextSpliterator.fromAsync(input)) {
+		const row = parseJSONStrict<Record<string, unknown>>(line)
+
+		summary.rows++
+
+		const source = String(row["source"] ?? "")
+		summary.bySource[source] = (summary.bySource[source] ?? 0) + 1
+
+		if (row["surface"]) {
+			summary.alreadyMigrated++
+
+			if (!sink.write(`${line}\n`)) {
+				await once(sink, "drain")
+			}
+
+			continue
+		}
+
+		const { synth_method: recipe, synth_base_id: baseSourceID, ...rest } = row
+
+		const migrated = {
+			...rest,
+			recipe: recipe ?? null,
+			base_source_id: baseSourceID ?? null,
+			register: SourceRegister.DerivedTuples,
+			surface: surfaceForSource(source),
+		}
+
+		if (!sink.write(`${stringifyJSON(migrated)}\n`)) {
+			await once(sink, "drain")
+		}
+	}
+
+	sink.end()
+	await once(sink, "finish")
+
+	return summary
+}

package/lib/tools/migrate/source-names.ts

@@ -0,0 +1,161 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Rewrite a parquet file's `source` column from the retired spelling to the current one.
+ *
+ *   A recipe-output `source` is a wire identifier. Every row of every corpus carrying it stores the value.
+ *   Every training config that weights it keys on it. Every model card trained on it quotes it. The
+ *   retired `synth-*` spelling labels a row fabricated where most of these rows are attested records a
+ *   recipe rendered, spliced or fragmented. `RECIPE_SOURCES` in `#recipes/sources` holds both spellings and
+ *   the operation each recipe performed.
+ *
+ *   The rewrite runs over an assembly's staging files before `overlay-manifest` computes each file's
+ *   `sha256`, so the manifest describes the renamed bytes. A corpus already assembled is never rewritten in
+ *   place. Its manifest records digests over the bytes it has. The configs that target it keep the
+ *   spelling it stores.
+ *
+ *   **What the verification establishes.** The check compares row count, distinct `source` set and an md5
+ *   over the ordered `source_id` column before and after. Together, those checks establish that the rewrite
+ *   changed the `source` column and the row order of no other column. They do not establish that the
+ *   output's other columns are unchanged. A full-file digest cannot show that either because the rewrite
+ *   re-encodes the file.
+ */
+
+import { md5Hex } from "@mailwoman/core/hash"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { escapeSQLIdentifier, escapeSQLString, openDuckDB } from "#parquet/duckdb"
+import { ROW_GROUP_SIZE } from "#parquet/schema"
+import { currentSourceName } from "#recipes/sources"
+
+/**
+ * What one file's rewrite established.
+ */
+export interface SourceNameRewrite {
+	parquet: string
+	rows: number
+	/**
+	 * The mapping applied, keyed by the retired spelling.
+	 *
+	 * A file whose rows already use current spellings is reported with an empty map and left untouched.
+	 */
+	renamed: Record<string, string>
+	/**
+	 * The md5 over the ordered `source_id` column, identical before and after.
+	 */
+	sourceIDDigest: string
+}
+
+/**
+ * Read the distinct `source` values and the ordered-`source_id` digest of one parquet file.
+ */
+async function readSourceFacts(
+	db: Awaited<ReturnType<typeof openDuckDB>>,
+	path: string
+): Promise<{ rows: number; sources: string[]; digest: string }> {
+	const literal = escapeSQLString(path)
+
+	const counted = await db.runAndReadAll(`SELECT COUNT(*) AS rows FROM read_parquet('${literal}')`)
+	const rows = Number((counted.getRowObjectsJS()[0] as { rows: bigint | number }).rows)
+
+	const distinct = await db.runAndReadAll(`SELECT DISTINCT source FROM read_parquet('${literal}') ORDER BY source`)
+
+	const sources = (distinct.getRowObjectsJS() as Array<{ source: string }>).map((row) => row.source)
+
+	// The digest reads `source_id` in file order rather than sorted, so a rewrite
+	// that reordered rows produces a different value.
+	// `preserve_insertion_order` keeps the order.
+	// This check verifies that it did.
+	const ids = await db.runAndReadAll(`SELECT source_id FROM read_parquet('${literal}')`)
+	const digest = md5Hex((ids.getRowObjectsJS() as Array<{ source_id: string }>).map((row) => row.source_id).join("\n"))
+
+	return { rows, sources, digest }
+}
+
+/**
+ * Rewrite one parquet file's `source` column in place, through a temporary output.
+ *
+ * @throws When a row has a `source` the table does not name under either spelling.
+ * An unknown value is refused rather than preserved, because preserving it would leave one
+ * file holding both spellings and no later reader could tell which rows were rewritten.
+ * @throws When the row count, the distinct `source` count or the ordered-`source_id`
+ * digest differs after the rewrite.
+ */
+export async function rewriteSourceNames(parquetPath: PathBuilderLike): Promise<SourceNameRewrite> {
+	const path = PathBuilder.from(parquetPath)
+	const pathString = path.toString()
+
+	using db = await openDuckDB()
+
+	await db.run("SET preserve_insertion_order=true")
+
+	const before = await readSourceFacts(db, pathString)
+	const renamed: Record<string, string> = {}
+	const unknown: string[] = []
+
+	for (const source of before.sources) {
+		const current = currentSourceName(source)
+
+		if (!current) {
+			unknown.push(source)
+
+			continue
+		}
+
+		if (current !== source) {
+			renamed[source] = current
+		}
+	}
+
+	if (unknown.length) {
+		throw new Error(
+			`${pathString} carries ${unknown.length} source value(s) that RECIPE_SOURCES names under neither ` +
+				`spelling: ${unknown.join(", ")}. Add each to packages/corpus/lib/recipes/sources.ts, or to ` +
+				`CARRIED_SOURCES where the name is already current, before rewriting this file.`
+		)
+	}
+
+	if (!Object.keys(renamed).length) {
+		return { parquet: pathString, rows: before.rows, renamed, sourceIDDigest: before.digest }
+	}
+
+	const cases = Object.entries(renamed)
+		.map(([retired, current]) => `WHEN '${escapeSQLString(retired)}' THEN '${escapeSQLString(current)}'`)
+		.join(" ")
+
+	const staged = path.dirname()(`${path.basename()}.renamed`)
+	const stagedString = staged.toString()
+
+	await db.run(
+		`COPY (SELECT * REPLACE (CASE source ${cases} ELSE source END AS ${escapeSQLIdentifier("source")}) ` +
+			`FROM read_parquet('${escapeSQLString(pathString)}')) ` +
+			`TO '${escapeSQLString(stagedString)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${ROW_GROUP_SIZE})`
+	)
+
+	const after = await readSourceFacts(db, stagedString)
+
+	if (after.rows !== before.rows) {
+		throw new Error(
+			`${pathString}: the rewrite wrote ${after.rows.toLocaleString()} rows and the input holds ` +
+				`${before.rows.toLocaleString()}. ${stagedString} is left in place for inspection.`
+		)
+	}
+
+	if (after.sources.length !== before.sources.length) {
+		throw new Error(
+			`${pathString}: the input holds ${before.sources.length} distinct source values and the rewrite ` +
+				`wrote ${after.sources.length}. Two retired names mapping to one current name would do that, and it ` +
+				`makes the two sources indistinguishable. ${stagedString} is left in place for inspection.`
+		)
+	}
+
+	if (after.digest !== before.digest) {
+		throw new Error(
+			`${pathString}: the ordered source_id digest changed from ${before.digest} to ${after.digest}, so the ` +
+				`rewrite reordered rows. ${stagedString} is left in place for inspection.`
+		)
+	}
+
+	return { parquet: pathString, rows: before.rows, renamed, sourceIDDigest: before.digest }
+}

package/lib/tools/{slice → overlay}/kryptonite.ts

@@ -3,44 +3,48 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Build a parquet slice from the DeepSeek-generated kryptonite JSONL and emit the corpus-v0.4.0
- *   MANIFEST. corpus-v0.4.0 is a pure adapter-addition revision: it points at every slice from
- *   v0.3.0 plus the new kryptonite slice(s). No v0.3.0 bytes are touched or re-shuffled.
+ *   Build a parquet file from the DeepSeek-generated kryptonite jsonl and emit the corpus-v0.4.0
+ *   manifest. corpus-v0.4.0 is a pure adapter-addition revision: it points at every parquet file from
+ *   v0.3.0 plus the new kryptonite file(s). No v0.3.0 bytes are touched or re-shuffled.
  *
- *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for the why; that doc also pins the
- *   DeepSeek model version + prompt versions used to produce the JSONL.
+ *   See docs/records/engineering/CORPUS_V0_4_0_GENERATION.mdx for the why. that doc also pins the
+ *   DeepSeek model version + prompt versions used to produce the jsonl.
  *
  *   Invoke via `mailwoman corpus slice kryptonite \
  *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
- *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/MANIFEST.json \
+ *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/manifest.json \
  *   --out-dir /data/corpus/versioned/v0.4.0`
  */
 
-import { pathExists, readLocalJSONFile } from "@mailwoman/core/fs/readers"
+import { delimitedSource } from "@mailwoman/core/fs/delimited"
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
+import { pathExists } from "@mailwoman/core/fs/readers/stat"
 import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers"
-import { join } from "path-ts"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
 import { JSONSpliterator } from "spliterator"
 
-import type { CanonicalRow, LabeledRow } from "#types"
-import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, SLICE_COMPRESSION, writeSlices } from "#utils"
-import type { SliceManifest } from "#utils"
+import { corpusDirectoryName } from "#directory"
+import { PARQUET_COLUMNS, PARQUET_COMPRESSION, ROW_GROUP_SIZE } from "#parquet/schema"
+import { type ParquetManifest, writeParquetSplits } from "#parquet/writers"
+import { type CanonicalRow, type LabeledRow, requireSurface } from "#types"
+import { alignRow } from "#utils"
 
-export interface SliceKryptoniteOptions {
+export interface KryptoniteOverlayOptions {
 	jsonl: string
 	baseManifest: string
-	outDir: string
+	outDir: PathBuilderLike
 	/**
-	 * Default `"0.4.0"`.
+	 * @defaultValue `"0.4.0"`
 	 */
 	corpusVersion?: string
 	/**
-	 * Default `"deepseek-kryptonite"`.
+	 * @defaultValue `"deepseek-kryptonite"`
 	 */
 	source?: string
 }
 
 async function* canonicalRows(jsonl: string, corpusVersion: string): AsyncIterable<CanonicalRow> {
-	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(jsonl)) {
+	for await (const raw of JSONSpliterator.fromAsync<Record<string, unknown>>(delimitedSource(jsonl))) {
 		// Strip sidecar underscore-prefixed fields the generator left behind for debugging.
 		const components = raw["components"] as Record<string, string>
 
@@ -53,7 +57,9 @@ async function* canonicalRows(jsonl: string, corpusVersion: string): AsyncIterab
 			source_id: raw["source_id"] as string,
 			corpus_version: corpusVersion,
 			license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
-			synth: raw["synth"] as CanonicalRow["synth"],
+			recipe: raw["recipe"] as CanonicalRow["recipe"],
+			register: (raw["register"] as string | null) ?? null,
+			surface: requireSurface(raw, "deepseek-kryptonite"),
 		}
 	}
 }
@@ -70,8 +76,8 @@ async function* labeledRows(jsonl: string, corpusVersion: string, quarantineLog:
 	}
 }
 
-export async function buildKryptoniteSlice(
-	options: SliceKryptoniteOptions,
+export async function buildKryptoniteOverlay(
+	options: KryptoniteOverlayOptions,
 	report?: (line: string) => void
 ): Promise<void> {
 	const corpusVersion = options.corpusVersion ?? "0.4.0"
@@ -81,37 +87,40 @@ export async function buildKryptoniteSlice(
 
 	if (!(await pathExists(options.baseManifest))) throw new Error(`base-manifest not found: ${options.baseManifest}`)
 
-	await makeDirectories(options.outDir)
+	const outDir = PathBuilder.from(options.outDir)
+	const corpusDir = outDir(corpusDirectoryName(corpusVersion))
+	await makeDirectories(outDir)
 
 	const quarantine: string[] = []
 
-	const newManifest = await writeSlices(
+	const newManifest = await writeParquetSplits(
 		{ train: labeledRows(options.jsonl, corpusVersion, quarantine) },
-		{ outputDir: options.outDir, corpusVersion }
+		{ outputDir: outDir, corpusVersion }
 	)
 
 	report?.(
-		`wrote ${newManifest.total_rows} rows into ${newManifest.slices.length} slice(s); ` +
+		`wrote ${newManifest.total_rows} rows into ${newManifest.slices.length} parquet file(s); ` +
 			`quarantined ${quarantine.length}`
 	)
 
 	if (quarantine.length) {
-		const qPath = join(options.outDir, `corpus-v${corpusVersion}`, "quarantine-kryptonite.tsv")
-		await writeLocalTextFile(quarantine.join("\n") + "\n", qPath)
+		const qPath = corpusDir("quarantine-kryptonite.tsv")
+		await writeLocalTextFile(quarantine, qPath)
 		report?.(`quarantine log → ${qPath}`)
 	}
 
-	// Stamp the new slice's source field for audit.ts (which prefers slice.source over
-	// first_source_id-prefix inference). Without this, deepseek-kryptonite IDs would have
-	// to match a prefix in KNOWN_SOURCE_PREFIXES — we add it there too as a belt-and-braces.
-	for (const sh of newManifest.slices) {
-		sh.source = source
+	// Stamp the new file's source field for audit.ts
+	// (which prefers the descriptor's `source` over first_source_id-prefix inference).
+	// Without this, deepseek-kryptonite IDs would have to match a prefix in KNOWN_SOURCE_PREFIXES.
+	// We add it there too as a belt-and-braces.
+	for (const file of newManifest.slices) {
+		file.source = source
 	}
 
-	// Compose the final corpus-v0.4.0 manifest: every slice from base + the new slice(s).
-	const base = await readLocalJSONFile<SliceManifest>(options.baseManifest)
+	// Compose the final corpus-v0.4.0 manifest: every parquet file from base + the new file(s).
+	const base = await readLocalJSONFile<ParquetManifest>(options.baseManifest)
 
-	const combined: SliceManifest = {
+	const combined: ParquetManifest = {
 		corpus_version: corpusVersion,
 		schema: PARQUET_COLUMNS,
 		rows_per_slice: base.rows_per_slice,
@@ -125,13 +134,12 @@ export async function buildKryptoniteSlice(
 		total_rows: base.total_rows + newManifest.total_rows,
 	}
 
-	// Stamp source on the legacy v0.3.0 slices too, so audit's slice.source path is the
-	// authoritative one. v0.3.0 slices mix sources; we use the first_source_id-prefix
-	// inference for them (audit.ts will re-derive on its own when slice.source is absent).
-	const combinedPath = join(options.outDir, `corpus-v${corpusVersion}`, "MANIFEST.json")
+	// The v0.3.0 files have no `source`: they mix sources, so audit.ts falls back to its
+	// first_source_id-prefix inference for them (it re-derives on its own when `source` is absent).
+	const combinedPath = corpusDir("MANIFEST.json")
 	await writeLocalJSONFile(combined, combinedPath)
 	report?.(`wrote combined manifest → ${combinedPath}`)
 	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newManifest.total_rows})`)
-	report?.(`  slices=${combined.slices.length} (base=${base.slices.length}, added=${newManifest.slices.length})`)
-	report?.(`  compression=${SLICE_COMPRESSION}`)
+	report?.(`  files=${combined.slices.length} (base=${base.slices.length}, added=${newManifest.slices.length})`)
+	report?.(`  compression=${PARQUET_COMPRESSION}`)
 }