@mailwoman/corpus 10.6.0 → 10.7.0

package/lib/tools/overlay/manifest.ts

@@ -9,13 +9,14 @@
 
 import { dataRootPath } from "@mailwoman/core/data-root"
 import { readLocalBuffer, readLocalJSONFile } from "@mailwoman/core/fs/readers"
-import { statPath } from "@mailwoman/core/fs/readers/stat"
+import { pathExists, statPath } from "@mailwoman/core/fs/readers/stat"
 import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
 import { sha256Hex } from "@mailwoman/core/hash"
 import { stringifyJSON } from "@mailwoman/core/json"
 import { basename, dirname, PathBuilder, type PathBuilderLike } from "path-ts"
 
 import { escapeSQLString, openDuckDB } from "#parquet/duckdb"
+import { openSourceRightsAdmission, type SourceRightsRecord } from "#source-register/corpus-inputs"
 import type { SplitName } from "#utils/split"
 
 interface ParquetFileDescriptor {
@@ -53,6 +54,10 @@ interface BaseManifest {
 	total_rows: number
 	[PRE_RENAME_FILES_KEY]?: ManifestFile[]
 	[PRE_RENAME_ROWS_PER_FILE_KEY]?: unknown
+	/**
+	 * The source-rights record a build or assembly wrote when it checked its sources.
+	 */
+	source_rights?: SourceRightsRecord
 }
 
 /**
@@ -251,6 +256,106 @@ export interface OverlayManifestOptions {
 	 * Each source in this list must appear in the base, and its rows leave the split counts.
 	 */
 	replaceSources?: readonly string[]
+	/**
+	 * Why a local experiment admits sources whose register rows lack an elected
+	 * decision permitting `transform` and `train`.
+	 *
+	 * Without it the assembly refuses them, and with it the manifest records the corpus as not shippable.
+	 */
+	unshippableReason?: string | null
+}
+
+/**
+ * Reads the distinct `source` values of base files whose manifest entries name no source.
+ *
+ * Every file must be present under the local data root.
+ * When one is absent, or a row's `source` is null, `sources` is null and `unreadable` names the cause.
+ *
+ * The caller therefore cannot mistake a partial read for the whole base.
+ */
+async function readBaseFileSources(
+	files: readonly ManifestFile[]
+): Promise<{ sources: string[]; unreadable: null } | { sources: null; unreadable: string }> {
+	const paths = files.map((file) => localManifestFilePath(file.path))
+	const absent: string[] = []
+
+	for (const path of paths) {
+		if (!(await pathExists(path))) {
+			absent.push(path)
+		}
+	}
+
+	if (absent.length) {
+		return { sources: null, unreadable: `${absent.length} of them are absent locally (first: ${absent[0]})` }
+	}
+
+	using db = await openDuckDB()
+
+	const list = paths.map((path) => `'${escapeSQLString(path)}'`).join(", ")
+	const result = await db.runAndReadAll(`SELECT DISTINCT source FROM read_parquet([${list}])`)
+	const values = result.getRowObjects().map((row) => row.source as string | null)
+
+	if (values.some((value) => value == null)) {
+		return { sources: null, unreadable: "a row among them carries a null source" }
+	}
+
+	return { sources: values as string[], unreadable: null }
+}
+
+/**
+ * Checks every corpus source the assembled manifest will hold, before the manifest or plan is written.
+ *
+ * The base manifest's own record covers a base file that records no source, when the base has a record.
+ * A base without a record is checked by reading its files' `source` column.
+ *
+ * When a file cannot be read, the check refuses the base rather than reading the absence as clear.
+ */
+async function checkOverlaySources(
+	args: OverlayManifestOptions,
+	base: BaseManifest,
+	kept: readonly ManifestFile[]
+): Promise<SourceRightsRecord> {
+	const rights = await openSourceRightsAdmission({
+		context: `overlay assembly ${args.version}`,
+		unshippableReason: args.unshippableReason ?? null,
+	})
+
+	const named = new Set<string>(args.files.map((file) => file.source))
+	const unnamed = kept.filter((file) => file.source === undefined)
+
+	for (const file of kept) {
+		if (file.source !== undefined) {
+			named.add(file.source)
+		}
+	}
+
+	if (unnamed.length && base.source_rights) {
+		for (const source of base.source_rights.sources_checked) {
+			named.add(source)
+		}
+	}
+
+	// A base without a record is read instead: its files' own `source` column is what the loader groups rows by.
+	const read = unnamed.length && !base.source_rights ? await readBaseFileSources(unnamed) : null
+
+	for (const source of read?.sources ?? []) {
+		named.add(source)
+	}
+
+	rights.admitAll(named)
+
+	if (unnamed.length && !base.source_rights && read?.unreadable) {
+		rights.admitUnverified(
+			`${unnamed.length} file(s) of base ${args.base}, which name no source while the base manifest carries no ` +
+				`\`source_rights\` record, and ${read.unreadable}`
+		)
+	}
+
+	if (base.source_rights && !base.source_rights.shippable) {
+		rights.admitUnverified(`base ${args.base}, whose own record reads not shippable`)
+	}
+
+	return rights.record()
 }
 
 /**
@@ -357,6 +462,8 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 		.filter((s) => !dropped.includes(s))
 		.map((s) => ({ ...s, path: rerootBaseFilePath(s.path, args.base) }))
 
+	const sourceRights = await checkOverlaySources(args, base, kept)
+
 	const added: ParquetFileDescriptor[] = []
 
 	const newDir = PathBuilder.from(args.newDir)
@@ -416,6 +523,7 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 		licenses_cover:
 			`no license set is measured here. The base build's MANIFEST.json covers the rows it aligned, and ` +
 			`these ${added.length} added file(s) carry their own: ${[...new Set(args.files.map((file) => file.source))].toSorted().join(", ")}.`,
+		source_rights: sourceRights,
 	}
 
 	const out = newDir("MANIFEST.json")

package/lib/tools/source-register/build.ts

@@ -89,7 +89,9 @@ const RESEARCH_PASS_BY_ORIGIN: Readonly<Record<string, ResearchPass>> = {
 	"2026-09-30_rights_review": ResearchPass.RightsReview,
 	"2026-10-01_rights_review": ResearchPass.RightsReview,
 	"2026-10-02_rights_review": ResearchPass.RightsReview,
+	"2026-10-17_rights_review": ResearchPass.RightsReview,
 	"2026-10-03_decoder_exposure": ResearchPass.DecoderExposureSurvey,
+	"2026-10-17_corpus_input_mapping": ResearchPass.CorpusInputMapping,
 }
 
 const GEOMETRY_BY_NAME: Readonly<Record<string, SourceGeometry>> = {
@@ -169,6 +171,12 @@ const LICENSE_DECISIONS: ReadonlyArray<readonly [statement: string, licenseID: s
 		"unchecked-access-licensed-or-query",
 		"An access label: licensed, or answerable per query. It states no terms.",
 	],
+	[
+		"CORPUS INPUT, TERMS UNREAD",
+		"unchecked-corpus-input",
+		"A corpus source reads this publication, and the row exists so that its terms show as unread rather than absent. " +
+			"No reviewer has recorded a reading of them.",
+	],
 ]
 
 /**
@@ -252,6 +260,15 @@ export interface BuildSourceRegisterOptions {
 	 * A path that does not exist is read as no resolutions recorded.
 	 */
 	resolutionsPath?: PathBuilderLike
+	/**
+	 * Source rows a later pass added in the research CSV's own columns, read after the CSV's rows.
+	 *
+	 * `source-additions.json` holds the publications a corpus source reads that no research pass recorded.
+	 * They sit in this repository because the corpus-source table that names them does,
+	 * and they are read as CSV rows so their ids and license scopes come from the same derivation.
+	 * A path that does not exist is read as no additions.
+	 */
+	additionsPath?: PathBuilderLike
 	version: string
 	/**
 	 * ISO 8601 calendar date the research pass was taken, `yyyy-MM-DD`.
@@ -435,8 +452,36 @@ function scopedLicenseID(
 	return licenseID
 }
 
+/**
+ * The shape of `source-additions.json`: rows in the research CSV's column names.
+ */
+interface SourceAdditionsFile {
+	rows?: ReadonlyArray<Readonly<Record<string, string>>>
+}
+
+/**
+ * The research CSV's rows followed by the recorded additions.
+ *
+ * An absent additions file contributes zero rows.
+ * A file that exists and cannot be parsed raises, because reading it as empty
+ * would drop rows without saying so.
+ */
+async function* researchRows(
+	sourcesPath: PathBuilderLike,
+	additionsPath: PathBuilderLike | undefined
+): AsyncGenerator<Readonly<Record<string, string | undefined>>> {
+	yield* readCSVRecords(sourcesPath)
+
+	if (!additionsPath || !(await pathExists(additionsPath))) return
+
+	const file = await readLocalJSONFile<SourceAdditionsFile>(additionsPath)
+
+	yield* file.rows ?? []
+}
+
 async function readSources(
 	sourcesPath: PathBuilderLike,
+	additionsPath: PathBuilderLike | undefined,
 	licenseByStatement: ReadonlyMap<string, string>,
 	fired: Set<string>,
 	scopes: Map<string, LicenseScope>
@@ -446,7 +491,7 @@ async function readSources(
 	let discoveryRailRowsDropped = 0
 	let row = 1
 
-	for await (const record of readCSVRecords(sourcesPath)) {
+	for await (const record of researchRows(sourcesPath, additionsPath)) {
 		row++
 
 		const origin = required(record["origin"], "origin", row)
@@ -733,6 +778,7 @@ export async function buildSourceRegister(options: BuildSourceRegisterOptions):
 
 	const { sources: derived, discoveryRailRowsDropped } = await readSources(
 		options.sourcesPath,
+		options.additionsPath,
 		licenseByStatement,
 		fired,
 		scopes