@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/types.ts

@@ -3,137 +3,303 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Canonical row schemas for the corpus pipeline (per #6 / Phase 1 plan).
+ *   Row, adapter and provenance types for the corpus pipeline.
+ */
+
+import type { BIOLabel, ComponentTag } from "@mailwoman/codex/component"
+import { stringifyJSON } from "@mailwoman/core/json"
+import type { PathBuilderLike } from "path-ts"
+
+/**
+ * The role that an address plays for the entity that the source describes.
+ * The values are wire values.
  *
- *   The corpus pipeline produces two row shapes:
+ * Roles have different grammars.
+ * For example, mailing addresses often use post-office boxes, so mixing roles in
+ * a corpus shifts what the premise parser learns.
+ */
+export const AddressRole = {
+	/**
+	 * The location of the addressable object, as in a national address register.
+	 */
+	Premise: "premise",
+	/**
+	 * The seat that an entity files with a registrar, such as a siège social.
+	 */
+	RegisteredOffice: "registered-office",
+	/**
+	 * The address a third party records for a transacting party on its own transaction record,
+	 * such as the supplier on a government contract notice.
+	 *
+	 * The party supplied the address to the recording body rather than to a registrar.
+	 * No registrar filed it as a seat, and no address register confirmed it as a premise.
+	 * A sole trader's entry can hold a home address.
+	 */
+	Counterparty: "counterparty",
+	/**
+	 * The place where post is delivered.
+	 * This can be a post-office box.
+	 */
+	Mailing: "mailing",
+	/**
+	 * The place where a licensed person or organization practices.
+	 */
+	Practice: "practice",
+	/**
+	 * A site that an entity operates, such as a clinic, a school or a plant.
+	 */
+	Facility: "facility",
+	/**
+	 * The address of a property's owner, recorded against the property.
+	 */
+	Owner: "owner",
+	/**
+	 * The place where a service is delivered or an obligation is discharged.
+	 */
+	Service: "service",
+	/**
+	 * The address for service of legal process.
+	 */
+	LegalNotice: "legal-notice",
+	/**
+	 * A role that this vocabulary does not cover.
+	 *
+	 * Add a new constant when a second uncovered role appears.
+	 */
+	Other: "other",
+} as const
+
+/**
+ * One of the {@link AddressRole} values.
+ */
+export type AddressRole = (typeof AddressRole)[keyof typeof AddressRole]
+
+/**
+ * The role of a row without an `addressRole` field.
  *
- *   1. `CanonicalRow`: an adapter's raw output. Carries a free-form `raw` string, a per-component
- *        ground-truth dict, provenance, and an optional augmentation marker. Adapters emit these.
- *   2. `LabeledRow`: alignment's output. Adds a SentencePiece token list and a parallel BIO label list,
- *        suitable for direct ingestion by the neural training loop.
+ * Corpora written before the field existed came from premise sources.
+ */
+export const DEFAULT_ADDRESS_ROLE: AddressRole = AddressRole.Premise
+
+/**
+ * Returns the row's address role, or {@link DEFAULT_ADDRESS_ROLE} when the field is absent.
+ */
+export function addressRoleOf(row: Pick<CanonicalRow, "addressRole">): AddressRole {
+	return row.addressRole ?? DEFAULT_ADDRESS_ROLE
+}
+
+/**
+ * How a row's `raw` text was produced.
+ * The values are wire values.
  *
- *   `CorpusAdapter` is the contract every data source implements; `AdapterOptions` is the
- *   per-invocation knob set (input path, optional country filter, row cap, abort signal).
+ * This is separate from {@link CanonicalRow.register}, which records where the underlying record came from.
+ * A rendering of a real register record is `Rendered` or `Composed`, and it is still a real address.
  */
+export const SurfaceOrigin = {
+	/**
+	 * The publisher's own string, preserved unchanged.
+	 *
+	 * No adapter in the tree emits this today: each one builds its line from the source's fields.
+	 * The value is reserved for a source that publishes the written line itself,
+	 * such as a register storing `address1` as a person typed it.
+	 */
+	Attested: "attested",
+	/**
+	 * An adapter assembled the line from one record's fields, in that locale's canonical order.
+	 *
+	 * The order comes from the codex layout or the adapter's own template rather than from the
+	 * publisher, so the components are the source's and the written form is this repository's.
+	 * Distinct from {@link SurfaceOrigin.Composed} because an adapter renders one canonical
+	 * form per record while a recipe varies the form deliberately.
+	 *
+	 * A source id beginning `rendered-` does not take this value.
+	 * `SourceOperation.Rendered` in `#recipes/sources` names a recipe operation with the same word,
+	 * and those recipes record {@link SurfaceOrigin.Composed} because they vary the form.
+	 *
+	 * Read the row's `surface` rather than its source name to learn which of the two produced it.
+	 */
+	Rendered: "rendered",
+	/**
+	 * A recipe assembled the text from the published fields of one real record.
+	 *
+	 * The recipe chooses the order, punctuation and casing.
+	 * It varies them to teach a surface.
+	 */
+	Composed: "composed",
+	/**
+	 * A synthetic row, such as a post-office box or an intersection, that corresponds to no published record.
+	 */
+	Invented: "invented",
+} as const
 
-import type { BIOLabel, ComponentTag } from "@mailwoman/core/types"
+/**
+ * One of the {@link SurfaceOrigin} values.
+ */
+export type SurfaceOrigin = (typeof SurfaceOrigin)[keyof typeof SurfaceOrigin]
+
+const SURFACE_ORIGINS = new Set<string>(Object.values(SurfaceOrigin))
 
 /**
- * Provenance + augmentation metadata that travels with every corpus row.
+ * Returns the `surface` that a JSONL row declares.
  *
- * `synth` is `undefined` for natural (un-augmented) rows; present only when a row was produced by the synthesis
- * pipeline (see `synthesize.ts`).
+ * @throws When the row lacks a valid `surface`.
+ * Older files must be regenerated instead of read with a default.
+ */
+export function requireSurface(raw: Record<string, unknown>, producer: string): SurfaceOrigin {
+	const value = raw["surface"]
+
+	if (typeof value === "string" && SURFACE_ORIGINS.has(value)) return value as SurfaceOrigin
+
+	throw new Error(
+		`${producer}: row ${stringifyJSON(raw["source_id"])} declares no surface. ` +
+			`Every row carries one of ${[...SURFACE_ORIGINS].join(", ")}. ` +
+			`A jsonl written before the field existed has to be regenerated rather than read with a default.`
+	)
+}
+
+/**
+ * Provenance fields present on every corpus row.
+ *
+ * Two related facts are recorded per source rather than per row.
+ * `createIneligibilityReader` in `build/eligibility.ts` joins `source` against the
+ * address-source register and refuses the row at build time, so a release-profile corpus
+ * holds eligible rows by construction and `TRAINING_SOURCES.json` records each refusal.
+ *
+ * A per-row eligibility field would copy a per-source decision onto every row
+ * and would disagree with the register after a review changed a license's state.
+ * Whether a row reached a checkpoint is a property of a run rather than of the corpus.
+ *
+ * And the sampler draws a different sample per epoch, so `deriveEffectiveTrainingManifest`
+ * records it per source from a config and an `audit_epoch_mixture` output.
  */
 export interface SourceProvenance {
 	/**
-	 * Adapter id that emitted this row, e.g. `"wof-admin"`, `"ban"`, `"openaddresses"`.
+	 * The id of the adapter or recipe that emitted the row, such as `"ban"` or `"openaddresses"`.
 	 */
 	source: string
 
 	/**
-	 * Stable id within the adapter's source. For SQLite-backed adapters this is the row's primary key; for CSV/GeoJSON, a
-	 * hash of the canonical components. Must be stable across reruns so that dedup and holdout manifests are
-	 * reproducible.
+	 * An id that is unique within the source and stable across reruns,
+	 * so that deduplication and holdouts reproduce.
+	 *
+	 * SQLite-backed adapters use the primary key.
+	 * File-backed adapters hash the components.
 	 */
 	source_id: string
 
 	/**
-	 * Corpus version string. Stamped by the runner, not the adapter. Locked together with the tokenizer version:
-	 * `corpus-v0.1.0` ships with `tokenizer-v0.1.0`.
+	 * The corpus version stamped by the runner.
 	 */
 	corpus_version: string
 
 	/**
-	 * Short license label or SPDX id for _this_ row. Defaults to the adapter's `defaultLicense`, but per-row sources
-	 * (OpenAddresses) override.
+	 * The license label or SPDX id for this row.
+	 *
+	 * It defaults to the adapter's `defaultLicense`, and sources with per-record terms override it.
 	 */
 	license: string
 }
 
 /**
- * Marker placed on rows produced by `synthesize.ts`. Allows downstream code to weight, stratify, or exclude
- * augmentations.
+ * The recipe that produced a row and the row it was derived from.
  */
-export interface SynthMarker {
+export interface RecipeMarker {
 	/**
-	 * Pipeline id describing what augmentation produced this row. Free-form but stable — e.g. `"case-perturb"`,
-	 * `"accent-strip"`, `"abbrev-swap"`, `"compose:case-perturb+typo"`.
+	 * A stable recipe id, such as `"german"` or `"compose:case-perturb+typo"`.
 	 */
-	method: string
+	recipe: string
 
 	/**
-	 * `source_id` of the un-augmented row this was derived from. Allows tracing every synthetic row back to its natural
-	 * ancestor.
+	 * The `source_id` of the row that this row was derived from.
 	 */
 	base_source_id: string
 }
 
 /**
- * One address row, before tokenization + BIO labeling.
- *
- * `raw` is what a parser would see in the wild — possibly multi-line, with arbitrary whitespace. `components` is the
- * ground-truth tagging: every `ComponentTag` present in the source data, mapped to its surface form _as it appears in
- * `raw`_. Alignment uses this to assign BIO labels.
+ * One address row before tokenization and labeling.
  *
- * Country is ISO 3166-1 alpha-2 (`"US"`, `"FR"`). Locale is BCP-47 (`"en-US"`, `"fr-FR"`) and is optional; adapters
- * that can't be sure leave it empty and let the runner default by country.
+ * Each value in `components` must appear in `raw`, within the alignment edit-distance
+ * threshold, or alignment quarantines the row.
  */
 export interface CanonicalRow extends SourceProvenance {
 	/**
-	 * Address string as it might appear in source data.
+	 * The address text as it might appear in source data, possibly spanning lines.
 	 */
 	raw: string
 
 	/**
-	 * Component-by-tag ground truth. Surface forms must occur in `raw` (within the alignment edit distance threshold) or
-	 * the row will land in the quarantine pile.
+	 * The ground-truth text of each component, as it appears in `raw`.
 	 */
 	components: Partial<Record<ComponentTag, string>>
 
 	/**
-	 * ISO 3166-1 alpha-2 country code.
+	 * The ISO 3166-1 alpha-2 country code.
 	 */
 	country: string
 
 	/**
-	 * Optional BCP-47 locale. Defaulted by country if absent.
+	 * The BCP-47 locale.
+	 * The runner derives it from the country when absent.
 	 */
 	locale?: string
 
 	/**
-	 * Present only on synthetic rows.
+	 * The role of this address.
+	 *
+	 * The runner stamps the adapter's `addressRole` on rows that omit it.
+	 * An adapter sets it per row only when one source has several roles.
+	 * Read it with {@link addressRoleOf}.
 	 */
-	synth?: SynthMarker
+	addressRole?: AddressRole
+
+	/**
+	 * The stable id of the publication that the row's record came from, such as `"us-census-tiger"`.
+	 *
+	 * `null` means that the row corresponds to no published record.
+	 * The runner stamps {@link CorpusAdapter.register} on rows that leave it unset.
+	 * Rights records and supply counts read this field.
+	 */
+	register?: string | null
+
+	/**
+	 * How the row's `raw` text was produced.
+	 *
+	 * The runner stamps {@link CorpusAdapter.surface} on rows that omit it.
+	 */
+	surface?: SurfaceOrigin
+
+	/**
+	 * The recipe that produced the row, when a recipe did.
+	 */
+	recipe?: RecipeMarker
 }
 
 /**
- * Output of `align.ts`. Carries everything `CanonicalRow` does, plus parallel `tokens` and `labels` arrays of identical
- * length (`labels[i]` is the BIO tag for `tokens[i]`) and — as of the v0.5.0 char-offset migration (#519) — parallel
- * char-span arrays addressing `raw` directly.
+ * A row after alignment, with tokens, BIO labels and character spans.
  *
- * The span triple is the v0.5.0 source of truth; `tokens`/`labels` remain emitted during the transition (and stay
- * derivable afterwards: whitespace split + span lookup). The reverse derivation — today's token labels — is the lossy
- * direction (punctuation-mute).
+ * The span arrays are the source of truth.
+ * Consumers can derive token labels from them.
  */
 export interface LabeledRow extends CanonicalRow {
 	/**
-	 * SentencePiece subword tokens for `raw`.
+	 * Subword tokens for `raw`.
 	 */
 	tokens: readonly string[]
 
 	/**
-	 * BIO labels, one per token. Same length as `tokens`.
+	 * BIO labels, one per token.
 	 */
 	labels: readonly BIOLabel[]
 
 	/**
-	 * Char-offset label spans over `raw` (parallel arrays, per the #519 ruling): `span_starts[i]` is the inclusive start
-	 * offset (UTF-16 code units) of span `i`, `span_ends[i]` its exclusive end, `span_tags[i]` its component tag.
-	 * Invariants — enforced loudly by `alignRow`, documented for every other producer: sorted ascending by start,
-	 * non-overlapping. `raw` must be NFC-normalized or the offsets are ambiguous (also enforced by `alignRow`).
+	 * Inclusive start offsets of the label spans over `raw`, in UTF-16 code units.
+	 *
+	 * The three span arrays are parallel.
+	 * Spans are sorted by start and never overlap.
 	 *
-	 * Optional during the v0.4.x → v0.5.0 transition only: alignment always emits the triple; frozen historical corpora
-	 * and not-yet-migrated synthesis paths may lack it. Required once v0.5.0 lands and the token path is deleted.
+	 * `raw` must be NFC-normalized.
+	 * `alignRow` enforces both rules.
+	 * Older corpora and some synthesis paths omit the spans.
 	 */
 	span_starts?: readonly number[]
 
@@ -149,11 +315,7 @@ export interface LabeledRow extends CanonicalRow {
 }
 
 /**
- * A row that alignment refused to label. Lands in `/data/corpus/quarantine/` for human review.
- *
- * The `reason` is human-readable; common values are `"component-not-found:<tag>"`, `"edit-distance-exceeded:<tag>"`,
- * `"raw-empty"`. Re-running alignment after a fix should re-emit the quarantined rows; the runner keys them by
- * `source_id`.
+ * A row that alignment refused to label, with a reason such as `"component-not-found:<tag>"`.
  */
 export interface QuarantinedRow {
 	row: CanonicalRow
@@ -161,87 +323,128 @@ export interface QuarantinedRow {
 }
 
 /**
- * Per-invocation knobs handed to an adapter by the runner.
- *
- * `inputPath` is interpreted by the adapter — it might be a single file path, a directory of slices, or even an HTTPS
- * URL. Each adapter documents its own expected shape in its README.
- *
- * `country` filters to a single ISO 3166-1 alpha-2 country _at the adapter level_. Adapters that hold multi-country
- * data (OSM PBF, OpenAddresses) MUST honor this; single-country adapters (BAN) may ignore it but should reject
- * mismatches.
- *
- * `limit` is a soft cap on rows emitted; useful for fixture-driven tests and smoke runs.
- *
- * `signal` allows the runner to cancel a long-running scan cleanly.
+ * Per-run options that the runner passes to an adapter.
  */
 export interface AdapterOptions {
 	/**
-	 * Path to the adapter's input data (file, directory, or URL — adapter-specific).
+	 * The adapter's input.
+	 * Each adapter interprets it as a file, directory or URL.
 	 */
-	inputPath: string
+	inputPath: PathBuilderLike
 
 	/**
-	 * Optional output directory, available to adapters that maintain side state (rare).
+	 * An output directory for adapters that keep side state.
 	 */
 	outputDir?: string
 
 	/**
-	 * ISO 3166-1 alpha-2 country filter.
+	 * An ISO 3166-1 alpha-2 country filter.
+	 *
+	 * Multi-country adapters must apply it.
+	 * Single-country adapters should reject a mismatch.
 	 */
 	country?: string
 
 	/**
-	 * Soft row cap. Adapters should stop iterating once this is reached.
+	 * A soft cap on emitted rows.
 	 */
 	limit?: number
 
 	/**
-	 * Cancellation hook. Adapters should respect this on every iteration boundary.
+	 * The fraction of rows that include an explicit `country` component.
+	 *
+	 * A source without country tokens teaches the model that country tokens are rare.
+	 * That lowers `country` recall.
+	 *
+	 * A value of `0` or no value adds no country and consumes no random draw,
+	 * so existing outputs stay identical.
+	 */
+	countryFraction?: number
+
+	/**
+	 * The seed for the {@linkcode AdapterOptions.countryFraction} draw.
+	 * The draw has a fixed default.
+	 */
+	seed?: number
+
+	/**
+	 * A cancellation signal that adapters should check at each iteration.
 	 */
 	signal?: AbortSignal
+
+	/**
+	 * Counts of input records the adapter refused or trimmed, keyed by reason.
+	 *
+	 * An adapter that drops a record increments a `row:<reason>` key, and one that keeps the row
+	 * but discards a component increments a `component:<tag>:<reason>` key.
+	 * A `kept:<condition>` key counts records kept under a condition the reader should see,
+	 * such as a country the record does not state.
+	 *
+	 * The runner writes the map into `MANIFEST.json` as `dropped`, so the gap between a
+	 * source's record count and the rows written has a counted reason.
+	 * An adapter that does not count leaves the map empty, which the manifest records
+	 * as an empty object rather than as zero drops of any named kind.
+	 */
+	dropped?: Map<string, number>
 }
 
 /**
- * The contract every data source implements.
- *
- * Adapters are async generators: they yield `CanonicalRow`s one at a time, the runner consumes them (writing JSONL +
- * maintaining checksums + driving alignment). Streaming is mandatory — many sources are tens of millions of rows and
- * cannot be buffered.
+ * Adds one to `reason` in an adapter's {@link AdapterOptions.dropped} map, when the caller supplied one.
+ */
+export function countDropped(opts: Pick<AdapterOptions, "dropped">, reason: string): void {
+	opts.dropped?.set(reason, (opts.dropped.get(reason) ?? 0) + 1)
+}
+
+/**
+ * The interface that every data source implements.
  *
- * `defaultLicense` is stamped onto every emitted row's `license` field unless the adapter sets `license` explicitly
- * (e.g. OpenAddresses, which carries per-source licenses).
+ * Adapters stream rows, because many sources hold tens of millions of rows.
  */
 export interface CorpusAdapter {
 	/**
-	 * Stable, machine-friendly id used in paths and CLI args. E.g. `"wof-admin"`.
+	 * A stable id used in paths and CLI arguments, such as `"wof-admin"`.
 	 */
 	readonly id: string
 
 	/**
-	 * Default SPDX-ish license label for rows from this adapter. Per-row overrides allowed.
+	 * The license label stamped on rows that do not set their own.
 	 */
 	readonly defaultLicense: string
 
 	/**
-	 * One-sentence description shown by `npx mailwoman corpus list`.
+	 * The dominant role of the source's addresses.
+	 *
+	 * The field has no default because only the adapter's author knows the source.
+	 * The runner stamps it on rows that leave `addressRole` unset.
 	 */
-	readonly description: string
+	readonly addressRole: AddressRole
 
 	/**
-	 * Async iterable of canonical rows.
-	 *
-	 * Implementations MUST:
+	 * The publication that this adapter reads, or `null` for an adapter that invents rows.
 	 *
-	 * - Honor `opts.country` (filter or reject mismatches).
-	 * - Honor `opts.limit` (stop after N rows).
-	 * - Respect `opts.signal` on every iteration.
-	 * - Set `source` to `this.id` on every emitted row.
-	 * - Set `license` to `this.defaultLicense` unless overriding per-row.
+	 * The field has no default because only the adapter's author knows the source.
+	 * The runner stamps it on rows that leave `register` unset.
+	 */
+	readonly register: string | null
+
+	/**
+	 * The dominant way the adapter produces `raw` text.
 	 *
-	 * Implementations MUST NOT:
+	 * The runner stamps it on rows that leave `surface` unset.
+	 */
+	readonly surface: SurfaceOrigin
+
+	/**
+	 * The one-sentence description that `mailwoman corpus list` shows.
+	 */
+	readonly description: string
+
+	/**
+	 * Streams canonical rows.
 	 *
-	 * - Set `corpus_version` (the runner stamps it).
-	 * - Mutate previously-yielded rows.
+	 * Implementations must apply `opts.country` and `opts.limit`, check `opts.signal` at each
+	 * iteration, set `source` to `this.id`, and set `license` unless a row overrides it.
+	 * They must not set `corpus_version`, which the runner stamps, or mutate rows already yielded.
 	 */
 	rows(opts: AdapterOptions): AsyncIterable<CanonicalRow>
 }