@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/nl/recipes/postcode.ts

@@ -3,35 +3,39 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `nl-postcode` — the Dutch full-form postcode slice (#924). The model reads the DIGITS-FIRST NL
- *   postcode "1012 LG" as a house number + a 2-letter street ("1012 LG Amsterdam" → house_number
- *   1012 / street "LG" / locality Amsterdam), and that spurious street context then pulls the
- *   locality into the US situs tier (Amsterdam → Amsterdam, NY). Letters-first postcodes (UK
- *   "SW1A 1AA") parse natively; the `\d{4} [A-Z]{2}` shape does not, and the soft query-shape prior
- *   (0.9 log-odds) can't overcome the strong house-number reading of a leading 4-digit token.
+ * `nl-postcode`: the Dutch full-form postcode recipe.
  *
- *   This is the model-first fix as DATA (the #723/#901 discipline — teach the boundary, don't
- *   override the decoder): real NL (street, number, postcode, city) tuples in the orders Dutch
- *   addresses actually use, with the full postcode tagged as ONE postcode span. Both the SPACED
- *   ("1012 LG", the failing form — a 2-token span) and UNSPACED ("1012LG", 1 token) forms are
- *   emitted so the model learns the digits-first postcode regardless of spacing; the three orders
- *   keep polarity balanced (the v1.9.9 lesson).
+ * The model reads the digits-first NL postcode "1012 LG" as a house number plus a two-letter
+ * street, so "1012 LG Amsterdam" becomes house_number 1012, street "LG", locality Amsterdam. That
+ * spurious street context then pulls the locality into the US situs tier. The `\d{4} [A-Z]{2}`
+ * shape does not parse natively. The soft query-shape prior cannot overcome the strong
+ * house-number reading of a leading four-digit token.
+ *
+ * This recipe is the model-first fix as data. It supplies real NL (street, number, postcode, city)
+ * tuples in the orders Dutch addresses use, with the full postcode tagged as one postcode span. Both
+ * the spaced and unspaced forms are emitted so the model learns the digits-first postcode regardless
+ * of spacing. The three orders keep polarity balanced.
  */
 
+import { isNLPostcodeKey } from "@mailwoman/codex/nl"
 import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
 
-import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
+import { alignAndWrite, readTuples, requireRegister, type CorpusRecipe, recipeSourceID } from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
+import { SurfaceOrigin } from "#types"
 
 /**
- * "1012LG" → "1012 LG". The tuples carry the unspaced OA form; the spaced form is the failing case.
+ * "1012LG" → "1012 LG".
+ *
+ * The tuples contain the unspaced OA form.
+ * The spaced form is the failing case.
  */
 function spacePostcode(pc: string): string {
 	return pc.replace(/^(\d{4})([A-Z]{2})$/, "$1 $2")
 }
 
 /**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
+ * Recipe registered with the corpus builder.
  */
 export const nlPostcodeRecipe: CorpusRecipe = {
 	name: "nl-postcode",
@@ -44,6 +48,11 @@ export const nlPostcodeRecipe: CorpusRecipe = {
 		let emitted = 0
 		let skipped = 0
 
+		const NL_POSTCODE_PROVENANCE = {
+			register: requireRegister(opts, "nl-postcode"),
+			surface: SurfaceOrigin.Composed,
+		}
+
 		for await (const t of readTuples(opts.input!)) {
 			read++
 			const street = String(t.street ?? "").trim()
@@ -55,19 +64,20 @@ export const nlPostcodeRecipe: CorpusRecipe = {
 				.toUpperCase()
 				.replaceAll(/\s+/g, "")
 
-			if (!street || !city || !number || !/^\d{4}[A-Z]{2}$/.test(rawPostcode)) {
+			if (!street || !city || !number || !isNLPostcodeKey(rawPostcode)) {
 				skipped++
 
 				continue
 			}
 
-			// Spacing rotates so the model sees BOTH the failing spaced form and the unspaced form; the
-			// components.postcode value MUST match the raw form so alignment tags the right span.
+			// The recipe rotates spacing so the model sees both the failing spaced form and the unspaced form.
+			// The components.postcode value must match the raw form so alignment tags the right span.
 			const spaced = read % 2 === 0
 			const postcode = spaced ? spacePostcode(rawPostcode) : rawPostcode
 
-			// The three orders Dutch addresses use. `street number, postcode city` is canonical; the
-			// pc-first form is where the leading digits most strongly mis-read as a house number.
+			// The three orders Dutch addresses use.
+			// `street number, postcode city` is canonical.
+			// The pc-first form is where the leading digits most strongly mis-read as a house number.
 			const order = read % 3
 			let raw: string
 
@@ -86,7 +96,11 @@ export const nlPostcodeRecipe: CorpusRecipe = {
 				locality: city,
 			}
 
-			const source_id = sliceSourceID("synth-nl-postcode", {
+			// One resolution for both, because `source_id` includes the source as its prefix
+			// and a pair that disagreed would name a source no row of this output declares.
+			const source = defaultRecipeSource("synth-nl-postcode")
+
+			const source_id = recipeSourceID(source, {
 				...components,
 				o: String(order),
 				s: spaced ? "1" : "0",
@@ -98,14 +112,22 @@ export const nlPostcodeRecipe: CorpusRecipe = {
 				components,
 				country: "NL",
 				locale: "nl-NL",
-				source: "synth-nl-postcode",
+				source,
 				source_id,
 				corpus_version: "0.10.0",
 				license:
 					"Synthetic — nl-postcode; (street, number, postcode, city) from OpenAddresses NL (per-source attribution in the model card)",
 			}
 
-			if (alignAndWrite(write, canonical, "nl-postcode")) {
+			// Per row, because the id names this tuple's record.
+			// A tuples file written before `sourceID` existed has none.
+			// `null` means the recipe had no id to forward.
+			if (
+				alignAndWrite(write, canonical, "nl-postcode", {
+					...NL_POSTCODE_PROVENANCE,
+					baseSourceID: t.sourceID ?? null,
+				})
+			) {
 				emitted++
 			} else {
 				skipped++

package/lib/nl/tools/fetch/kadaster.ts

@@ -0,0 +1,302 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Downloads the INSPIRE Addresses theme Kadaster publishes for the Netherlands.
+ *
+ *   ## One feed and one file
+ *
+ *   PDOK serves an ATOM download service with a single entry, and that entry's one data link is the
+ *   whole country. There is no service document above it and no dataset feed below it, so the
+ *   resolution is one request:
+ *
+ *       adressen_inspire_geharmoniseerd.xml  →  addresses.gml.gz
+ *
+ *   The entry's `<id>` names a sibling feed, `adressen_inspire_geharmoniseerd_epsg4258.xml`, which
+ *   answers HTTP 404. A reader that followed the `<id>` rather than the `rel="alternate"` link would
+ *   therefore raise on that 404, so this one reads the link.
+ *
+ *   ## Why the download stays compressed
+ *
+ *   `#nl/adapters/kadaster/adapter` inflates `.gml.gz` through `gunzipChunks` as it streams. The
+ *   member is 29,677,448,685 bytes against the download's 819,465,603, so inflating it here would
+ *   cost 36 times the disk for no reader that wants it.
+ *
+ *   ## Freshness
+ *
+ *   The entry's `<updated>` is the signal recorded in the manifest. A skip requires the feed to
+ *   state an `<updated>` value:
+ *   where it states none both sides read `null`, an equality test would hold, and a download of
+ *   unknown age would be kept for as long as the feed stayed silent.
+ */
+
+import { APIClient, assertNoOGCServiceException } from "@mailwoman/core/api"
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { tryStat } from "@mailwoman/core/fs/readers/stat"
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { NL_KADASTER_ADAPTER_ID, NL_KADASTER_DEFAULT_LICENSE } from "#nl/adapters/kadaster/adapter"
+import { feedChunks, linkWithRel, readAtomFeed } from "#tools/fetch/atom"
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+
+/**
+ * The ATOM download service for the theme.
+ * It holds the country's one entry.
+ */
+export const NL_KADASTER_FEED_URL = "https://service.pdok.nl/kadaster/ad/atom/adressen_inspire_geharmoniseerd.xml"
+
+/**
+ * The name the download is written under.
+ * The adapter's `inputPath` states this name.
+ *
+ * The publisher's own name for the file.
+ * The adapter reads a `.gml.gz` by inflating it, so the suffix is load-bearing.
+ */
+export const NL_KADASTER_DOWNLOAD_FILENAME = "addresses.gml.gz"
+
+/**
+ * The attribution recorded in the manifest.
+ *
+ * CC0 1.0 reserves no act and owes no attribution clause.
+ * This attribution is a credit the repository chooses, rather than a condition of the grant.
+ */
+export const NL_KADASTER_ATTRIBUTION = "Kadaster"
+
+const SLUG = NL_KADASTER_ADAPTER_ID
+
+/**
+ * What one run recorded.
+ *
+ * `feed_updated` is the entry's own `<updated>`.
+ * A later run downloads again when that value changes.
+ *
+ * That is the freshness signal the ATOM feed offers in place of an HTTP validator.
+ *
+ * `rights` is the license URL the feed states, recorded so a change of terms is
+ * visible in the artifact rather than only in the register.
+ */
+export interface NLKadasterManifest extends SourceManifest {
+	feed_url: string
+	feed_updated: string | null
+	rights: string | null
+	license: string
+	attribution: string
+}
+
+/**
+ * Per-invocation options.
+ */
+export interface DownloadNLKadasterOptions {
+	outputDir: PathBuilderLike
+	/**
+	 * Downloads the file even where the feed reports the recorded `<updated>`.
+	 */
+	force?: boolean
+	retries?: number
+	retryDelayMs?: number
+	signal?: AbortSignal
+	report?: (line: string) => void
+}
+
+/**
+ * Where the download is, and what the feed states about it.
+ */
+export interface NLKadasterPublication {
+	downloadURL: string
+	/**
+	 * The entry's own `<updated>`, or `null` where the feed states none.
+	 *
+	 * `updated` sits on the Atom entry as well as on the feed, and this reads
+	 * the entry that held the data link.
+	 * The feed reported `2026-09-02T11:45:47Z` when this reader was written.
+	 */
+	feedUpdated: string | null
+	/**
+	 * The entry's `<rights>`, or `null` where it states none.
+	 */
+	rights: string | null
+	/**
+	 * The byte count the link claims, or `null` where it claims none.
+	 *
+	 * Never taken as the size of the download: the bytes recorded in the manifest
+	 * are counted off the delivered body.
+	 */
+	claimedBytes: number | null
+}
+
+/**
+ * Reads the feed and answers the one entry's data link.
+ *
+ * Raises where the feed lists no entry, or where its entry offers no data link.
+ * The alternative would return `{fetched: 0, skipped: 0, failed: 0}`, which a caller
+ * reads as a fetch that completed and found the publisher empty.
+ *
+ * Separate from {@linkcode downloadNLKadaster} because the resolution is the part a test can drive.
+ * The download runs on global `fetch`, which a unit test cannot intercept.
+ */
+export async function readNLKadasterPublication(
+	client: Pick<APIClient, "fetch">,
+	options: { signal?: AbortSignal; report?: (line: string) => void } = {}
+): Promise<NLKadasterPublication> {
+	const { report } = options
+
+	report?.(`=== ${SLUG}: reading ${NL_KADASTER_FEED_URL}`)
+
+	const { data: feedXML } = await client.fetch<string>({
+		url: NL_KADASTER_FEED_URL,
+		timeout: 120_000,
+		signal: options.signal,
+	})
+
+	assertNoOGCServiceException(String(feedXML), `${SLUG} download service feed`)
+
+	const feed = await readAtomFeed(feedChunks(String(feedXML)))
+
+	const offered = feed.entries.flatMap((entry) => {
+		const link = linkWithRel(entry.links, "enclosure") ?? linkWithRel(entry.links, "alternate")
+
+		return link ? [{ entry, link }] : []
+	})[0]
+
+	if (!offered) {
+		throw new Error(
+			`${SLUG}: the feed offers no download through a rel="enclosure" or rel="alternate" link, ` +
+				`so no file can be resolved. It listed ${feed.entries.length} entries.`
+		)
+	}
+
+	return {
+		downloadURL: offered.link.href,
+		feedUpdated: offered.entry.updated || null,
+		rights: offered.entry.rights,
+		claimedBytes: offered.link.length,
+	}
+}
+
+/**
+ * Whether the file on disk is the one the feed currently offers.
+ *
+ * A skip requires the feed to state an `<updated>` value.
+ * Where it states none, both sides read `null` and an equality test would hold.
+ *
+ * A skip there would keep a download of unknown age for as long as the feed stayed silent.
+ *
+ * A second 782 MiB download is the slower error and the recoverable one.
+ */
+export function nlKadasterPublicationIsRecorded(
+	recorded: NLKadasterManifest,
+	bytesOnDisk: number,
+	publication: NLKadasterPublication
+): boolean {
+	if (!publication.feedUpdated) return false
+
+	return recorded.feed_updated === publication.feedUpdated && recorded.bytes === bytesOnDisk
+}
+
+/**
+ * Resolves the feed to its one file and downloads it.
+ */
+export async function downloadNLKadaster(
+	client: Pick<APIClient, "fetch">,
+	options: DownloadNLKadasterOptions
+): Promise<FetchSummary> {
+	const { report } = options
+	const destDir = PathBuilder.from(options.outputDir)
+	const downloadPath = destDir(NL_KADASTER_DOWNLOAD_FILENAME)
+	const manifestPath = destDir("MANIFEST.json")
+
+	// Resolved before the directory is made, so a publisher that answers with an
+	// exception report leaves no empty source directory behind.
+	const publication = await readNLKadasterPublication(client, { signal: options.signal, report })
+
+	await makeDirectories(destDir)
+
+	const recorded = await readManifest<NLKadasterManifest>(manifestPath)
+	const stat = await tryStat(downloadPath)
+
+	if (!options.force && recorded && stat && nlKadasterPublicationIsRecorded(recorded, stat.size, publication)) {
+		report?.(`  present, and the feed reports the recorded updated ${publication.feedUpdated}`)
+
+		return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+	}
+
+	report?.(
+		`  download: ${publication.downloadURL}` +
+			(publication.claimedBytes === null
+				? ""
+				: ` (the feed claims ${ByteFormatter.formatIEC(publication.claimedBytes)})`)
+	)
+
+	const { bytes } = await downloadToFile({
+		url: publication.downloadURL,
+		dest: downloadPath,
+		retries: options.retries,
+		retryDelayMs: options.retryDelayMs,
+		report,
+	})
+
+	const sha256 = await sha256File(downloadPath)
+
+	report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)}  sha256=${sha256}`)
+
+	const manifest: NLKadasterManifest = {
+		source_url: publication.downloadURL,
+		feed_url: NL_KADASTER_FEED_URL,
+		filename: NL_KADASTER_DOWNLOAD_FILENAME,
+		bytes,
+		sha256,
+		feed_updated: publication.feedUpdated,
+		rights: publication.rights,
+		license: NL_KADASTER_DEFAULT_LICENSE,
+		attribution: NL_KADASTER_ATTRIBUTION,
+		downloaded_at: new Date().toISOString(),
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}
+
+/**
+ * The path `#nl/adapters/kadaster/adapter` reads, given the root a fetch wrote under.
+ */
+export function nlKadasterInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike {
+	return outRoot(SLUG, NL_KADASTER_DOWNLOAD_FILENAME)
+}
+
+/**
+ * Per-invocation options for the registry entry.
+ */
+export interface FetchNLKadasterOptions extends BaseFetchOptions {
+	force?: boolean
+	retries?: number
+	retryDelayMs?: number
+	signal?: AbortSignal
+}
+
+/**
+ * The registry entry.
+ */
+export async function fetchNLKadaster(
+	options: FetchNLKadasterOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	await using client = new APIClient({ displayName: SLUG, retry: true })
+
+	// Awaited rather than returned: `await using` disposes the client when this scope exits,
+	// and a disposed `APIClient` refuses every later request.
+	return await downloadNLKadaster(client, {
+		outputDir: options.outRoot(SLUG),
+		force: options.force,
+		retries: options.retries,
+		retryDelayMs: options.retryDelayMs,
+		signal: options.signal,
+		report,
+	})
+}

package/lib/no/adapters/matrikkelen/adapter.ts

@@ -0,0 +1,188 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `matrikkelen`: Kartverket's `Matrikkelen - Adresse` CSV adapter, covering Norway and Svalbard.
+ *
+ * Input is a semicolon-separated CSV from Geonorge, published per area beside a whole-country
+ * extract. Both have the same 46 columns, so one reader serves either. The adapter reads
+ * `adressenavn` as the street, `nummer` and `bokstav` as the house number, and `postnummer` with
+ * `poststed` as the postcode and its postal place.
+ *
+ * A row's country comes from `kommunenummer` rather than from the caller. Svalbard is municipality
+ * 2100 and is its own ISO 3166-1 jurisdiction, so the dataset's own area list is what separates the
+ * two: Geonorge offers Svalbard as `kommune` 2100 and `fylke` 21 beside the `landsdekkende` whole of
+ * the mainland.
+ *
+ * Kartverket states CC BY 4.0 for this dataset across five agreeing metadata fields, so the adapter
+ * records that license on every row and the model card must attribute Kartverket.
+ *
+ * The adapter streams with `CSVSpliterator.fromAsync`, so the 2.6M-row mainland extract never sits
+ * in memory. It honors `opts.limit`, `opts.signal` and `opts.country`.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { CSVSpliterator } from "spliterator"
+
+import { UnsupportedCountryError } from "#adapters/errors"
+import { stableSourceID } from "#adapters/source-id"
+import { composeHouseNumber } from "#adapters/street-line"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const MATRIKKELEN_ADAPTER_ID = "matrikkelen"
+
+/**
+ * The municipality number Kartverket gives Svalbard.
+ *
+ * Every other number in the dataset belongs to a mainland municipality.
+ * Jan Mayen has no entry, although ISO 3166-1 assigns `SJ` to it as well as to Svalbard.
+ */
+const SVALBARD_MUNICIPALITY = "2100"
+
+/**
+ * Every jurisdiction this adapter emits, checked against a caller's `--country`.
+ */
+export const MATRIKKELEN_COUNTRIES: readonly string[] = ["NO", "SJ"]
+
+/**
+ * The subset of the 46 columns the adapter consults.
+ *
+ * The explicit shape catches a column rename early.
+ * The first column's name has a byte-order mark in the published file.
+ *
+ * The adapter reads `adresseId` instead, so no field here depends on that spelling.
+ */
+interface MatrikkelenRow {
+	kommunenummer: string
+	adressetype: string
+	adressenavn: string
+	nummer: string
+	bokstav: string
+	adresseTekst: string
+	postnummer: string
+	poststed: string
+	adresseId: string
+}
+
+/**
+ * The country a row belongs to, read from its municipality number.
+ */
+export function countryOfMunicipality(kommunenummer: string): string {
+	return kommunenummer.trim() === SVALBARD_MUNICIPALITY ? "SJ" : "NO"
+}
+
+export function createMatrikkelenAdapter(): CorpusAdapter {
+	return {
+		id: MATRIKKELEN_ADAPTER_ID,
+		defaultLicense: "CC-BY-4.0",
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.MatrikkelenAdresse,
+		surface: SurfaceOrigin.Rendered,
+		description: "Matrikkelen - Adresse (Kartverket): street-level addresses for Norway and Svalbard.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && !MATRIKKELEN_COUNTRIES.includes(opts.country)) {
+				throw new UnsupportedCountryError(MATRIKKELEN_ADAPTER_ID, MATRIKKELEN_COUNTRIES, opts.country)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+				columnDelimiter: ";",
+			})
+
+			let emitted = 0
+			let cadastral = 0
+
+			try {
+				for await (const record of rows as AsyncIterable<MatrikkelenRow>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const country = countryOfMunicipality(record.kommunenummer ?? "")
+
+					if (opts.country && country !== opts.country) continue
+
+					// A `matrikkeladresse` addresses a cadastral unit rather than a street, so its
+					// `adresseTekst` reads as a farm name and a holding number, such as `Øvrabø, 124/1`.
+					// It has no street to align, and the mainland file holds 34,522 of them
+					// against 2,568,832 `vegadresse` rows.
+					if ((record.adressetype ?? "").trim() !== "vegadresse") {
+						cadastral += 1
+
+						continue
+					}
+
+					const street = (record.adressenavn ?? "").trim()
+					// Kartverket splits `12B` into `nummer` 12 and `bokstav` B, and writes
+					// them joined with no separator in `adresseTekst`.
+					const house = composeHouseNumber(record.nummer ?? "", record.bokstav ?? "")
+					const postcode = (record.postnummer ?? "").trim()
+					const locality = (record.poststed ?? "").trim()
+
+					if (!street) continue
+
+					if (!postcode && !locality) continue
+
+					const components: CanonicalRow["components"] = {}
+
+					if (house) {
+						components.house_number = house
+					}
+
+					components.street = street
+
+					if (postcode) {
+						components.postcode = postcode
+					}
+
+					if (locality) {
+						components.locality = locality
+					}
+
+					const rendered = formatAddressRow(components, country, { singleLine: true })
+
+					if (!rendered) continue
+
+					const { raw, components: aligned } = rendered
+					const seed = (record.adresseId ?? "").trim()
+
+					const sourceID = seed ? `${MATRIKKELEN_ADAPTER_ID}-${seed}` : stableSourceID(MATRIKKELEN_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country,
+						// Norwegian is the language of both jurisdictions' addresses,
+						// and Bokmål is the written form Kartverket publishes.
+						locale: `nb-${country}`,
+						source: MATRIKKELEN_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: "CC-BY-4.0",
+					}
+
+					emitted++
+				}
+			} finally {
+				if (cadastral > 0) {
+					process.stderr.write(
+						`  matrikkelen: ${cadastral} matrikkeladresse rows carry no street and were dropped, ${emitted} kept\n`
+					)
+				}
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const matrikkelenAdapter = createMatrikkelenAdapter()