@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/ee/adapters/ads/adapter.ts

@@ -0,0 +1,246 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `ads`: Estonia's INSPIRE Addresses (AD) theme, published by Maa- ja Ruumiamet out of the
+ * Aadressiandmete süsteem.
+ *
+ * Input is newline-delimited GeoJSON: one `AD_Address:AD.Address` feature per line, as the service's
+ * `outputFormat=application/json` writes them. A page of the service is a `FeatureCollection`, and the
+ * acquisition step appends each page's `features` to the file, so the adapter streams a 729,973-feature
+ * extract without holding a page in memory.
+ *
+ * GeoServer flattens the INSPIRE application schema here, so every component value arrives inline on
+ * the feature's `properties` and the adapter resolves no reference. The county, the municipality, the
+ * settlement unit, the address area, the thoroughfare and the postal descriptor each occupy one of six
+ * fixed `component<n>_xlink_title` slots, and an unused slot carries the string `unpopulated` rather
+ * than a void object. The slot a value lands in is fixed: read through `JSONSpliterator` over 1,003
+ * features across ten windows spread through the 729,973, slot 1 is `AU.AdministrativeUnit_maakond`
+ * on all 1,003, slot 2 is `AU.AdministrativeUnit_ov` on all 1,003 and slot 6 is
+ * `AD.Address_PostalDescriptor` on all 1,003, whose title is five digits on all 1,003. Slots 3, 4 and
+ * 5 hold the settlement unit (926), the address area (4) and the thoroughfare (314) or `unpopulated`.
+ *
+ * The field named after the INSPIRE designator holds an internal ADS code, and the house number
+ * sits in the locator's name-spelling element. Estonia writes that ADS code in
+ * `locator_addresslocator_designator_locatordesignator_designator` — `7_1DOW`, `7_3P4Y`, `6_0MU3` —
+ * and 0 of 1,003 sampled values take a house-number shape. The human-readable number is in
+ * `locator_addresslocator_name_locatorname_name_spelling_text`. The designator's own type says
+ * `addressNumber` on the street rows, which is what makes that field tempting to read. Reading it
+ * would stamp `7_1DOW` as the house number on all 729,973 rows.
+ *
+ * Which of the two the locator name is follows the feature's structure rather than the value's shape.
+ * A feature carrying a thoroughfare or an address area writes its number there, and one carrying
+ * neither writes the farm or building name that designates the addressable unit. Over the same 1,003
+ * features the split is exact: 314 of 314 features with a thoroughfare carry a number-shaped locator
+ * name, 4 of 4 with an address area do, and 685 of 685 with neither carry a name instead. The two
+ * never co-occur. The adapter therefore reads `house_number` where a thoroughfare element exists and
+ * `venue` where none does, so a farm name never arrives labeled as a number.
+ *
+ * Slots 4 and 5 carry a malformed `href`: it addresses the `AU_haldusyksused` workspace with an
+ * `AD_Address` type and answers HTTP 400 `Unknown namespace [AD_Address]`. The adapter reads the slot's
+ * `title` and never follows an `href`.
+ *
+ * A row with no thoroughfare element is the ordinary rural case rather than a defect: 685 of those
+ * 1,003 features, against the whole-service share of 288,009 thoroughfare-bearing rows in 729,973,
+ * which is 39.5%.
+ *
+ * Maa- ja Ruumiamet's dataset record states Creative Commons CC0 1.0 in `otherConstraints`, which the
+ * address-source register elected and records under `spdx` as `CC0-1.0`. CC0 reserves no act, so the
+ * adapter stamps that label on every row and the model card needs no attribution clause, although
+ * crediting the publisher stays good practice.
+ *
+ * The adapter honors `opts.limit` and `opts.signal`. `opts.country` is optional and accepts only `EE`.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { JSONSpliterator } from "spliterator"
+
+import { UnsupportedCountryError } from "#adapters/errors"
+import { stableSourceID } from "#adapters/source-id"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const ADS_ADAPTER_ID = "ads"
+
+/**
+ * The one jurisdiction this adapter emits.
+ *
+ * The service covers Estonia whole, so the set has one member and `opts.country` either
+ * matches `EE` or asks for a jurisdiction the dataset does not publish.
+ */
+export const ADS_COUNTRIES: readonly string[] = ["EE"]
+
+/**
+ * The license the address-source register elected for this publisher.
+ *
+ * `unchecked-access-free-ee-maa-ja-ruumiamet` is elected with `spdx` `CC0-1.0`,
+ * from the `otherConstraints` of CSW record `b0bd266e-b4c6-45db-b7ea-f47486b6258b`.
+ */
+export const ADS_DEFAULT_LICENSE = "CC0-1.0"
+
+/**
+ * The sentinel Estonia writes in an unused component slot.
+ *
+ * It is the local name of an INSPIRE void reason, written as bare text where GeoServer would
+ * otherwise write a void object, and it appears in both the slot's `href` and its `title`.
+ * A reader that takes the title at face value stores the word as a street name.
+ */
+const UNPOPULATED = "unpopulated"
+
+/**
+ * The flattened `AD.Address` properties this adapter consults.
+ *
+ * The explicit shape catches a rename early, and leaving the designator out of
+ * it is deliberate: the property exists on every feature and holds an ADS code
+ * rather than a house number, so the adapter cannot read it by accident.
+ */
+interface ADSProperties {
+	gml_id?: number | string
+	inspireid_identifier_localid?: number | string
+	locator_addresslocator_name_locatorname_name_spelling_text?: string
+	component1_xlink_title?: string
+	component2_xlink_title?: string
+	component3_xlink_title?: string
+	component4_xlink_title?: string
+	component5_xlink_title?: string
+	component6_xlink_title?: string
+}
+
+/**
+ * One line of the input: a GeoJSON feature as the service's JSON writer emits it.
+ */
+interface ADSFeature {
+	id?: string
+	properties?: ADSProperties
+}
+
+/**
+ * A component slot's value, or an empty string where the slot is unused.
+ *
+ * The sentinel is compared case-insensitively because it renders an INSPIRE codelist term
+ * rather than a value the publisher authored.
+ */
+function slotValue(value: string | undefined): string {
+	const trimmed = (value ?? "").trim()
+
+	return trimmed.toLowerCase() === UNPOPULATED ? "" : trimmed
+}
+
+/**
+ * The thoroughfare element a feature addresses, or an empty string where it addresses none.
+ *
+ * A thoroughfare name and an address area never co-occur on one feature, and the
+ * publisher's own `alternativeidentifier` writes either one in the same position:
+ * `Veskimäe tee 8` for the first and `Vahatu vkt 19` for the second.
+ * The address area is therefore read as the street it stands in for rather than as a separate rung.
+ */
+export function thoroughfareOf(properties: ADSProperties): string {
+	return slotValue(properties.component5_xlink_title) || slotValue(properties.component4_xlink_title)
+}
+
+export function createADSAdapter(): CorpusAdapter {
+	return {
+		id: ADS_ADAPTER_ID,
+		defaultLicense: ADS_DEFAULT_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.EstoniaADS,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"Estonia's INSPIRE Addresses theme (Maa- ja Ruumiamet): address points with a thoroughfare or a named site.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && !ADS_COUNTRIES.includes(opts.country)) {
+				throw new UnsupportedCountryError(ADS_ADAPTER_ID, ADS_COUNTRIES, opts.country)
+			}
+
+			const features = JSONSpliterator.fromAsync<ADSFeature>(opts.inputPath)
+
+			let emitted = 0
+
+			for await (const feature of features) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const properties = feature.properties ?? {}
+
+				const region = slotValue(properties.component1_xlink_title)
+				const municipality = slotValue(properties.component2_xlink_title)
+				const settlement = slotValue(properties.component3_xlink_title)
+				const street = thoroughfareOf(properties)
+				const postcode = slotValue(properties.component6_xlink_title)
+				const locatorName = (properties.locator_addresslocator_name_locatorname_name_spelling_text ?? "").trim()
+
+				if (!municipality) continue
+
+				if (!postcode && !settlement) continue
+
+				const components: CanonicalRow["components"] = {}
+
+				if (street) {
+					// A thoroughfare or an address area makes the locator name this feature's
+					// number on that thoroughfare.
+					if (locatorName) {
+						components.house_number = locatorName
+					}
+
+					components.street = street
+				} else if (locatorName) {
+					// With no thoroughfare the locator name is the site or building name that
+					// designates the addressable unit, which is how a rural Estonian address works.
+					components.venue = locatorName
+				}
+
+				// The settlement unit sits inside the municipality, and Estonia writes it between
+				// the thoroughfare and the municipality in its own `alternativeidentifier`.
+				if (settlement) {
+					components.dependent_locality = settlement
+				}
+
+				if (postcode) {
+					components.postcode = postcode
+				}
+
+				components.locality = municipality
+
+				if (region) {
+					components.region = region
+				}
+
+				const rendered = formatAddressRow(components, "EE", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				const seed = String(properties.inspireid_identifier_localid ?? properties.gml_id ?? feature.id ?? "").trim()
+
+				const sourceID = seed ? `${ADS_ADAPTER_ID}-${seed}` : stableSourceID(ADS_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "EE",
+					locale: "et-EE",
+					source: ADS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: ADS_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const adsAdapter = createADSAdapter()

package/lib/ee/tools/fetch/ads.ts

@@ -0,0 +1,270 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Harvests Estonia's INSPIRE Addresses (AD) theme into the newline-delimited JSON file that
+ * `#ee/adapters/ads/adapter` reads.
+ *
+ * Maa- ja Ruumiamet publishes the Aadressiandmete süsteem through GeoServer at
+ * {@linkcode EE_ADS_WFS}. The service's dataset record states Creative Commons CC0 1.0 in
+ * `otherConstraints`, which the address-source register elected under `spdx` `CC0-1.0`. CC0 reserves
+ * no act, so paging the service and storing what it returns is granted and the harvest needs no
+ * attribution clause.
+ *
+ * `#tools/fetch/inspire-addresses` owns the capabilities read and the GeoJSON page read, and
+ * `#tools/fetch/wfs-harvest` owns the appending, the manifest and the resumption. This module states
+ * what is particular to Estonia, which is four measurements taken on 2026-10-02:
+ *
+ * 1. The payload needs no component resolution. GeoServer flattens the application schema here, so
+ *    each `component<n>_xlink_href` has a sibling `component<n>_xlink_title` carrying the referenced
+ *    feature's value, and the adapter reads the titles. The harvest therefore takes one feature type
+ *    rather than the five a harmonized INSPIRE service splits an address across.
+ * 2. `resultType=hits` at `startIndex=1` reports `numberMatched="729973"` and a feature page reports
+ *    the same number, so this service's count is real. It is read through
+ *    `readCheckedWFSFeatureCount`, which proves the number against a page before a caller uses it,
+ *    and no count is written into this file.
+ * 3. The service honors a sort. `ImplementsSorting` is `TRUE`, `sortBy=gml_id` answers HTTP 200 and
+ *    `sortBy=gml_id D` returns the type's last features first, which is what proves the parameter is
+ *    applied rather than accepted and dropped. `sortBy=gml:identifier` and `sortBy=inspireId` answer
+ *    HTTP 400 `Illegal property name`, so the sort property is one of the flattened names.
+ * 4. A `count=10000` page of `application/json` is 31,198,909 bytes and takes about 6 seconds. At
+ *    729,973 features that is 73 pages and about 2.3 GB, plus three requests for the capabilities and
+ *    the checked count.
+ *
+ * The harvest writes one feature per line, which is what lets the adapter stream the whole extract
+ * without holding a page. The `FeatureCollection` envelope is dropped: its counts describe one page
+ * and the manifest records them per page instead.
+ */
+
+import { readCheckedWFSFeatureCount, type APIClient, type ReadWFSFeatureCountOptions } from "@mailwoman/core/api"
+import { stringifyJSON } from "@mailwoman/core/json"
+import type { PathBuilderLike } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { readFeaturePage, readWFSCapabilities } from "#tools/fetch/inspire-addresses"
+import {
+	harvestPagedWFS,
+	runWFSHarvest,
+	statedFeatureCount,
+	type HarvestedPayload,
+	type WFSHarvestManifest,
+} from "#tools/fetch/wfs-harvest"
+
+/**
+ * The service this module harvests.
+ */
+export const EE_ADS_WFS = "https://inspire.geoportaal.ee/geoserver/AD_Address/wfs"
+
+/**
+ * The qualified name the service publishes its addresses under.
+ *
+ * Read from the capabilities document per run.
+ * This constant is the name a refusal quotes when the document stops publishing an Addresses type.
+ */
+export const EE_ADS_TYPE_NAME = "AD_Address:AD.Address"
+
+/**
+ * The property the pages are ordered by.
+ *
+ * A flattened property name, because the service refuses `gml:identifier`
+ * and `inspireId` with HTTP 400 `Illegal property name`.
+ */
+export const EE_ADS_SORT_BY = "gml_id"
+
+/**
+ * Features per page.
+ *
+ * The service's `CountDefault` is 1,000,000 and it honors `count=10000`, which is where the page
+ * size is set: a 10,000-feature page is 31,198,909 bytes, and a larger page buys fewer requests
+ * at the cost of holding a larger body in memory and losing more work when one request fails.
+ */
+export const EE_ADS_PAGE_SIZE = 10_000
+
+/**
+ * The file the adapter's `inputPath` points at, inside the harvest directory.
+ */
+export const EE_ADS_HARVEST_FILE = "address.jsonl"
+
+/**
+ * The adapter slug this module fetches for, and the directory it writes under.
+ */
+const SLUG = "ads"
+
+/**
+ * Names the service in every refusal this module raises.
+ */
+const CONTEXT = "ee ads wfs"
+
+/**
+ * The license the address-source register elected for this publisher.
+ */
+const LICENSE = "CC0-1.0"
+
+/**
+ * The spacing between requests.
+ *
+ * The service answers a 10,000-feature page in about 6 seconds, so this is not what limits the harvest.
+ * It bounds the harvest to one request per second against a government host whose published terms
+ * state no rate, which the shorter requests — the capabilities and the count — would otherwise exceed.
+ */
+const MIN_REQUEST_INTERVAL_MS = 1000
+
+/**
+ * One page of features as newline-delimited JSON, one feature per line.
+ *
+ * A trailing newline on a non-empty page, so the next page's first feature starts its own line.
+ */
+export function featuresAsJSONL(features: readonly unknown[]): string {
+	if (!features.length) return ""
+
+	return `${features.map((feature) => stringifyJSON(feature)).join("\n")}\n`
+}
+
+export interface HarvestADSOptions {
+	/**
+	 * Where the harvest is written.
+	 *
+	 * The adapter reads {@linkcode EE_ADS_HARVEST_FILE} inside it.
+	 */
+	outputDir: PathBuilderLike
+	/**
+	 * Features per page, defaulting to {@linkcode EE_ADS_PAGE_SIZE}.
+	 */
+	pageSize?: number
+	/**
+	 * Stop once the manifest holds this many pages, for a probe rather than a full harvest.
+	 */
+	maxPages?: number
+	signal?: AbortSignal
+	report?: (line: string) => void
+}
+
+/**
+ * Harvests the service into `options.outputDir` and writes the manifest beside the data file.
+ *
+ * The capabilities document is read first, so the type name, the JSON format
+ * and the paging support come from the service rather than from this file.
+ * A service that stops publishing any of the three raises here rather than
+ * producing a harvest of the wrong thing.
+ *
+ * @returns The manifest as written.
+ */
+export async function harvestADSEE(
+	client: Pick<APIClient, "fetch">,
+	options: HarvestADSOptions
+): Promise<WFSHarvestManifest> {
+	const capabilities = await readWFSCapabilities(client, { wfsURL: EE_ADS_WFS, context: CONTEXT })
+	const typeName = capabilities.typeNames.Address
+
+	if (!typeName) {
+		throw new Error(
+			`${CONTEXT}: the capabilities document publishes no Addresses feature type, so ${EE_ADS_TYPE_NAME} could not be confirmed`
+		)
+	}
+
+	if (!capabilities.jsonFormat) {
+		throw new Error(
+			`${CONTEXT}: the service advertises no JSON output format, and the adapter reads newline-delimited JSON — it advertises ${capabilities.outputFormats.join(", ")}`
+		)
+	}
+
+	if (!capabilities.supportsPaging) {
+		throw new Error(
+			`${CONTEXT}: the service does not advertise ImplementsResultPaging, and such a service answers every page with the first, so a paged harvest would store one page repeated`
+		)
+	}
+
+	const countOptions: ReadWFSFeatureCountOptions = { wfsURL: EE_ADS_WFS, typeNames: typeName, context: CONTEXT }
+
+	// The cap comes from the service's own `CountDefault`, because a `numberMatched` equal
+	// to it cannot be told apart from the largest page the service will serve.
+	const featureCount = statedFeatureCount(
+		await readCheckedWFSFeatureCount(client, countOptions),
+		capabilities.countDefault
+	)
+
+	options.report?.(`  Feature count: ${featureCount.count ?? "not stated"} — ${featureCount.because}`)
+
+	return await harvestPagedWFS({
+		context: CONTEXT,
+		source: SLUG,
+		outputDir: options.outputDir,
+		filename: EE_ADS_HARVEST_FILE,
+		request: {
+			wfsURL: EE_ADS_WFS,
+			typeName,
+			outputFormat: capabilities.jsonFormat,
+			sortBy: EE_ADS_SORT_BY,
+		},
+		license: LICENSE,
+		// CC0 reserves no act, so the elected terms require none.
+		attribution: "",
+		featureCount,
+		pageSize: options.pageSize ?? EE_ADS_PAGE_SIZE,
+		maxPages: options.maxPages,
+		signal: options.signal,
+		report: options.report,
+		readPage: async ({ startIndex, count }): Promise<HarvestedPayload> => {
+			const page = await readFeaturePage(client, {
+				wfsURL: EE_ADS_WFS,
+				typeName,
+				outputFormat: capabilities.jsonFormat ?? "",
+				count,
+				startIndex,
+				supportsPaging: capabilities.supportsPaging,
+				sortBy: EE_ADS_SORT_BY,
+				context: CONTEXT,
+			})
+
+			return {
+				payload: featuresAsJSONL(page.features),
+				numberReturned: page.numberReturned,
+				numberMatched: page.numberMatched,
+				retrievedAt: page.timeStamp,
+			}
+		},
+	})
+}
+
+export type FetchADSEEOptions = BaseFetchOptions &
+	Pick<HarvestADSOptions, "maxPages" | "pageSize" | "signal"> & {
+		/**
+		 * An injected client, for a test that supplies its own transport.
+		 *
+		 * A caller that leaves it out gets a paced client of this module's own.
+		 */
+		client?: Pick<APIClient, "fetch">
+	}
+
+/**
+ * Harvests Estonia's addresses under `<outRoot>/ads/`, for `mailwoman corpus fetch ads-ee`.
+ *
+ * Re-runnable: a complete harvest whose file still hashes to the manifest's digest makes
+ * no request, and an interrupted one resumes at the page after the last that reached disk.
+ */
+export async function fetchADSEE(options: FetchADSEEOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const pageSize = options.pageSize ?? EE_ADS_PAGE_SIZE
+
+	return await runWFSHarvest(
+		{
+			slug: SLUG,
+			outRoot: options.outRoot,
+			filename: EE_ADS_HARVEST_FILE,
+			expect: { wfsURL: EE_ADS_WFS, sortBy: EE_ADS_SORT_BY, pageSize },
+			displayName: "ee-ads",
+			minRequestIntervalMs: MIN_REQUEST_INTERVAL_MS,
+			client: options.client,
+			maxPages: options.maxPages,
+			harvest: async (client) =>
+				await harvestADSEE(client, {
+					outputDir: options.outRoot(SLUG),
+					pageSize,
+					maxPages: options.maxPages,
+					signal: options.signal,
+					report,
+				}),
+		},
+		report
+	)
+}

package/lib/env.ts

@@ -20,7 +20,9 @@ export const PublicCorpusEnvSchema = z.object({
 })
 
 /**
- * Credentials the corpus acquisition and golden-expansion tooling send. Never log their values.
+ * Credentials the corpus acquisition and golden-expansion tooling send.
+ *
+ * Never log their values.
  */
 export const PrivateCorpusEnvSchema = z.object({
 	// OpenAddresses batch-download API token (`corpus/lib/tools/fetch/openaddresses.ts`).
@@ -45,6 +47,8 @@ export const PrivateCorpusEnvSchema = z.object({
 export const $public = liveEnv(PublicCorpusEnvSchema, corePublic)
 
 /**
- * Live corpus credentials over core's. Never log their values.
+ * Live corpus credentials over core's.
+ *
+ * Never log their values.
  */
 export const $private = liveEnv(PrivateCorpusEnvSchema, corePrivate)