@mailwoman/corpus 10.4.0 → 10.5.0

package/lib/gb/os-open/open-names.ts

@@ -0,0 +1,180 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Reads the road rows of OS Open Names.
+ *
+ *   The CSV product is 819 files, one per 10 km National Grid square, with no header row. OS ships
+ *   the 34 column names in `Doc/OS_Open_Names_Header.csv`, and the first data row of each file opens
+ *   with a byte-order mark, which this reader strips from the `ID` value.
+ *
+ *   A road row's `ID` is the roadNameTOID that OS Open Roads links reference, such as
+ *   `osgb4000000010927776`. Two `LOCAL_TYPE` values are roads: `Named Road`, and
+ *   `Section Of Named Road` for a road that OS splits where it crosses a postcode district or a
+ *   settlement.
+ */
+
+import { CSVSpliterator } from "spliterator"
+
+/**
+ * The 34 columns of an OS Open Names CSV, in file order, as `Doc/OS_Open_Names_Header.csv` names them.
+ */
+export const OPEN_NAMES_COLUMNS = [
+	"ID",
+	"NAMES_URI",
+	"NAME1",
+	"NAME1_LANG",
+	"NAME2",
+	"NAME2_LANG",
+	"TYPE",
+	"LOCAL_TYPE",
+	"GEOMETRY_X",
+	"GEOMETRY_Y",
+	"MOST_DETAIL_VIEW_RES",
+	"LEAST_DETAIL_VIEW_RES",
+	"MBR_XMIN",
+	"MBR_YMIN",
+	"MBR_XMAX",
+	"MBR_YMAX",
+	"POSTCODE_DISTRICT",
+	"POSTCODE_DISTRICT_URI",
+	"POPULATED_PLACE",
+	"POPULATED_PLACE_URI",
+	"POPULATED_PLACE_TYPE",
+	"DISTRICT_BOROUGH",
+	"DISTRICT_BOROUGH_URI",
+	"DISTRICT_BOROUGH_TYPE",
+	"COUNTY_UNITARY",
+	"COUNTY_UNITARY_URI",
+	"COUNTY_UNITARY_TYPE",
+	"REGION",
+	"REGION_URI",
+	"COUNTRY",
+	"COUNTRY_URI",
+	"RELATED_SPATIAL_OBJECT",
+	"SAME_AS_DBPEDIA",
+	"SAME_AS_GEONAMES",
+] as const
+
+const column = Object.fromEntries(OPEN_NAMES_COLUMNS.map((name, index) => [name, index])) as Record<
+	(typeof OPEN_NAMES_COLUMNS)[number],
+	number
+>
+
+/**
+ * The `LOCAL_TYPE` values that mark a road row.
+ */
+export const OPEN_NAMES_ROAD_TYPES: ReadonlySet<string> = new Set(["Named Road", "Section Of Named Road"])
+
+/**
+ * One road row of OS Open Names, with the columns the corpus adapter reads.
+ */
+export interface OpenNamesRoad {
+	/**
+	 * The roadNameTOID, such as `osgb4000000010927776`.
+	 */
+	id: string
+	/**
+	 * The road's name, in the language OS lists first.
+	 */
+	name1: string
+	/**
+	 * The road's other-language name, Welsh or Gaelic, or an empty string.
+	 */
+	name2: string
+	/**
+	 * The outward code of the postcode district holding the road, such as `LS6`.
+	 */
+	postcodeDistrict: string
+	/**
+	 * The settlement holding the road, or an empty string.
+	 *
+	 * A bilingual settlement is written `<NAME1> / <NAME2>` of that settlement,
+	 * such as `Stornoway / Steòrnabhagh`.
+	 */
+	populatedPlace: string
+	districtBorough: string
+	countyUnitary: string
+}
+
+/**
+ * Counts from a read, with a reason for each row that did not yield.
+ */
+export interface OpenNamesReadStats {
+	/**
+	 * Rows read across every file.
+	 */
+	read: number
+	/**
+	 * Road rows yielded.
+	 */
+	roads: number
+	/**
+	 * Road rows skipped because their `ID` was not among the requested ids.
+	 */
+	unrequested: number
+	/**
+	 * Rows with a column count other than 34.
+	 */
+	malformed: number
+}
+
+/**
+ * A stats object with every count at zero.
+ */
+export function createOpenNamesReadStats(): OpenNamesReadStats {
+	return { read: 0, roads: 0, unrequested: 0, malformed: 0 }
+}
+
+/**
+ * Streams the road rows of one Open Names CSV.
+ *
+ * @param ids When given, only roads whose `ID` is in the set are yielded.
+ */
+export async function* readOpenNamesRoads(
+	csvPath: string,
+	stats: OpenNamesReadStats,
+	ids?: ReadonlySet<string>
+): AsyncGenerator<OpenNamesRoad> {
+	for await (const row of CSVSpliterator.fromAsync<string[]>(csvPath, { header: false })) {
+		stats.read++
+
+		if (row.length !== OPEN_NAMES_COLUMNS.length) {
+			stats.malformed++
+
+			continue
+		}
+
+		if (!OPEN_NAMES_ROAD_TYPES.has(row[column.LOCAL_TYPE]!)) continue
+
+		const id = row[column.ID]!.replace(/^/u, "")
+
+		if (ids && !ids.has(id)) {
+			stats.unrequested++
+
+			continue
+		}
+
+		stats.roads++
+
+		yield {
+			id,
+			name1: row[column.NAME1]!.trim(),
+			name2: row[column.NAME2]!.trim(),
+			postcodeDistrict: row[column.POSTCODE_DISTRICT]!.trim(),
+			populatedPlace: row[column.POPULATED_PLACE]!.trim(),
+			districtBorough: row[column.DISTRICT_BOROUGH]!.trim(),
+			countyUnitary: row[column.COUNTY_UNITARY]!.trim(),
+		}
+	}
+}
+
+/**
+ * The settlement's first-listed name from a `POPULATED_PLACE` value.
+ *
+ * OS writes a bilingual settlement as `<NAME1> / <NAME2>`, and NAME1 is the form the adapter emits.
+ */
+export function populatedPlaceName1(populatedPlace: string): string {
+	return populatedPlace.split(" / ")[0]!.trim()
+}

package/lib/gb/os-open/open-roads.ts

@@ -0,0 +1,145 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Reads the named road links of an OS Open Roads GML tile.
+ *
+ *   The GML product is one file per 100 km National Grid square. A tile holds `road:RoadLink`,
+ *   `road:RoadNode` and `road:MotorwayJunction` features. Only the links hold a centreline and a
+ *   name. A named link has a `road:name1` and a `road:roadNameTOID` whose `xlink:href` is `#` plus
+ *   the roadNameTOID that OS Open Names uses as its `ID`. A link with only a road number carries a
+ *   `road:roadNumberTOID` instead, and an unnamed link carries neither.
+ *
+ *   The centreline is a `gml:LineString` in EPSG:27700, written as a `gml:posList` of easting and
+ *   northing pairs. The reader honors the line's `srsDimension` so a 3D tile would still read its
+ *   first two ordinates.
+ */
+
+import { openReadStream } from "@mailwoman/core/fs/streams"
+import { elementAtPath, streamMarkupElements, textAtPath, type MarkupElement } from "@mailwoman/core/html/elements"
+import type { PathBuilderLike } from "path-ts"
+
+/**
+ * The feature element that carries a centreline.
+ */
+export const OPEN_ROADS_LINK_ELEMENT = "road:RoadLink"
+
+/**
+ * One named road link.
+ */
+export interface OpenRoadsLink {
+	/**
+	 * The link's `gml:id`.
+	 */
+	id: string
+	/**
+	 * `road:name1` as the tile spells it.
+	 */
+	name1: string
+	/**
+	 * The roadNameTOID, without the `#` of the reference, such as `osgb4000000010927776`.
+	 */
+	roadNameTOID: string
+	/**
+	 * The centreline as flat `[easting0, northing0, easting1, northing1, …]` in EPSG:27700 meters.
+	 */
+	vertices: Float64Array
+}
+
+/**
+ * Counts from a read.
+ */
+export interface OpenRoadsReadStats {
+	/**
+	 * Road links read.
+	 */
+	links: number
+	/**
+	 * Named links yielded.
+	 */
+	named: number
+	/**
+	 * Links with no `road:name1` or no `road:roadNameTOID`.
+	 */
+	unnamed: number
+	/**
+	 * Named links whose centreline held fewer than two vertices or a non-numeric ordinate.
+	 */
+	malformedGeometry: number
+}
+
+/**
+ * A stats object with every count at zero.
+ */
+export function createOpenRoadsReadStats(): OpenRoadsReadStats {
+	return { links: 0, named: 0, unnamed: 0, malformedGeometry: 0 }
+}
+
+const LINE_PATH = ["net:centrelineGeometry", "gml:LineString"] as const
+
+/**
+ * The named link a `road:RoadLink` element describes, or a reason it describes none.
+ */
+export function readOpenRoadsLink(element: MarkupElement): OpenRoadsLink | "unnamed" | "malformed-geometry" {
+	const name1 = (textAtPath(element, "road:name1") ?? "").trim()
+	const href = elementAtPath(element, "road:roadNameTOID")?.attributes["xlink:href"] ?? ""
+	const roadNameTOID = href.replace(/^#/u, "").trim()
+
+	if (!name1 || !roadNameTOID) return "unnamed"
+
+	const dimension = Number(elementAtPath(element, ...LINE_PATH)?.attributes.srsDimension ?? "2") || 2
+	const posList = (textAtPath(element, ...LINE_PATH, "gml:posList") ?? "").trim()
+	const ordinates = posList ? posList.split(/\s+/u) : []
+	const vertexCount = Math.floor(ordinates.length / dimension)
+
+	if (vertexCount < 2 || ordinates.length % dimension !== 0) return "malformed-geometry"
+
+	const vertices = new Float64Array(vertexCount * 2)
+
+	for (let vertex = 0; vertex < vertexCount; vertex++) {
+		const easting = Number(ordinates[vertex * dimension])
+		const northing = Number(ordinates[vertex * dimension + 1])
+
+		if (!Number.isFinite(easting) || !Number.isFinite(northing)) return "malformed-geometry"
+
+		vertices[vertex * 2] = easting
+		vertices[vertex * 2 + 1] = northing
+	}
+
+	return { id: element.attributes["gml:id"] ?? "", name1, roadNameTOID, vertices }
+}
+
+/**
+ * Streams the named links of one Open Roads GML tile.
+ */
+export async function* readOpenRoadsLinks(
+	gmlPath: PathBuilderLike,
+	stats: OpenRoadsReadStats,
+	options: { signal?: AbortSignal } = {}
+): AsyncGenerator<OpenRoadsLink> {
+	for await (const element of streamMarkupElements(openReadStream(gmlPath), OPEN_ROADS_LINK_ELEMENT, {
+		xml: true,
+		signal: options.signal,
+	})) {
+		stats.links++
+
+		const link = readOpenRoadsLink(element)
+
+		if (link === "unnamed") {
+			stats.unnamed++
+
+			continue
+		}
+
+		if (link === "malformed-geometry") {
+			stats.malformedGeometry++
+
+			continue
+		}
+
+		stats.named++
+
+		yield link
+	}
+}

package/lib/gb/tools/fetch/os-open.ts

@@ -0,0 +1,251 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Downloads the three OS OpenData products the `gb-os-open` adapter joins, from the OS Downloads API.
+ *
+ *   The API serves OpenData without a key. `GET /products/<id>/downloads` lists one archive per
+ *   format and area with its OS-published MD5, and each archive is checked against that MD5 after
+ *   transfer. A present archive whose MD5 already matches is kept rather than downloaded again.
+ *
+ *   Each archive is unpacked into the subdirectory the adapter reads, `code-point-open/`,
+ *   `open-names/` or `open-roads/`, so the fetch root is the adapter's `inputPath`. The archive
+ *   stays beside its members, because its digest is the value the address-source register records.
+ */
+
+import { APIClient } from "@mailwoman/core/api"
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { extractZipEntries } from "@mailwoman/core/fs/zip"
+import { md5File, sha256File } from "@mailwoman/core/hash"
+import { OS_DOWNLOADS_API_BASE, type OSDownload } from "@mailwoman/spatial/gb/os-downloads"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { GB_OS_OPEN_ADAPTER_ID, GB_OS_OPEN_PRODUCT_DIRECTORIES } from "#gb/adapters/os-open/adapter"
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"
+import { downloadToFile, writeManifest } from "#tools/fetch/download"
+
+/**
+ * The license all three products are published under.
+ */
+export const GB_OS_OPEN_LICENSE_NAME = "Open Government Licence v3.0"
+
+/**
+ * The attribution OGL v3 requires, in the wording OS asks for on Code-Point Open,
+ * which also carries Royal Mail and ONS data.
+ */
+export const GB_OS_OPEN_ATTRIBUTION =
+	"Contains OS data © Crown copyright and database right. Contains Royal Mail data © Royal Mail copyright and database right. Contains National Statistics data © Crown copyright and database right."
+
+/**
+ * One product the adapter reads: its OS Downloads API id, the archive format,
+ * and the subdirectory it unpacks into.
+ */
+export interface OSOpenProduct {
+	productID: string
+	format: string
+	directory: string
+}
+
+/**
+ * The three products, in the order they are fetched.
+ */
+export const GB_OS_OPEN_PRODUCTS: readonly OSOpenProduct[] = [
+	{ productID: "CodePointOpen", format: "CSV", directory: GB_OS_OPEN_PRODUCT_DIRECTORIES.codePointOpen },
+	{ productID: "OpenNames", format: "CSV", directory: GB_OS_OPEN_PRODUCT_DIRECTORIES.openNames },
+	{ productID: "OpenRoads", format: "GML", directory: GB_OS_OPEN_PRODUCT_DIRECTORIES.openRoads },
+]
+
+/**
+ * The manifest entry for one product's archive.
+ */
+export interface OSOpenFileManifest extends SourceManifest {
+	product_id: string
+	format: string
+	md5: string
+	members: number
+}
+
+/**
+ * The archive of `product` among the listed downloads, the Great Britain one where an area is stated.
+ *
+ * @throws Naming the formats offered, when none matches.
+ */
+export function selectOSDownload(product: OSOpenProduct, downloads: readonly OSDownload[]): OSDownload {
+	const matches = downloads.filter((download) => download.format === product.format)
+	const chosen = matches.find((download) => download.area === "GB") ?? (matches.length === 1 ? matches[0] : undefined)
+
+	if (!chosen) {
+		throw new Error(
+			`${GB_OS_OPEN_ADAPTER_ID}: OS Downloads API lists no single ${product.format} archive for ${product.productID} ` +
+				`(listed: ${downloads.map((download) => `${download.format}/${download.area}`).join(", ") || "nothing"})`
+		)
+	}
+
+	return chosen
+}
+
+/**
+ * The archives the OS Downloads API lists for one product.
+ */
+export async function readOSDownloads(
+	client: Pick<APIClient, "fetch">,
+	product: OSOpenProduct,
+	options: { signal?: AbortSignal } = {}
+): Promise<OSDownload[]> {
+	const { data } = await client.fetch<OSDownload[]>({
+		method: "GET",
+		url: `${OS_DOWNLOADS_API_BASE}/products/${product.productID}/downloads`,
+		timeout: 60_000,
+		signal: options.signal,
+	})
+
+	if (!Array.isArray(data)) {
+		throw new TypeError(
+			`${GB_OS_OPEN_ADAPTER_ID}: OS Downloads API answered ${product.productID} with no download list`
+		)
+	}
+
+	return data
+}
+
+/**
+ * Downloads and unpacks one product.
+ */
+export async function downloadOSOpenProduct(
+	client: Pick<APIClient, "fetch">,
+	product: OSOpenProduct,
+	options: {
+		outputDir: PathBuilderLike
+		force?: boolean
+		retries?: number
+		retryDelayMs?: number
+		signal?: AbortSignal
+		report?: (line: string) => void
+	}
+): Promise<{ entry: OSOpenFileManifest; downloaded: boolean }> {
+	const { report } = options
+	const download = selectOSDownload(product, await readOSDownloads(client, product, { signal: options.signal }))
+	const productDir = PathBuilder.from(options.outputDir)(product.directory)
+	const archivePath = productDir(download.fileName)
+
+	report?.(
+		`=== ${GB_OS_OPEN_ADAPTER_ID} / ${product.productID} ${download.fileName} (${ByteFormatter.formatIEC(download.size)})`
+	)
+
+	await makeDirectories(productDir)
+
+	const present = options.force ? null : await md5File(archivePath).catch(() => null)
+	const downloaded = present !== download.md5
+
+	if (downloaded) {
+		await downloadToFile({
+			url: download.url,
+			dest: archivePath,
+			retries: options.retries,
+			retryDelayMs: options.retryDelayMs,
+			report,
+		})
+
+		const md5 = await md5File(archivePath)
+
+		if (md5 !== download.md5) {
+			throw new Error(
+				`${GB_OS_OPEN_ADAPTER_ID}: ${download.fileName} has md5 ${md5}, and OS publishes ${download.md5} for it`
+			)
+		}
+	} else {
+		report?.(`  present, and its md5 matches the published ${download.md5}`)
+	}
+
+	const members = await extractZipEntries(archivePath, productDir, { skipExisting: true })
+	const sha256 = await sha256File(archivePath)
+
+	report?.(`  ${members.length} members unpacked  sha256=${sha256}`)
+
+	return {
+		entry: {
+			source_url: download.url,
+			filename: download.fileName,
+			product_id: product.productID,
+			format: product.format,
+			bytes: download.size,
+			md5: download.md5,
+			sha256,
+			members: members.length,
+			downloaded_at: new Date().toISOString(),
+		},
+		downloaded,
+	}
+}
+
+/**
+ * Per-invocation options for the registry entry.
+ */
+export interface FetchGBOSOpenOptions extends BaseFetchOptions {
+	force?: boolean
+	retries?: number
+	signal?: AbortSignal
+}
+
+/**
+ * The registry entry: fetches all three products and writes `MANIFEST.json` at the adapter's input root.
+ *
+ * A product that fails is counted and the others are still taken.
+ */
+export async function fetchGBOSOpen(
+	options: FetchGBOSOpenOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const outputDir = options.outRoot(GB_OS_OPEN_ADAPTER_ID)
+	const files: OSOpenFileManifest[] = []
+	const failedCodes: string[] = []
+
+	let fetched = 0
+	let skipped = 0
+
+	await using client = new APIClient({ displayName: GB_OS_OPEN_ADAPTER_ID, minRequestIntervalMs: 250, retry: true })
+
+	for (const product of GB_OS_OPEN_PRODUCTS) {
+		if (options.signal?.aborted) break
+
+		try {
+			const { entry, downloaded } = await downloadOSOpenProduct(client, product, {
+				outputDir,
+				force: options.force,
+				retries: options.retries,
+				retryDelayMs: options.retryDelayMs,
+				signal: options.signal,
+				report,
+			})
+
+			files.push(entry)
+
+			if (downloaded) {
+				fetched += 1
+			} else {
+				skipped += 1
+			}
+		} catch (error) {
+			report?.(`  ✗ ${product.productID}: ${error instanceof Error ? error.message : String(error)}`)
+
+			failedCodes.push(product.productID)
+		}
+	}
+
+	const manifest: SourceCollectionManifest = {
+		source: GB_OS_OPEN_ADAPTER_ID,
+		source_url: OS_DOWNLOADS_API_BASE,
+		license: GB_OS_OPEN_LICENSE_NAME,
+		attribution: GB_OS_OPEN_ATTRIBUTION,
+		downloaded_at: new Date().toISOString(),
+		files,
+	}
+
+	await writeManifest(outputDir("MANIFEST.json"), manifest)
+
+	report?.(`  MANIFEST written to ${outputDir("MANIFEST.json")}`)
+
+	return { fetched, skipped, failed: failedCodes.length, failedCodes }
+}

package/lib/nl/tools/fetch/kadaster.ts

@@ -59,7 +59,7 @@ export const NL_KADASTER_FEED_URL = "https://service.pdok.nl/kadaster/ad/atom/ad
 export const NL_KADASTER_DOWNLOAD_FILENAME = "addresses.gml.gz"
 
 /**
- * The attribution recorded in the manifest.
+ * The manifest records this attribution.
  *
  * CC0 1.0 reserves no act and owes no attribution clause.
  * This attribution is a credit the repository chooses, rather than a condition of the grant.

package/lib/registers.ts

@@ -204,6 +204,13 @@ export const SourceRegister = {
 	 * HM Land Registry Price Paid Data, England and Wales.
 	 */
 	LandRegistryPricePaid: "gb-hm-land-registry-ppd",
+	/**
+	 * Ordnance Survey's OpenData products under the Open Government Licence v3.0.
+	 *
+	 * The `gb-os-open` adapter joins three of them per row.
+	 * Code-Point Open, OS Open Names and OS Open Roads share one publisher and one license.
+	 */
+	OrdnanceSurveyOpenData: "gb-os-opendata",
 	/**
 	 * Australia's Geocoded National Address File.
 	 */

package/lib/source-register/effective-manifest.ts

@@ -54,7 +54,8 @@ export function epochMixtureAuditPath(configPath: string): PathBuilder {
  */
 export const ExclusionReason = {
 	/**
-	 * The config's `source_weights` map does not name the source, so the sampler never considers it.
+	 * Neither the config's `source_weights` map nor its `source_reps` map lists the source,
+	 * so the sampler never considers it.
 	 */
 	UnweightedSource: "unweighted-source",
 
@@ -87,7 +88,8 @@ export interface EffectiveSourceRecord {
 	license: string
 
 	/**
-	 * The weight `source_weights` gives it, or `null` where the map omits it.
+	 * The weight `source_weights` gives it, the repetition count `source_reps` gives it,
+	 * or `null` where both maps omit it.
 	 */
 	weight: number | null
 
@@ -348,7 +350,11 @@ export interface EffectiveConfigView {
 }
 
 /**
- * Read `data.source_weights` and `data.augment_exclude_sources` from a training config.
+ * Read `data.source_weights`, `data.source_reps` and `data.augment_exclude_sources` from a training config.
+ *
+ * `source_weights` and `source_reps` both admit a source to the sampler, one as a share
+ * and one as a repetition count, so an entry in either block counts as the source's weight here.
+ * A source in neither block is one the sampler never considers.
  *
  * A line scanner instead of a YAML parse.
  * `corpus audit` already reads these blocks this way.
@@ -368,7 +374,7 @@ export function readConfigView(text: string): EffectiveConfigView {
 	const sourceWeights: Record<string, number> = {}
 	const augmentExcludeSources: string[] = []
 
-	type ConfigField = "source_weights" | "augment_exclude_sources"
+	type ConfigField = "source_weights" | "source_reps" | "augment_exclude_sources"
 
 	let field: ConfigField | null = null
 	let fieldIndent = -1
@@ -385,7 +391,7 @@ export function readConfigView(text: string): EffectiveConfigView {
 		}
 
 		if (!field) {
-			const opener = /^([\t ]*)(source_weights|augment_exclude_sources):\s*(#.*)?$/.exec(raw)
+			const opener = /^([\t ]*)(source_weights|source_reps|augment_exclude_sources):\s*(#.*)?$/.exec(raw)
 
 			if (opener) {
 				field = opener[2] as ConfigField
@@ -399,7 +405,7 @@ export function readConfigView(text: string): EffectiveConfigView {
 			continue
 		}
 
-		if (field === "source_weights") {
+		if (field === "source_weights" || field === "source_reps") {
 			const entry = /^[\t ]+([\w-]+):\s*([\d.]+)/.exec(raw)
 
 			if (entry) {