@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/kr/adapters/localdata/key-index.ts

@@ -0,0 +1,185 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The LABEL register's key sets, the only thing a permit string is allowed to align against (#2204 §5).
+ *
+ *   The permit registry is a NOISY source: a clerk typed each address, in both address systems, with no validation. So
+ *   nothing in a permit string is taken on its own word. The region must be a 시도 the register lists, the 시군구 one
+ *   that region lists, the road one that 시군구 lists, the 동 one that 시군구 lists, the 리 one that 동 lists. A string
+ *   that satisfies the whole key becomes a training row; one that does not is a BOARD row — an address the model will
+ *   be read on and never trained on.
+ *
+ *   Built by one pass over the LABEL rows, which is why it lives beside them rather than inside the aligner.
+ */
+
+import { type JusoLabelRow, REGION_ALIASES } from "#kr/adapters/juso/label-rows"
+
+/**
+ * The separator between the parts of a composite key, written as an escape so no NUL byte enters the source.
+ *
+ * A space cannot serve. A 시군구 is written with one (`수원시 장안구`), so `unitKey("A", "B C")` and `unitKey("A B", "C")` would
+ * produce one key for two different places. No place name contains NUL.
+ */
+const UNIT_SEPARATOR = "\0"
+
+/**
+ * The key sets, keyed as the register nests them.
+ *
+ * A composite string key where the register's own key is a tuple, because a nested map costs a second lookup on the hot
+ * path and every permit row does several.
+ */
+export interface KeyIndex {
+	regions: Set<string>
+	/**
+	 * 시도 → its 시군구 names.
+	 */
+	sigunguByRegion: Map<string, Set<string>>
+	/**
+	 * 시도 and 시군구 → the unit's road names.
+	 */
+	roadsByUnit: Map<string, Set<string>>
+	/**
+	 * 시도 and 시군구 → the unit's 동 and 읍/면 names.
+	 */
+	dongsByUnit: Map<string, Set<string>>
+	/**
+	 * 시도, 시군구 and 동 → the 동's 리 names.
+	 */
+	risByDong: Map<string, Set<string>>
+}
+
+/**
+ * The parts of a composite key, joined so no two different part lists produce the same key.
+ */
+export function unitKey(...parts: readonly string[]): string {
+	return parts.join(UNIT_SEPARATOR)
+}
+
+export function emptyKeyIndex(): KeyIndex {
+	return {
+		regions: new Set(),
+		sigunguByRegion: new Map(),
+		roadsByUnit: new Map(),
+		dongsByUnit: new Map(),
+		risByDong: new Map(),
+	}
+}
+
+function add(map: Map<string, Set<string>>, key: string, value: string): void {
+	const existing = map.get(key)
+
+	if (existing) {
+		existing.add(value)
+	} else {
+		map.set(key, new Set([value]))
+	}
+}
+
+/**
+ * Add one LABEL row's names to the key sets.
+ *
+ * The 읍/면 joins the road set's unit as a road-form token too, since the road address writes it between the 시군구 and the
+ * road.
+ */
+export function indexLabelRow(index: KeyIndex, row: JusoLabelRow): void {
+	index.regions.add(row.region)
+	add(index.sigunguByRegion, row.region, row.sigungu)
+
+	const unit = unitKey(row.region, row.sigungu)
+
+	add(index.roadsByUnit, unit, row.road)
+
+	if (row.eupmyeon) {
+		add(index.dongsByUnit, unit, row.eupmyeon)
+	}
+
+	if (row.dong) {
+		add(index.dongsByUnit, unit, row.dong)
+
+		if (row.ri) {
+			add(index.risByDong, unitKey(row.region, row.sigungu, row.dong), row.ri)
+		}
+	}
+}
+
+/**
+ * Admit the pre-merger region names as aliases of the register's current one, SHARING its key sets.
+ *
+ * The sets are shared by reference rather than copied: an alias is the same place under an older name, and a later
+ * `indexLabelRow` under either name must reach the same set.
+ */
+export function aliasKeyIndex(index: KeyIndex): void {
+	for (const [legacy, current] of Object.entries(REGION_ALIASES)) {
+		if (!index.regions.has(current)) continue
+
+		index.regions.add(legacy)
+
+		const sigungu = index.sigunguByRegion.get(current)
+
+		if (sigungu) {
+			index.sigunguByRegion.set(legacy, sigungu)
+		}
+
+		// oxlint-disable no-useless-spread -- each loop writes into the map it reads, so it walks a snapshot.
+		for (const [key, roads] of [...index.roadsByUnit]) {
+			const [region, unit] = key.split(UNIT_SEPARATOR)
+
+			if (region === current) {
+				index.roadsByUnit.set(unitKey(legacy, unit!), roads)
+			}
+		}
+
+		for (const [key, dongs] of [...index.dongsByUnit]) {
+			const [region, unit] = key.split(UNIT_SEPARATOR)
+
+			if (region === current) {
+				index.dongsByUnit.set(unitKey(legacy, unit!), dongs)
+			}
+		}
+
+		for (const [key, ris] of [...index.risByDong]) {
+			const [region, unit, dong] = key.split(UNIT_SEPARATOR)
+
+			if (region === current) {
+				index.risByDong.set(unitKey(legacy, unit!, dong!), ris)
+			}
+		}
+		// oxlint-enable no-useless-spread
+	}
+}
+
+/**
+ * How many tokens from `start` form one of the region's 시군구 — 2 for `수원시 장안구`, 1 for `종로구`, 0 for none.
+ *
+ * Two-token first: `수원시` alone is also a listed name, and taking it would leave `장안구` to be read as a road.
+ */
+export function sigunguSpan(index: KeyIndex, region: string, tokens: readonly string[], start: number): number {
+	const candidates = index.sigunguByRegion.get(region)
+
+	if (!candidates) return 0
+
+	for (const width of [2, 1]) {
+		if (start + width <= tokens.length && candidates.has(tokens.slice(start, start + width).join(" "))) {
+			return width
+		}
+	}
+
+	return 0
+}
+
+/**
+ * The key sets the permit aligner reads, from one pass over the LABEL rows.
+ */
+export async function buildKeyIndex(rows: AsyncIterable<JusoLabelRow>): Promise<KeyIndex> {
+	const index = emptyKeyIndex()
+
+	for await (const row of rows) {
+		indexLabelRow(index, row)
+	}
+
+	aliasKeyIndex(index)
+
+	return index
+}

package/lib/kr/adapters/localdata/permit-rows.ts

@@ -0,0 +1,186 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The Korean permit registry (지방행정인허가데이터) as a NOISY corpus source: reading the delivered CSVs, and turning
+ *   an aligned string into a corpus row (#2204 §5).
+ *
+ *   The publisher delivers CP949, one file per business category, with both address systems typed by a clerk plus a
+ *   planar coordinate in EPSG:5174 and both postcodes. Reading is separated from ALIGNING so the alignment rate per
+ *   file can be measured before any row enters a corpus.
+ */
+
+import { decodeByteStream, openReadStream } from "@mailwoman/core/fs/streams"
+import { basename, extname, join, type PathBuilderLike } from "path-ts"
+import { CSVSpliterator } from "spliterator"
+import { Globerator } from "spliterator/node/fs"
+
+import type { Aligned } from "#kr/adapters/localdata/align"
+
+/**
+ * The source label every row of this adapter carries, and the name a config's `source_weights` addresses it by.
+ */
+export const SOURCE = "localdata-kr"
+
+/**
+ * Every permit in this registry is Korean; the publisher issues no others.
+ */
+export const COUNTRY = "KR"
+
+/**
+ * The 영업상태명 of a business still trading. A closed one's address records somewhere that was.
+ */
+export const OPEN_STATUS = "영업/정상"
+
+/**
+ * The publisher's encoding. `cp949` rather than `euc-kr` for the reason `decodeByteStream` states.
+ */
+const ENCODING = "cp949"
+
+/**
+ * One permit, with both address systems as the clerk typed them.
+ */
+export interface PermitRow {
+	/**
+	 * The file's basename: the business category the publisher splits by.
+	 */
+	category: string
+	name: string
+	status: string
+	roadAddress: string
+	lotAddress: string
+	roadPostcode: string
+	lotPostcode: string
+	x: number | null
+	y: number | null
+}
+
+/**
+ * The first of these column spellings the row carries, trimmed, or the empty string.
+ *
+ * The publisher's older and newer exports name the same field differently — `도로명주소` against `도로명전체주소`, `좌표정보(X)`
+ * against `좌표정보(x)` — and a directory holds both vintages, so each field names every spelling it has been delivered
+ * under.
+ */
+function firstColumn(row: Record<string, string | undefined>, ...names: readonly string[]): string {
+	for (const name of names) {
+		const value = row[name]
+
+		if (value !== undefined && value !== null) return String(value).trim()
+	}
+
+	return ""
+}
+
+function toNumber(value: string): number | null {
+	if (!value.trim()) return null
+
+	const parsed = Number(value)
+
+	return Number.isFinite(parsed) ? parsed : null
+}
+
+export interface ReadPermitOptions {
+	/**
+	 * Keep only these 영업상태명 values. Empty admits every status.
+	 */
+	statuses?: readonly string[]
+}
+
+/**
+ * Stream every permit of one CSV.
+ */
+export async function* readPermitFile(
+	path: PathBuilderLike,
+	options: ReadPermitOptions = {}
+): AsyncGenerator<PermitRow> {
+	const wanted = new Set(options.statuses ?? [OPEN_STATUS])
+	const category = basename(String(path), extname(String(path)))
+	const bytes = openReadStream(path)
+
+	for await (const row of CSVSpliterator.fromAsync<Record<string, string>>(decodeByteStream(bytes, ENCODING))) {
+		const status = firstColumn(row, "영업상태명")
+
+		if (wanted.size && !wanted.has(status)) continue
+
+		const roadAddress = firstColumn(row, "도로명주소", "도로명전체주소")
+		const lotAddress = firstColumn(row, "지번주소", "소재지전체주소")
+
+		if (!roadAddress && !lotAddress) continue
+
+		yield {
+			category,
+			name: firstColumn(row, "사업장명"),
+			status,
+			roadAddress,
+			lotAddress,
+			roadPostcode: firstColumn(row, "도로명우편번호"),
+			lotPostcode: firstColumn(row, "소재지우편번호"),
+			x: toNumber(firstColumn(row, "좌표정보(X)", "좌표정보(x)")),
+			y: toNumber(firstColumn(row, "좌표정보(Y)", "좌표정보(y)")),
+		}
+	}
+}
+
+/**
+ * Stream every permit of every CSV in a directory, in filename order.
+ */
+export async function* readPermitDirectory(
+	directory: PathBuilderLike,
+	options: ReadPermitOptions = {}
+): AsyncGenerator<PermitRow> {
+	const files = await Globerator.from("*", { cwd: directory, withFileTypes: true, onlyFiles: false })
+		.filter((entry) => entry.isFile() && entry.name.endsWith(".csv"))
+		.map((entry) => entry.name)
+		.toSorted()
+
+	for (const name of files) {
+		yield* readPermitFile(join(directory, name), options)
+	}
+}
+
+/**
+ * One aligned string as a row in the CJK slice schema.
+ *
+ * The token labels are derived from the spans rather than carried beside them: a token is `B-<tag>` when its FIRST
+ * character falls inside a span, `O` otherwise. A span covering several tokens therefore labels only the token it
+ * starts in, which is what the char-path trainer reads the spans for.
+ */
+export function alignedToSliceRow(aligned: Aligned): Record<string, unknown> {
+	const raw = aligned.raw
+	const tokens: string[] = []
+	const labels: string[] = []
+	let cursor = 0
+
+	for (const token of raw.split(/\s+/u).filter((piece) => piece.length)) {
+		const index = raw.indexOf(token, cursor)
+
+		cursor = index + token.length
+
+		let label = "O"
+
+		for (let span = 0; span < aligned.spanStarts.length; span += 1) {
+			if (aligned.spanStarts[span]! <= index && index < aligned.spanEnds[span]!) {
+				label = `B-${aligned.spanTags[span]}`
+
+				break
+			}
+		}
+
+		tokens.push(token)
+		labels.push(label)
+	}
+
+	return {
+		raw,
+		tokens,
+		labels,
+		span_starts: [...aligned.spanStarts],
+		span_ends: [...aligned.spanEnds],
+		span_tags: [...aligned.spanTags],
+		country: COUNTRY,
+		source: SOURCE,
+		register: aligned.register,
+	}
+}

package/lib/kr/tools/fetch/juso.ts

@@ -0,0 +1,172 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the Korean road-name address register (도로명주소 주소DB) from the ministry's address-industry
+ *   portal, business.juso.go.kr. This is the LABEL source for Korean addresses: about 6,000,000
+ *   road-name addresses and 8,000,000 lot-number (지번) records, split by the publisher into four
+ *   pipe-delimited CP949 text files per region (도로명코드 / 주소 / 지번 / 부가정보), with the postcode in
+ *   the supplementary file. No coordinates. The public-data portal labels it "이용허락범위 제한 없음"
+ *   (no restriction on the scope of use), and it is a direct download with no application step — the
+ *   distinction that matters against the ministry's entrance-coordinate products, which are provided
+ *   only after a purpose-of-use review (`docs/superpowers/plans/counsel-dossier.md` §4).
+ *
+ *   The portal is a single-page app; the file behind it is reached in two calls. `selectAttrbDBDwldList`
+ *   lists the monthly full files for one product (`rtlDtaDtlSn` 8 is the 주소DB, 2 the English
+ *   road-name DB), and `/api/jst/download` streams one by the parameters the listing carried. The
+ *   English DB rides along because it is the Latin-script half of the same register.
+ *
+ *   Invoke via `mailwoman corpus fetch juso-kr --out-root <path>`.
+ */
+
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { join } from "path-ts"
+
+import type {
+	BaseFetchOptions,
+	FetchSummary,
+	SourceCollectionManifest,
+	SourceManifest,
+} from "#tools/fetch/download/index"
+import { resumableDownload, writeManifest } from "#tools/fetch/download/index"
+
+const SLUG = "juso-kr"
+const PORTAL = "https://business.juso.go.kr"
+const LIST_URL = `${PORTAL}/api/jst/selectAttrbDBDwldList`
+const DOWNLOAD_URL = `${PORTAL}/api/jst/download`
+const REFERER = `${PORTAL}/jst/jstAddressDownload?menu=30`
+
+/**
+ * The portal's product ids for the two registers this module fetches.
+ */
+const PRODUCTS = [
+	{ sn: "8", name: "주소DB" },
+	{ sn: "2", name: "도로명주소 영문" },
+] as const
+
+const LICENSE =
+	"이용허락범위 제한 없음 (data.go.kr label for 행정안전부_도로명주소 주소DB); 공공누리 attribution to 행정안전부"
+
+const ATTRIBUTION = "행정안전부 (Ministry of the Interior and Safety), 도로명주소 (juso.go.kr)"
+
+interface ListedFile {
+	crtrYm: string
+	fileTypeNm: string
+	fileNm: string
+	isExist: "Y" | "N"
+	ctpvClsfCd: string
+	tmprFileNm: string
+	fileSn: string | null
+	atflNo: string | null
+}
+
+interface ListResponse {
+	results?: { allMonthFileList?: ListedFile[] }
+}
+
+export interface FetchJusoKROptions extends BaseFetchOptions {
+	/**
+	 * The month to fetch as `YYYYMM`. Defaults to the latest month the portal lists as present.
+	 */
+	month?: string
+}
+
+const BROWSER_HEADERS = {
+	"user-agent": "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch",
+	referer: REFERER,
+}
+
+/**
+ * The monthly full files the portal lists for one product over the twelve months ending at `year`/`month`.
+ */
+async function listMonthlyFiles(sn: string, year: number, month: number): Promise<ListedFile[]> {
+	const res = await fetch(LIST_URL, {
+		method: "POST",
+		headers: { ...BROWSER_HEADERS, accept: "application/json", "content-type": "application/json;charset=UTF-8" },
+		body: JSON.stringify({ rtlDtaDtlSn: sn, year, month, expand: "Y" }),
+	})
+
+	if (!res.ok) throw new Error(`juso-kr: listing product ${sn} answered HTTP ${res.status}`)
+	const json = (await res.json()) as ListResponse
+
+	return (json.results?.allMonthFileList ?? []).filter((file) => file.isExist === "Y")
+}
+
+function downloadURL(file: ListedFile, regYmd: string): string {
+	const query = new URLSearchParams({
+		reqType: file.fileTypeNm,
+		ctprvnCd: file.ctpvClsfCd,
+		stdde: file.crtrYm,
+		fileName: file.fileNm,
+		realFileName: file.tmprFileNm,
+		intFileNo: file.fileSn ?? "0",
+		intNum: file.atflNo ?? "0",
+		regYmd,
+	})
+
+	return `${DOWNLOAD_URL}?${query.toString()}`
+}
+
+export async function fetchJusoKR(options: FetchJusoKROptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	await makeDirectories(destDir)
+
+	const now = new Date()
+	const files: SourceManifest[] = []
+	const failedCodes: string[] = []
+
+	for (const product of PRODUCTS) {
+		const listed = await listMonthlyFiles(product.sn, now.getFullYear(), now.getMonth() + 1)
+		const file = options.month ? listed.find((entry) => entry.crtrYm === options.month) : listed.at(-1)
+
+		if (!file) {
+			report?.(`  ✗ ${product.name}: no full file listed${options.month ? ` for ${options.month}` : ""}`)
+			failedCodes.push(product.name)
+
+			continue
+		}
+
+		const url = downloadURL(file, String(now.getFullYear()))
+		const dest = join(destDir, file.tmprFileNm)
+		report?.(`=== ${SLUG} / ${product.name} ${file.crtrYm} → ${file.tmprFileNm}`)
+		let bytes: number
+
+		try {
+			// The portal drops a connection every few megabytes and answers Range with 206, so the transfer resumes
+			// from what landed rather than starting the 181 MB over.
+			bytes = await resumableDownload({ url, dest, headers: BROWSER_HEADERS, report })
+		} catch (error) {
+			report?.(`  ✗ ${(error as Error).message}`)
+			failedCodes.push(product.name)
+
+			continue
+		}
+
+		const sha = await sha256File(dest)
+
+		files.push({
+			source_url: url,
+			downloaded_at: new Date().toISOString(),
+			filename: file.tmprFileNm,
+			sha256: sha,
+			bytes,
+		})
+
+		report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
+	}
+
+	const manifest: SourceCollectionManifest = {
+		source: SLUG,
+		source_url: REFERER,
+		license: LICENSE,
+		attribution: ATTRIBUTION,
+		downloaded_at: new Date().toISOString(),
+		files,
+	}
+
+	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+
+	return { fetched: files.length, skipped: 0, failed: failedCodes.length, failedCodes }
+}