@mailwoman/corpus 9.3.0 → 9.4.0

package/lib/tools/fetch/download.ts

@@ -6,9 +6,9 @@
  *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
  */
 
-import { pathExists, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { pathExists, readLocalTextFile, tryStat } from "@mailwoman/core/fs/readers"
 import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
-import { writeLocalFile, writeLocalTextFile } from "@mailwoman/core/fs/writers"
+import { movePath, writeLocalFile, writeLocalTextFile } from "@mailwoman/core/fs/writers"
 import { tryParsingJSON } from "@mailwoman/core/json"
 import { sleep } from "@mailwoman/core/utils/sleep"
 import type { PathBuilderLike } from "path-ts"
@@ -18,6 +18,17 @@ import type { PathBuilderLike } from "path-ts"
  */
 const HTTP_TOO_MANY_REQUESTS = 429
 
+/**
+ * The answer to a satisfiable `Range` request; anything else means the host ignored the range and would send the whole
+ * body again.
+ */
+const HTTP_PARTIAL_CONTENT = 206
+
+/**
+ * The answer to a range that starts at or past the end of the body: the resume point is already the whole file.
+ */
+const HTTP_RANGE_NOT_SATISFIABLE = 416
+
 /**
  * Lowest 5xx status. Server-side failures are retryable; 4xx are not.
  */
@@ -260,3 +271,207 @@ export async function loadManifestEntries<T>(path: string, key: (entry: T) => st
 export async function writeManifest(path: string, manifest: unknown): Promise<void> {
 	await writeLocalTextFile(JSON.stringify(manifest, null, 2) + "\n", path)
 }
+
+/**
+ * The sibling `MANIFEST.json` shape for a source that is a COLLECTION of files behind one portal (a monthly register
+ * published per region, per industry, or per first letter). It carries what a trained artifact has to be able to cite
+ * later: the license the portal labels the data with, the attribution wording it requires, and one
+ * {@link SourceManifest} per file.
+ */
+export interface SourceCollectionManifest {
+	source: string
+	source_url: string
+	license: string
+	attribution: string
+	downloaded_at: string
+	files: SourceManifest[]
+}
+
+/**
+ * Pipe a response body to `dest` and answer the byte count. The one primitive the portal fetchers share when the
+ * request is not a bare GET — a session cookie, a CSRF header, or a form POST stands between the listing and the file,
+ * so {@link streamDownload}'s URL-only contract does not fit and each module builds its own `Response` first. Writes a
+ * `.tmp` sibling and renames, so an interrupted transfer never lands at the final path looking complete.
+ */
+export async function streamBodyToFile(res: Response, dest: string): Promise<number> {
+	if (!res.body) throw new HTTPStatusError(res.status, `HTTP ${res.status} with no body — ${res.url}`)
+	let bytes = 0
+
+	const counter = new TransformStream<Uint8Array, Uint8Array>({
+		transform(chunk, controller) {
+			bytes += chunk.byteLength
+			controller.enqueue(chunk)
+		},
+	})
+
+	const tmp = dest + ".tmp"
+	await pipeline(Readable.fromWeb(res.body.pipeThrough(counter)), openWriteStream(tmp))
+	await movePath(tmp, dest)
+
+	return bytes
+}
+
+/**
+ * The per-file entries of a {@link SourceCollectionManifest} keyed by file name, or an empty map when the manifest is
+ * missing or not in the collection shape, so a re-run after an interruption fetches only what is missing. The
+ * single-file modules' `loadManifestEntries` reads a bare array and is not this.
+ */
+export async function loadCollectionFiles(path: string): Promise<Map<string, SourceManifest>> {
+	const parsed = await readManifest<SourceCollectionManifest>(path)
+	const entries = new Map<string, SourceManifest>()
+
+	for (const entry of Array.isArray(parsed?.files) ? parsed.files : []) {
+		entries.set(entry.filename, entry)
+	}
+
+	return entries
+}
+
+/**
+ * Run one transfer up to `1 + retries` times, pausing `retryDelayMs` between attempts. A government portal drops a
+ * multi-hundred-megabyte connection often enough that a collection fetch which lets the first `TypeError: terminated`
+ * propagate loses the whole run to one file; the per-file loop calls the transfer through this and records the failure
+ * only once the attempts are spent. The last error is rethrown so the caller can name the file it lost.
+ */
+export async function withRetries<T>(
+	transfer: () => Promise<T>,
+	options: { retries?: number; retryDelayMs?: number; report?: (line: string) => void; label?: string } = {}
+): Promise<T> {
+	const { retries = 3, retryDelayMs = DEFAULT_RETRY_DELAY_MS, report, label = "transfer" } = options
+	let lastError: unknown
+
+	for (let attempt = 0; attempt <= retries; attempt++) {
+		if (attempt > 0) {
+			report?.(`  retry ${attempt}/${retries} after ${retryDelayMs}ms — ${label}`)
+			await sleep(retryDelayMs)
+		}
+
+		try {
+			return await transfer()
+		} catch (error) {
+			lastError = error
+		}
+	}
+
+	throw lastError instanceof Error ? lastError : new Error(String(lastError))
+}
+
+/**
+ * The byte count already at `path`, or 0 when nothing is there — the resume point of an interrupted range download.
+ */
+async function bytesOnDisk(path: string): Promise<number> {
+	const stat = await tryStat(path)
+
+	return stat?.size ?? 0
+}
+
+/**
+ * Download `url` to `dest` in ranges, resuming from whatever the `.tmp` sibling already holds. For a host that drops a
+ * long connection every few megabytes but answers `Range` with 206 (the Korean address portal does both, measured at
+ * 0.9–12 MB per connection against a 181 MB file), a whole-body transfer never finishes and a plain retry starts over;
+ * this one asks for the remainder each time and keeps what landed. The total comes from the first `Content-Range`, and
+ * the loop gives up after `maxConnections` connections so a host that keeps answering 206 with no bytes cannot spin.
+ * Answers the byte count written; throws when the host answers anything but 206 for a range.
+ */
+export async function resumableDownload(options: {
+	url: string
+	dest: string
+	headers?: Record<string, string>
+	maxConnections?: number
+	retryDelayMs?: number
+	report?: (line: string) => void
+}): Promise<number> {
+	const { url, dest, headers = {}, maxConnections = 400, retryDelayMs = DEFAULT_RETRY_DELAY_MS, report } = options
+	const tmp = dest + ".tmp"
+	let have = await bytesOnDisk(tmp)
+	let total: number | undefined
+
+	for (let connection = 0; connection < maxConnections; connection++) {
+		if (total !== undefined && have >= total) break
+
+		let res: Response
+
+		try {
+			res = await fetch(url, { headers: { ...headers, range: `bytes=${have}-` }, signal: AbortSignal.timeout(600_000) })
+		} catch {
+			await sleep(retryDelayMs)
+
+			continue
+		}
+
+		if (res.status === HTTP_RANGE_NOT_SATISFIABLE && have > 0) {
+			// Nothing past `have`: the file on disk is already the whole body (a parallel filler or an earlier run
+			// landed it), and `Content-Range: bytes */<total>` says how long it is.
+			const whole = /\*\/(\d+)/.exec(res.headers.get("content-range") ?? "")?.[1]
+			total = whole ? Number(whole) : have
+			await res.body?.cancel()
+
+			break
+		}
+
+		if (res.status !== HTTP_PARTIAL_CONTENT) {
+			throw new HTTPStatusError(res.status, `HTTP ${res.status} for a range request — ${url}`)
+		}
+
+		const range = /bytes \d+-\d+\/(\d+)/.exec(res.headers.get("content-range") ?? "")?.[1]
+
+		if (range) {
+			total = Number(range)
+		}
+
+		const before = have
+
+		try {
+			// Append: the stream opens the file for appending so a partial body extends what earlier connections left.
+			await pipeline(
+				Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]),
+				openWriteStream(tmp, { flags: "a" })
+			)
+		} catch {
+			// The connection dropped mid-body; the bytes that landed are on disk, and the next range starts after them.
+		}
+
+		have = await bytesOnDisk(tmp)
+
+		if (have === before) {
+			await sleep(retryDelayMs)
+		} else {
+			report?.(`  ${(have / 1024 / 1024).toFixed(1)} MB${total ? ` of ${(total / 1024 / 1024).toFixed(1)} MB` : ""}`)
+		}
+	}
+
+	if (total === undefined || have < total) {
+		throw new Error(`resumable download stalled at ${have} of ${total ?? "?"} bytes — ${url}`)
+	}
+
+	await movePath(tmp, dest)
+
+	return have
+}
+
+/**
+ * The `Set-Cookie` values of a response folded into one `Cookie` header value, so a second request to the same portal
+ * carries the session the first one opened.
+ */
+export function cookieHeader(res: Response): string {
+	return res.headers
+		.getSetCookie()
+		.map((cookie) => cookie.split(";")[0])
+		.join("; ")
+}
+
+/**
+ * The file name a `Content-Disposition: attachment` header names, RFC 5987 form (`filename*=utf-8''…`) first, plain
+ * `filename="…"` second, or `fallback` when the header carries neither.
+ */
+export function attachmentFilename(res: Response, fallback: string): string {
+	const disposition = res.headers.get("content-disposition") ?? ""
+	const extended = /filename\*=(?:utf-8|UTF-8)'[^']*'([^;]+)/.exec(disposition)?.[1]
+
+	if (extended) return decodeURIComponent(extended.trim())
+	const plain = /filename="?([^";]+)"?/.exec(disposition)?.[1]
+
+	if (plain) return decodeURIComponent(plain.trim())
+
+	return fallback
+}

package/lib/tools/fetch/gcis-tw.ts

@@ -0,0 +1,175 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch Taiwan's company and business registers (公司登記資料, 商業登記資料) from the Ministry of
+ *   Economic Affairs' commerce open-data platform, data.gcis.nat.gov.tw. The platform publishes the
+ *   full registers as CSV files split by region (the six special municipalities each on their own,
+ *   the rest of the island in four bands) and by industry letter, 148 files in all, each row carrying
+ *   the registered address (公司地址) and, for companies, the tax office's business address
+ *   (營業地址) as free text. This is the NOISY source for Taiwanese addresses: a person typed these.
+ *
+ *   License: 政府資料開放授權條款－第1版 (Open Government Data License, Taiwan, v1.0). The platform's
+ *   own dialog states the condition that binds: attribution in the form it prescribes, or the grant is
+ *   void from the start ("未盡顯名標示義務者，視為自始未取得開放資料之授權"). The manifest records the
+ *   prescribed wording per file so the corpus build and the model card can carry it.
+ *
+ *   Invoke via `mailwoman corpus fetch gcis-tw --out-root <path>`.
+ */
+
+import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
+import { pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { sleep } from "@mailwoman/core/utils/sleep"
+import { join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"
+import { loadCollectionFiles, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download"
+
+const SLUG = "gcis-tw"
+const PORTAL = "https://data.gcis.nat.gov.tw"
+const CATALOG_URL = `${PORTAL}/od/datacategory`
+const LICENSE = "政府資料開放授權條款－第1版 (Open Government Data License, Taiwan, v1.0) — http://data.gov.tw/license"
+const ATTRIBUTION = "提供機關／經濟部商業發展署 [<dataset title>] — the 顯名聲明 each file's dialog prescribes"
+const PACE_MS = 300
+
+/**
+ * The bulk register files: `<region><register>資料-<industry letter><industry>`. The per-industry API datasets and the
+ * monthly new-registration lists share the catalog and are not these.
+ */
+const BULK_TITLE = /(公司登記資料|商業登記資料)-[A-Z]/
+
+export type FetchGCISTWOptions = BaseFetchOptions
+
+interface Dataset {
+	title: string
+	detailPath: string
+}
+
+/**
+ * A file name a shell and a manifest can carry: the title with its punctuation folded to underscores.
+ */
+function filenameFor(title: string): string {
+	return `${title
+		.replaceAll(/[()（）、，/\\\s]+/g, "_")
+		.replaceAll(/_+/g, "_")
+		.replaceAll(/^_|_$/g, "")}.csv`
+}
+
+async function listBulkDatasets(): Promise<Dataset[]> {
+	const res = await fetch(CATALOG_URL, { headers: { accept: "text/html" } })
+
+	if (!res.ok) throw new Error(`gcis-tw: the catalog answered HTTP ${res.status}`)
+	const html = await res.text()
+	const datasets: Dataset[] = []
+
+	for (const [, path, rawTitle] of html.matchAll(/<a[^>]*href="(\/od\/detail[^"]*)"[^>]*>([^<]+)<\/a>/g)) {
+		const title = rawTitle?.trim()
+
+		if (!path || !title || !BULK_TITLE.test(title)) continue
+		datasets.push({ title, detailPath: path.replace(/;jsessionid=[^?]*/, "") })
+	}
+
+	return datasets
+}
+
+/**
+ * The `/od/file?oid=…` link a dataset's detail page hands its download dialog.
+ */
+async function fileURLFor(dataset: Dataset): Promise<string | undefined> {
+	const res = await fetch(`${PORTAL}${dataset.detailPath}`, { headers: { accept: "text/html" } })
+
+	if (!res.ok) return undefined
+	const html = await res.text()
+	const path = /showDialog\('(\/od\/file\?oid=[^']+)'\)/.exec(html)?.[1]
+
+	return path ? `${PORTAL}${path}` : undefined
+}
+
+export async function fetchGCISTW(options: FetchGCISTWOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	await makeDirectories(destDir)
+	const manifestPath = join(destDir, "MANIFEST.json")
+
+	const datasets = await listBulkDatasets()
+	report?.(`=== ${SLUG}: ${datasets.length} bulk register files in the catalog`)
+
+	const previous = await loadCollectionFiles(manifestPath)
+	const files = new Map<string, SourceManifest>()
+	let fetched = 0
+	let skipped = 0
+	const failedCodes: string[] = []
+
+	for (const dataset of datasets) {
+		const filename = filenameFor(dataset.title)
+		const dest = join(destDir, filename)
+		const before = previous.get(filename)
+
+		if (before && (await pathExists(dest))) {
+			files.set(filename, before)
+
+			skipped++
+
+			continue
+		}
+
+		await sleep(PACE_MS)
+		const url = await fileURLFor(dataset)
+
+		if (!url) {
+			report?.(`  ✗ no file link on the detail page for ${dataset.title}`)
+			failedCodes.push(dataset.title)
+
+			continue
+		}
+
+		report?.(`--- ${dataset.title}`)
+		let bytes: number
+
+		try {
+			bytes = await withRetries(
+				async () => {
+					const res = await fetch(url, { headers: { accept: "*/*" }, signal: AbortSignal.timeout(1_800_000) })
+
+					if (!res.ok) throw new Error(`HTTP ${res.status} for ${dataset.title}`)
+
+					return streamBodyToFile(res, dest)
+				},
+				{ report, label: dataset.title }
+			)
+		} catch (error) {
+			report?.(`  ✗ ${(error as Error).message}`)
+			failedCodes.push(dataset.title)
+
+			continue
+		}
+
+		if (bytes < BYTES_PER_KIB) {
+			report?.(`  ✗ ${bytes} bytes — an error page, not the register`)
+			failedCodes.push(dataset.title)
+
+			continue
+		}
+
+		const sha = await sha256File(dest)
+		files.set(filename, { source_url: url, downloaded_at: new Date().toISOString(), filename, sha256: sha, bytes })
+
+		fetched++
+		report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
+
+		const manifest: SourceCollectionManifest = {
+			source: SLUG,
+			source_url: CATALOG_URL,
+			license: LICENSE,
+			attribution: ATTRIBUTION,
+			downloaded_at: new Date().toISOString(),
+			files: [...files.values()],
+		}
+
+		await writeManifest(manifestPath, manifest)
+	}
+
+	return { fetched, skipped, failed: failedCodes.length, failedCodes }
+}

package/lib/tools/fetch/houjin-jp.ts

@@ -0,0 +1,135 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the National Tax Agency's corporate-number register (法人番号公表サイト 全件データ): every
+ *   corporation assigned a number, about 5,000,000 rows, with the head-office address as one string
+ *   (国内所在地) plus the prefecture and municipality split out and the postcode. The nationwide CSV
+ *   in Unicode is one zip of about 255 MB. This is the NOISY source for Japanese addresses; the
+ *   Overture rows the JP corpus is built from are the LABEL half.
+ *
+ *   The agency states the three published fields may be used freely by anyone ("どなたでも自由にご利用
+ *   いただくことができます"), with no attribution condition on the download page.
+ *
+ *   The file sits behind a form: the page hands out a per-session token, and a POST with that token,
+ *   `event=download` and the file number of the nationwide Unicode CSV answers the zip. The file
+ *   number is read off the page rather than pinned, because the agency re-issues the files monthly.
+ *
+ *   Invoke via `mailwoman corpus fetch houjin-jp --out-root <path>`.
+ */
+
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest } from "#tools/fetch/download"
+import { attachmentFilename, cookieHeader, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download"
+
+const SLUG = "houjin-jp"
+const PAGE_URL = "https://www.houjin-bangou.nta.go.jp/download/zenken/"
+const FORM_URL = `${PAGE_URL}index.html`
+const TOKEN_FIELD = "jp.go.nta.houjin_bangou.framework.web.common.CNSFWTokenProcessor.request.token"
+const LICENSE = "国税庁法人番号公表サイト — the three published fields are free for anyone to use (利用規約)"
+const ATTRIBUTION = "国税庁 法人番号公表サイト (National Tax Agency, Corporate Number Publication Site)"
+const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch"
+
+export type FetchHoujinJPOptions = BaseFetchOptions
+
+/**
+ * The nationwide row of the "CSV形式・Unicode" table carries the file number in its `doDownload(N)` handler.
+ */
+function nationwideUnicodeFileNumber(html: string): string | undefined {
+	const start = html.indexOf('id="csv-unicode"')
+
+	if (start === -1) return undefined
+	const section = html.slice(start, html.indexOf('id="xml-unicode"', start))
+	const row = /全国[\s\S]{0,600}?doDownload\((\d+)\)/.exec(section)
+
+	return row?.[1]
+}
+
+export async function fetchHoujinJP(
+	options: FetchHoujinJPOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	await makeDirectories(destDir)
+
+	const page = await fetch(PAGE_URL, { headers: { "user-agent": USER_AGENT, accept: "text/html" } })
+
+	if (!page.ok) throw new Error(`houjin-jp: the download page answered HTTP ${page.status}`)
+	const html = await page.text()
+	const token = new RegExp(`${TOKEN_FIELD.replaceAll(".", "\\.")}" value="([^"]+)"`).exec(html)?.[1]
+	const fileNumber = nationwideUnicodeFileNumber(html)
+
+	if (!token || !fileNumber) {
+		report?.(
+			`  ✗ the page carried ${token ? "a token" : "no token"} and ${fileNumber ? `file ${fileNumber}` : "no nationwide Unicode CSV row"}`
+		)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: ["zenkoku-csv-unicode"] }
+	}
+
+	report?.(`=== ${SLUG}: nationwide CSV (Unicode), file ${fileNumber}`)
+
+	let filename = `zenkoku_all_${fileNumber}.zip`
+	let bytes: number
+
+	try {
+		bytes = await withRetries(
+			async () => {
+				// A fresh page per attempt: the token is bound to the session cookie, and both may have lapsed.
+				const attemptPage = await fetch(PAGE_URL, { headers: { "user-agent": USER_AGENT, accept: "text/html" } })
+
+				const attemptToken = new RegExp(`${TOKEN_FIELD.replaceAll(".", "\\.")}" value="([^"]+)"`).exec(
+					await attemptPage.text()
+				)?.[1]
+
+				if (!attemptToken) throw new Error("the download page carried no token")
+
+				const res = await fetch(FORM_URL, {
+					method: "POST",
+					headers: {
+						"user-agent": USER_AGENT,
+						cookie: cookieHeader(attemptPage),
+						referer: PAGE_URL,
+						"content-type": "application/x-www-form-urlencoded",
+					},
+					body: new URLSearchParams({ [TOKEN_FIELD]: attemptToken, event: "download", selDlFileNo: fileNumber }),
+					signal: AbortSignal.timeout(1_800_000),
+				})
+
+				if (!res.ok || !(res.headers.get("content-type") ?? "").includes("octet-stream")) {
+					throw new Error(`HTTP ${res.status} ${res.headers.get("content-type") ?? ""}`)
+				}
+
+				filename = attachmentFilename(res, filename)
+
+				return streamBodyToFile(res, join(destDir, filename))
+			},
+			{ report, label: filename }
+		)
+	} catch (error) {
+		report?.(`  ✗ ${(error as Error).message}`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: ["zenkoku-csv-unicode"] }
+	}
+
+	const dest = join(destDir, filename)
+	const sha = await sha256File(dest)
+	report?.(`  ✓ ${filename} ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
+
+	const manifest: SourceCollectionManifest = {
+		source: SLUG,
+		source_url: PAGE_URL,
+		license: LICENSE,
+		attribution: ATTRIBUTION,
+		downloaded_at: new Date().toISOString(),
+		files: [{ source_url: FORM_URL, downloaded_at: new Date().toISOString(), filename, sha256: sha, bytes }],
+	}
+
+	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}

package/lib/tools/fetch/index.ts

@@ -57,6 +57,18 @@
  *   - `state-hi-schools` — Hawaii DOE school directory (original XLSX; HIDOE + PCS sheets). Tier A
  *       (state PD-equivalent).
  *   - `tiger-full` — US Census TIGER 2024 ADDRFEAT, all US counties. Tier A (US PD).
+ *   - `juso-kr` — the Korean road-name address register (주소DB, plus the English road-name DB) from
+ *       business.juso.go.kr, the LABEL source for Korean. Tier B ("이용허락범위 제한 없음", attribution).
+ *   - `localdata-kr` — Korea's local-government permit registry, one CSV per category, each row carrying
+ *       the lot-number AND road-name address of one premises plus a coordinate. The NOISY source for
+ *       Korean. Tier B (same label as `juso-kr`).
+ *   - `gcis-tw` — Taiwan's company and business registers from the economic ministry's open-data
+ *       platform, 148 files by region and industry. The NOISY source for Taiwanese. Tier B (Open
+ *       Government Data License v1.0 — the prescribed attribution is a condition of the grant).
+ *   - `acra-sg` — Singapore's ACRA corporate-entity register, 27 CSVs with the address FIELDED. The
+ *       NOISY source for Singapore. Tier B (Singapore Open Data Licence 1.0).
+ *   - `houjin-jp` — the National Tax Agency's corporate-number register, the nationwide Unicode CSV.
+ *       The NOISY source for Japanese. Tier A-equivalent (free use, no attribution condition stated).
  *
  *   License tiers per `docs/licensing-strategy.md` (or the playpen knowledge base mirror at
  *   `docs/docs/projects/mailwoman/licensing-strategy.md`). `openaddresses` is a **Tier-mixed**
@@ -97,11 +109,16 @@
  *   5. Add the source's adapter (or extend an existing one) under `corpus/src/adapters/`.
  */
 
+import { fetchACRASG } from "#tools/fetch/acra-sg"
 import { fetchBan } from "#tools/fetch/ban"
+import { fetchGCISTW } from "#tools/fetch/gcis-tw"
 import { fetchGeonamesDumps } from "#tools/fetch/geonames-dump"
 import { fetchGeonamesPostal } from "#tools/fetch/geonames-postal"
+import { fetchHoujinJP } from "#tools/fetch/houjin-jp"
 import { fetchHRSA } from "#tools/fetch/hrsa"
 import { fetchIMLSPLS } from "#tools/fetch/imls-pls"
+import { fetchJusoKR } from "#tools/fetch/juso-kr"
+import { fetchLocaldataKR } from "#tools/fetch/localdata-kr"
 import { fetchNAD } from "#tools/fetch/nad"
 import { fetchNPPES } from "#tools/fetch/nppes"
 import { fetchOpenAddresses } from "#tools/fetch/openaddresses"
@@ -111,11 +128,16 @@ import { fetchStateSources } from "#tools/fetch/state-sources"
 import { fetchTigerFull } from "#tools/fetch/tiger-full"
 import { fetchWikidataSubVenue } from "#tools/fetch/wikidata-subvenue"
 
+export * from "#tools/fetch/acra-sg"
 export * from "#tools/fetch/ban"
+export * from "#tools/fetch/gcis-tw"
 export * from "#tools/fetch/geonames-dump"
 export * from "#tools/fetch/geonames-postal"
+export * from "#tools/fetch/houjin-jp"
 export * from "#tools/fetch/hrsa"
 export * from "#tools/fetch/imls-pls"
+export * from "#tools/fetch/juso-kr"
+export * from "#tools/fetch/localdata-kr"
 export * from "#tools/fetch/nad"
 export * from "#tools/fetch/nppes"
 export * from "#tools/fetch/openaddresses"
@@ -129,7 +151,12 @@ export * from "#tools/fetch/wikidata-subvenue"
  * The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
  */
 export const FETCH_SOURCES = {
+	"acra-sg": fetchACRASG,
 	ban: fetchBan,
+	"gcis-tw": fetchGCISTW,
+	"houjin-jp": fetchHoujinJP,
+	"juso-kr": fetchJusoKR,
+	"localdata-kr": fetchLocaldataKR,
 	nad: fetchNAD,
 	"geonames-dump": fetchGeonamesDumps,
 	"geonames-postal": fetchGeonamesPostal,