@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/tools/fetch/imls-pls.ts

@@ -1,149 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the IMLS Public Libraries Survey (PLS) outlet-level data. Each US public library branch
- *   (outlet) is one row, ~17K rows with address fields. Source for the `usgov-imls-pls` adapter. US
- *   Public Domain (federal statistical survey).
- *
- *   The FY 2023 release is the most current as of 2026-05. IMLS ships a single ZIP containing CSV,
- *   SAS, and SPSS variants. We extract the outlet-level CSV (pls_fy*_outlet*.csv or similar) and
- *   discard the rest. The administrative-entity (system-level) CSV is intentionally skipped — it has
- *   no per-branch address detail.
- *
- *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum. The
- *   ZIP is unpacked with the `unzip` binary via `node:child_process` (no clean Node equivalent for
- *   member listing + selective extraction).
- *
- *   Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
- *   and sha matches MANIFEST, skips download.
- */
-
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { statPath, pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
-import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
-import { sha256File } from "@mailwoman/core/hash"
-import { basename, join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
-import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-
-/**
- * The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
- */
-const ZIP_URL = "https://www.imls.gov/sites/default/files/2025-08/pls_fy2023_csv.zip"
-const SLUG = "usgov-imls-pls"
-
-export type FetchIMLSPLSOptions = BaseFetchOptions
-
-export async function fetchIMLSPLS(
-	options: FetchIMLSPLSOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-
-	const zipDest = join(destDir, basename(ZIP_URL))
-	const manifestPath = join(destDir, "MANIFEST.json")
-
-	report?.(`=== ${SLUG}`)
-
-	// MARK: Idempotency check
-
-	const recorded = await readManifest<Partial<SourceManifest>>(manifestPath)
-
-	if (recorded?.sha256 && recorded.filename) {
-		const recordedPath = join(destDir, recorded.filename)
-
-		if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
-			report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.")
-
-			return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
-		}
-	}
-
-	// MARK: Download ZIP
-
-	report?.(`  Downloading ${ZIP_URL} ...`)
-
-	const { bytes: zipSize } = await downloadToFile({
-		url: ZIP_URL,
-		dest: zipDest,
-		timeoutMs: 600_000,
-		headers: { "Accept-Encoding": "gzip, br" },
-		report,
-	})
-
-	report?.(`  Downloaded: ${ByteFormatter.formatIEC(zipSize)}`)
-
-	if (zipSize < BYTES_PER_KIB) {
-		report?.(`  ✗ Response too small (${zipSize} bytes) — probable error page`)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	// Discover the outlet-level CSV inside the ZIP.
-	// Outlet files match: pls_fy*outlet*.csv (case-insensitive)
-	// Administrative-entity files match: pls_fy*ae*.csv — we skip those.
-	report?.("  Inspecting ZIP contents ...")
-	const entries = (await listZipEntries(zipDest)).map((entry) => entry.name)
-
-	let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name))
-
-	// Fallback: if IMLS renames the file, grab any CSV that is NOT the ae file.
-	if (!csvName) {
-		csvName = entries.find((name) => /\.csv$/i.test(name) && !/system|state|_ae\b|_se\b/i.test(name))
-	}
-
-	if (!csvName) {
-		report?.("  Available files in ZIP:")
-
-		for (const name of entries) {
-			report?.(`    ${name}`)
-		}
-
-		report?.("  ✗ Could not identify outlet CSV — inspect above listing and update this module")
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	report?.(`  Extracting outlet CSV: ${csvName}`)
-
-	const csvDest = join(destDir, basename(csvName))
-
-	await extractZipEntry(zipDest, csvName, csvDest)
-	const csvSize = (await statPath(csvDest)).size
-	const csvSha = await sha256File(csvDest)
-
-	// MARK: Remove ZIP (small, but keep destDir clean)
-
-	await removePathIfPresent(zipDest)
-	report?.("  Removed ZIP (CSV kept)")
-
-	// MARK: Write MANIFEST
-
-	const manifest: SourceManifest = {
-		source_url: ZIP_URL,
-		downloaded_at: new Date().toISOString(),
-		filename: basename(csvName),
-		sha256: csvSha,
-		bytes: csvSize,
-	}
-
-	await writeManifest(manifestPath, manifest)
-
-	report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`)
-	report?.(`  MANIFEST written to ${manifestPath}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}

package/lib/tools/fetch/index.ts

@@ -1,174 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   The `mailwoman corpus fetch <source>` family — reproducible bulk-download recovery modules for
- *   the open-data sources the corpus build pipeline consumes. Each module writes the raw download
- *   files **plus** a sibling `MANIFEST.json` capturing the origin URL, fetch timestamp, byte count,
- *   and sha256 so downstream adapters can verify provenance.
- *
- *   The corpus build pipeline itself does NOT call these modules — the existing convention is for
- *   operators to pre-download into an out-root and point adapters at the resulting files. These
- *   modules exist for **reproducibility** (disk-loss recovery, weekly refresh, fresh-environment
- *   bootstrap).
- *
- *   ## Usage
- *
- *   ```sh
- *   # Default: writes under ./data/corpus/sources/ relative to the working directory
- *   mailwoman corpus fetch state-sources
- *   mailwoman corpus fetch hrsa
- *
- *   # Or point at the standard mailwoman data root
- *   mailwoman corpus fetch state-sources --out-root /data/corpus/sources
- *   ```
- *
- *   Each adapter under `corpus/src/adapters/<adapter>/README.md` documents the specific URL its
- *   input was pulled from; these modules mirror those URLs in a single executable place.
- *
- *   ## Coverage
- *
- *   - `ban` — French BAN (Base Adresse Nationale), all départements incl. DOM/TOM. Tier B (Licence
- *       Ouverte 2.0).
- *   - `nad` — US DOT National Address Database (~97M address points, ArcGIS FeatureServer). Tier A
- *       (US PD).
- *   - `geonames-postal` — GeoNames per-country postal exports (~80 countries). Tier B (CC-BY-4.0,
- *       attribute "GeoNames"). The only source in this family carrying `(postcode, locality, region)`
- *       with the names INLINE, which is why it exists: the `parent_id` join route covers exactly five
- *       countries and the nearest-centroid fallback measured under 50% agreement on four of those five.
- *       GeoNames does not publish every country — Venezuela 404s.
- *   - `hrsa` — HRSA Health Center Service Delivery Sites (federal). Tier A (US PD).
- *   - `imls-pls` — IMLS Public Libraries Survey, outlet-level (~17K library branches, FY 2023).
- *       Tier A (US PD).
- *   - `nppes` — NPPES NPI registry, full monthly dissemination (~7M provider venue+address rows).
- *       Tier A (US PD).
- *   - `openaddresses` — OpenAddresses country collections (default: Canada / `ca`). Tier B/C mixed
- *       — per-row filter.
- *   - `ourairports` — OurAirports global airport CSVs (~83K airports + the country/region/runway
- *       joins). Tier A (public domain). The VENUE half of the sub-venue arc (#35); it carries no
- *       interior structure, which comes from `@mailwoman/osm/sdk`'s `extractOSMSubVenues`.
- *   - `wikidata-subvenue` — the multilingual sub-venue DESIGNATOR vocabulary, pulled as the labels
- *       and aliases of eight Wikidata concepts (terminal, gate, concourse, campus, …) plus the
- *       airport-terminal instance labels as attested usage. Tier A (CC0). The only module in this
- *       family built on `APIClient` — see its docstring for why.
- *   - `state-sources` — NY/TX/DE/OR notaries, IA contractors, WA health providers, HI lobbyists.
- *       Tier A (state PD-equivalent).
- *   - `state-hi-schools` — Hawaii DOE school directory (original XLSX; HIDOE + PCS sheets). Tier A
- *       (state PD-equivalent).
- *   - `tiger-full` — US Census TIGER 2024 ADDRFEAT, all US counties. Tier A (US PD).
- *   - `juso-kr` — the Korean road-name address register (주소DB, plus the English road-name DB) from
- *       business.juso.go.kr, the LABEL source for Korean. Tier B ("이용허락범위 제한 없음", attribution).
- *   - `localdata-kr` — Korea's local-government permit registry, one CSV per category, each row carrying
- *       the lot-number AND road-name address of one premises plus a coordinate. The NOISY source for
- *       Korean. Tier B (same label as `juso-kr`).
- *   - `gcis-tw` — Taiwan's company and business registers from the economic ministry's open-data
- *       platform, 148 files by region and industry. The NOISY source for Taiwanese. Tier B (Open
- *       Government Data License v1.0 — the prescribed attribution is a condition of the grant).
- *   - `acra-sg` — Singapore's ACRA corporate-entity register, 27 CSVs with the address FIELDED. The
- *       NOISY source for Singapore. Tier B (Singapore Open Data Licence 1.0).
- *   - `houjin-jp` — the National Tax Agency's corporate-number register, the nationwide Unicode CSV.
- *       The NOISY source for Japanese. Tier A-equivalent (free use, no attribution condition stated).
- *
- *   License tiers per `docs/licensing-strategy.md` (or the playpen knowledge base mirror at
- *   `docs/docs/projects/mailwoman/licensing-strategy.md`). `openaddresses` is a **Tier-mixed**
- *   source: the downloaded collection includes CC0, CC-BY, OGL, and ODbL/CC-BY-SA rows. The per-row
- *   `LICENSE` filter in the `openaddresses` adapter is essential — Tier-C (ODbL, CC-BY-SA) rows are
- *   dropped at ingest by default to protect proprietary-weights training.
- *
- *   ### OpenAddresses authentication (as of 2026-05-18)
- *
- *   `batch.openaddresses.io` now requires a free registered account for bulk downloads (auth check
- *   prevents CDN abuse; data remains openly licensed). `fetchOpenAddresses` reads `OA_BATCH_TOKEN`
- *   from the environment:
- *
- *   ```sh
- *   # One-time: register at https://batch.openaddresses.io/register
- *   # Log in → Profile → "Create Token" → copy token
- *   export OA_BATCH_TOKEN=<your-token>
- *
- *   # Download Canada (~2 GiB compressed, ~7 GiB uncompressed)
- *   mailwoman corpus fetch openaddresses --country ca \
- *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Or any other OA country code
- *   mailwoman corpus fetch openaddresses --country fr
- *   ```
- *
- *   Without a token the command prints setup instructions and reports the failure.
- *
- *   ## Adding a new source
- *
- *   1. Pick the right module (or create a sibling one if the source is from a meaningfully
- *      different family).
- *   2. Append to the `SOURCES` array: `{ slug, filename, url }`.
- *   3. Confirm the destination URL via `curl -sI -L <url> | head` before committing — state
- *      open-data portals occasionally rotate Socrata view IDs.
- *   4. Run the command against a scratch `--out-root` to verify the download succeeds + the
- *      MANIFEST is well-formed.
- *   5. Add the source's adapter (or extend an existing one) under `corpus/src/adapters/`.
- */
-
-import { fetchACRASG } from "#tools/fetch/acra-sg"
-import { fetchBan } from "#tools/fetch/ban"
-import { fetchGCISTW } from "#tools/fetch/gcis-tw"
-import { fetchGeonamesDumps } from "#tools/fetch/geonames-dump"
-import { fetchGeonamesPostal } from "#tools/fetch/geonames-postal"
-import { fetchHoujinJP } from "#tools/fetch/houjin-jp"
-import { fetchHRSA } from "#tools/fetch/hrsa"
-import { fetchIMLSPLS } from "#tools/fetch/imls-pls"
-import { fetchJusoKR } from "#tools/fetch/juso-kr"
-import { fetchLocaldataKR } from "#tools/fetch/localdata-kr"
-import { fetchNAD } from "#tools/fetch/nad"
-import { fetchNPPES } from "#tools/fetch/nppes"
-import { fetchOpenAddresses } from "#tools/fetch/openaddresses"
-import { fetchOurAirports } from "#tools/fetch/ourairports"
-import { fetchStateHISchools } from "#tools/fetch/state-hi-schools"
-import { fetchStateSources } from "#tools/fetch/state-sources"
-import { fetchTigerFull } from "#tools/fetch/tiger-full"
-import { fetchWikidataSubVenue } from "#tools/fetch/wikidata-subvenue"
-
-export * from "#tools/fetch/acra-sg"
-export * from "#tools/fetch/ban"
-export * from "#tools/fetch/gcis-tw"
-export * from "#tools/fetch/geonames-dump"
-export * from "#tools/fetch/geonames-postal"
-export * from "#tools/fetch/houjin-jp"
-export * from "#tools/fetch/hrsa"
-export * from "#tools/fetch/imls-pls"
-export * from "#tools/fetch/juso-kr"
-export * from "#tools/fetch/localdata-kr"
-export * from "#tools/fetch/nad"
-export * from "#tools/fetch/nppes"
-export * from "#tools/fetch/openaddresses"
-export * from "#tools/fetch/ourairports"
-export * from "#tools/fetch/state-hi-schools"
-export * from "#tools/fetch/state-sources"
-export * from "#tools/fetch/tiger-full"
-export * from "#tools/fetch/wikidata-subvenue"
-
-/**
- * The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
- */
-export const FETCH_SOURCES = {
-	"acra-sg": fetchACRASG,
-	ban: fetchBan,
-	"gcis-tw": fetchGCISTW,
-	"houjin-jp": fetchHoujinJP,
-	"juso-kr": fetchJusoKR,
-	"localdata-kr": fetchLocaldataKR,
-	nad: fetchNAD,
-	"geonames-dump": fetchGeonamesDumps,
-	"geonames-postal": fetchGeonamesPostal,
-	hrsa: fetchHRSA,
-	"imls-pls": fetchIMLSPLS,
-	nppes: fetchNPPES,
-	openaddresses: fetchOpenAddresses,
-	ourairports: fetchOurAirports,
-	"state-sources": fetchStateSources,
-	"state-hi-schools": fetchStateHISchools,
-	"tiger-full": fetchTigerFull,
-	"wikidata-subvenue": fetchWikidataSubVenue,
-} as const
-
-export type FetchSourceID = keyof typeof FETCH_SOURCES

package/lib/tools/fetch/juso-kr.ts

@@ -1,165 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the Korean road-name address register (도로명주소 주소DB) from the ministry's address-industry
- *   portal, business.juso.go.kr. This is the LABEL source for Korean addresses: about 6,000,000
- *   road-name addresses and 8,000,000 lot-number (지번) records, split by the publisher into four
- *   pipe-delimited CP949 text files per region (도로명코드 / 주소 / 지번 / 부가정보), with the postcode in
- *   the supplementary file. No coordinates. The public-data portal labels it "이용허락범위 제한 없음"
- *   (no restriction on the scope of use), and it is a direct download with no application step — the
- *   distinction that matters against the ministry's entrance-coordinate products, which are provided
- *   only after a purpose-of-use review (`docs/superpowers/plans/counsel-dossier.md` §4).
- *
- *   The portal is a single-page app; the file behind it is reached in two calls. `selectAttrbDBDwldList`
- *   lists the monthly full files for one product (`rtlDtaDtlSn` 8 is the 주소DB, 2 the English
- *   road-name DB), and `/api/jst/download` streams one by the parameters the listing carried. The
- *   English DB rides along because it is the Latin-script half of the same register.
- *
- *   Invoke via `mailwoman corpus fetch juso-kr --out-root <path>`.
- */
-
-import { makeDirectories } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"
-import { resumableDownload, writeManifest } from "#tools/fetch/download"
-
-const SLUG = "juso-kr"
-const PORTAL = "https://business.juso.go.kr"
-const LIST_URL = `${PORTAL}/api/jst/selectAttrbDBDwldList`
-const DOWNLOAD_URL = `${PORTAL}/api/jst/download`
-const REFERER = `${PORTAL}/jst/jstAddressDownload?menu=30`
-
-/**
- * The portal's product ids for the two registers this module fetches.
- */
-const PRODUCTS = [
-	{ sn: "8", name: "주소DB" },
-	{ sn: "2", name: "도로명주소 영문" },
-] as const
-
-const LICENSE =
-	"이용허락범위 제한 없음 (data.go.kr label for 행정안전부_도로명주소 주소DB); 공공누리 attribution to 행정안전부"
-const ATTRIBUTION = "행정안전부 (Ministry of the Interior and Safety), 도로명주소 (juso.go.kr)"
-
-interface ListedFile {
-	crtrYm: string
-	fileTypeNm: string
-	fileNm: string
-	isExist: "Y" | "N"
-	ctpvClsfCd: string
-	tmprFileNm: string
-	fileSn: string | null
-	atflNo: string | null
-}
-
-interface ListResponse {
-	results?: { allMonthFileList?: ListedFile[] }
-}
-
-export interface FetchJusoKROptions extends BaseFetchOptions {
-	/**
-	 * The month to fetch as `YYYYMM`. Defaults to the latest month the portal lists as present.
-	 */
-	month?: string
-}
-
-const BROWSER_HEADERS = {
-	"user-agent": "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch",
-	referer: REFERER,
-}
-
-/**
- * The monthly full files the portal lists for one product over the twelve months ending at `year`/`month`.
- */
-async function listMonthlyFiles(sn: string, year: number, month: number): Promise<ListedFile[]> {
-	const res = await fetch(LIST_URL, {
-		method: "POST",
-		headers: { ...BROWSER_HEADERS, accept: "application/json", "content-type": "application/json;charset=UTF-8" },
-		body: JSON.stringify({ rtlDtaDtlSn: sn, year, month, expand: "Y" }),
-	})
-
-	if (!res.ok) throw new Error(`juso-kr: listing product ${sn} answered HTTP ${res.status}`)
-	const json = (await res.json()) as ListResponse
-
-	return (json.results?.allMonthFileList ?? []).filter((file) => file.isExist === "Y")
-}
-
-function downloadURL(file: ListedFile, regYmd: string): string {
-	const query = new URLSearchParams({
-		reqType: file.fileTypeNm,
-		ctprvnCd: file.ctpvClsfCd,
-		stdde: file.crtrYm,
-		fileName: file.fileNm,
-		realFileName: file.tmprFileNm,
-		intFileNo: file.fileSn ?? "0",
-		intNum: file.atflNo ?? "0",
-		regYmd,
-	})
-
-	return `${DOWNLOAD_URL}?${query.toString()}`
-}
-
-export async function fetchJusoKR(options: FetchJusoKROptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-
-	const now = new Date()
-	const files: SourceManifest[] = []
-	const failedCodes: string[] = []
-
-	for (const product of PRODUCTS) {
-		const listed = await listMonthlyFiles(product.sn, now.getFullYear(), now.getMonth() + 1)
-		const file = options.month ? listed.find((entry) => entry.crtrYm === options.month) : listed.at(-1)
-
-		if (!file) {
-			report?.(`  ✗ ${product.name}: no full file listed${options.month ? ` for ${options.month}` : ""}`)
-			failedCodes.push(product.name)
-
-			continue
-		}
-
-		const url = downloadURL(file, String(now.getFullYear()))
-		const dest = join(destDir, file.tmprFileNm)
-		report?.(`=== ${SLUG} / ${product.name} ${file.crtrYm} → ${file.tmprFileNm}`)
-		let bytes: number
-
-		try {
-			// The portal drops a connection every few megabytes and answers Range with 206, so the transfer resumes
-			// from what landed rather than starting the 181 MB over.
-			bytes = await resumableDownload({ url, dest, headers: BROWSER_HEADERS, report })
-		} catch (error) {
-			report?.(`  ✗ ${(error as Error).message}`)
-			failedCodes.push(product.name)
-
-			continue
-		}
-
-		const sha = await sha256File(dest)
-
-		files.push({
-			source_url: url,
-			downloaded_at: new Date().toISOString(),
-			filename: file.tmprFileNm,
-			sha256: sha,
-			bytes,
-		})
-		report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
-	}
-
-	const manifest: SourceCollectionManifest = {
-		source: SLUG,
-		source_url: REFERER,
-		license: LICENSE,
-		attribution: ATTRIBUTION,
-		downloaded_at: new Date().toISOString(),
-		files,
-	}
-
-	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
-
-	return { fetched: files.length, skipped: 0, failed: failedCodes.length, failedCodes }
-}

package/lib/tools/fetch/localdata-kr.ts

@@ -1,194 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch Korea's local-government permit registry (지방행정인허가데이터): one CSV per permit category,
- *   every business a local authority has licensed, about 195 categories. Each row carries BOTH address
- *   forms of the same premises — `소재지전체주소` (the lot-number form, 지번) and `도로명전체주소` (the
- *   road-name form) — plus both postcodes and a planar coordinate (`좌표정보(x/y)`, EPSG:5174). That
- *   pairing is the cheapest dual-format signal Korean addresses offer, and the coordinate is the
- *   board's coordinate half, which is why this is the NOISY source for Korean and not a side dish.
- *
- *   The public-data portal labels the category files "이용허락범위 제한 없음" and links them to
- *   `file.localdata.go.kr`, which serves them behind a session: the category page sets the XSRF cookie,
- *   `/file/validate/download-count` is the portal's own rate check (429 when it wants a pause), and
- *   `/file/download/<slug>/info` streams the CSV. The files are CP949 as delivered; the adapter
- *   decodes. The restaurant category alone is about 700 MB.
- *
- *   Invoke via `mailwoman corpus fetch localdata-kr --out-root <path>`, optionally with
- *   `--categories general_restaurants,rest_cafes`.
- */
-
-import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
-import { pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { sleep } from "@mailwoman/core/utils/sleep"
-import { join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"
-import { cookieHeader, loadCollectionFiles, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download"
-
-const SLUG = "localdata-kr"
-const PORTAL = "https://file.localdata.go.kr"
-const INDEX_URL = `${PORTAL}/file/general_restaurants/info`
-const LICENSE = "이용허락범위 제한 없음 (data.go.kr label on each category file, e.g. 행정안전부_식품_일반음식점)"
-const ATTRIBUTION = "행정안전부 (Ministry of the Interior and Safety), 지방행정인허가데이터"
-const HTTP_TOO_MANY_REQUESTS = 429
-const RATE_PAUSE_MS = 30_000
-
-export interface FetchLocaldataKROptions extends BaseFetchOptions {
-	/**
-	 * Category slugs to fetch (the path segment of `/file/<slug>/info`). Defaults to every category the portal lists.
-	 */
-	categories?: string[]
-}
-
-interface Category {
-	slug: string
-	name: string
-}
-
-interface Session {
-	cookie: string
-	xsrf: string | undefined
-	categories: Category[]
-}
-
-const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch"
-
-/**
- * Open a session on the portal: the category page answers the XSRF cookie and the navigation that names every category.
- */
-async function openSession(): Promise<Session> {
-	const res = await fetch(INDEX_URL, {
-		headers: { "user-agent": USER_AGENT, accept: "text/html", referer: "https://www.data.go.kr/" },
-	})
-
-	if (!res.ok) throw new Error(`localdata-kr: the category page answered HTTP ${res.status}`)
-	const html = await res.text()
-	const cookie = cookieHeader(res)
-	const xsrf = /XSRF-TOKEN=([^;]+)/.exec(cookie)?.[1]
-	const categories: Category[] = []
-	const seen = new Set<string>()
-
-	for (const [, slug, name] of html.matchAll(/href="\/file\/([a-z0-9_]+)\/info"[^>]*>\s*<span>([^<]+)<\/span>/g)) {
-		if (!slug || !name || seen.has(slug)) continue
-		seen.add(slug)
-		categories.push({ slug, name: name.trim() })
-	}
-
-	return { cookie, xsrf, categories }
-}
-
-/**
- * The portal's own rate check; a 429 carries the pause it asks for in prose, so the caller sleeps and retries.
- */
-async function validateDownloadCount(session: Session): Promise<boolean> {
-	const headers: Record<string, string> = { "user-agent": USER_AGENT, cookie: session.cookie, referer: INDEX_URL }
-
-	if (session.xsrf) {
-		headers["X-XSRF-TOKEN"] = session.xsrf
-	}
-	const res = await fetch(`${PORTAL}/file/validate/download-count`, { headers })
-
-	return res.status !== HTTP_TOO_MANY_REQUESTS
-}
-
-export async function fetchLocaldataKR(
-	options: FetchLocaldataKROptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-	const manifestPath = join(destDir, "MANIFEST.json")
-
-	const session = await openSession()
-
-	const wanted = options.categories?.length
-		? session.categories.filter((category) => options.categories!.includes(category.slug))
-		: session.categories
-
-	report?.(`=== ${SLUG}: ${wanted.length} of ${session.categories.length} categories`)
-
-	// A previous run's entries survive, keyed by file name, so a re-run after an interruption fetches only what is missing.
-	const previous = await loadCollectionFiles(manifestPath)
-	const files = new Map<string, SourceManifest>()
-	let fetched = 0
-	let skipped = 0
-	const failedCodes: string[] = []
-
-	for (const category of wanted) {
-		const filename = `${category.slug}.csv`
-		const dest = join(destDir, filename)
-		const url = `${PORTAL}/file/download/${category.slug}/info`
-		const before = previous.get(filename)
-
-		if (before && (await pathExists(dest))) {
-			files.set(filename, before)
-
-			skipped++
-
-			continue
-		}
-
-		report?.(`--- ${category.name} (${category.slug})`)
-
-		while (!(await validateDownloadCount(session))) {
-			report?.(`  portal asks for a pause — sleeping ${RATE_PAUSE_MS / 1000}s`)
-			await sleep(RATE_PAUSE_MS)
-		}
-
-		let bytes: number
-
-		try {
-			bytes = await withRetries(
-				async () => {
-					const res = await fetch(url, {
-						headers: { "user-agent": USER_AGENT, cookie: session.cookie, referer: INDEX_URL, accept: "*/*" },
-						signal: AbortSignal.timeout(1_800_000),
-					})
-
-					if (!res.ok || !(res.headers.get("content-type") ?? "").includes("text/csv")) {
-						throw new Error(`HTTP ${res.status} ${res.headers.get("content-type") ?? ""} for ${category.slug}`)
-					}
-
-					return streamBodyToFile(res, dest)
-				},
-				{ report, label: category.slug }
-			)
-		} catch (error) {
-			report?.(`  ✗ ${(error as Error).message}`)
-			failedCodes.push(category.slug)
-
-			continue
-		}
-
-		if (bytes < BYTES_PER_KIB) {
-			report?.(`  ✗ ${bytes} bytes — an error page, not the register`)
-			failedCodes.push(category.slug)
-
-			continue
-		}
-
-		const sha = await sha256File(dest)
-		files.set(filename, { source_url: url, downloaded_at: new Date().toISOString(), filename, sha256: sha, bytes })
-
-		fetched++
-		report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
-
-		const manifest: SourceCollectionManifest = {
-			source: SLUG,
-			source_url: INDEX_URL,
-			license: LICENSE,
-			attribution: ATTRIBUTION,
-			downloaded_at: new Date().toISOString(),
-			files: [...files.values()],
-		}
-
-		await writeManifest(manifestPath, manifest)
-	}
-
-	return { fetched, skipped, failed: failedCodes.length, failedCodes }
-}