@mailwoman/corpus 9.3.0 → 9.4.0

package/out/tools/fetch/download.js

@@ -5,15 +5,24 @@
  *
  *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
  */
-import { pathExists, readLocalTextFile } from "@mailwoman/core/fs/readers";
+import { pathExists, readLocalTextFile, tryStat } from "@mailwoman/core/fs/readers";
 import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams";
-import { writeLocalFile, writeLocalTextFile } from "@mailwoman/core/fs/writers";
+import { movePath, writeLocalFile, writeLocalTextFile } from "@mailwoman/core/fs/writers";
 import { tryParsingJSON } from "@mailwoman/core/json";
 import { sleep } from "@mailwoman/core/utils/sleep";
 /**
  * Rate limited — retryable, the server is asking us to back off.
  */
 const HTTP_TOO_MANY_REQUESTS = 429;
+/**
+ * The answer to a satisfiable `Range` request; anything else means the host ignored the range and would send the whole
+ * body again.
+ */
+const HTTP_PARTIAL_CONTENT = 206;
+/**
+ * The answer to a range that starts at or past the end of the body: the resume point is already the whole file.
+ */
+const HTTP_RANGE_NOT_SATISFIABLE = 416;
 /**
  * Lowest 5xx status. Server-side failures are retryable; 4xx are not.
  */
@@ -157,4 +166,153 @@ export async function loadManifestEntries(path, key) {
 export async function writeManifest(path, manifest) {
     await writeLocalTextFile(JSON.stringify(manifest, null, 2) + "\n", path);
 }
+/**
+ * Pipe a response body to `dest` and answer the byte count. The one primitive the portal fetchers share when the
+ * request is not a bare GET — a session cookie, a CSRF header, or a form POST stands between the listing and the file,
+ * so {@link streamDownload}'s URL-only contract does not fit and each module builds its own `Response` first. Writes a
+ * `.tmp` sibling and renames, so an interrupted transfer never lands at the final path looking complete.
+ */
+export async function streamBodyToFile(res, dest) {
+    if (!res.body)
+        throw new HTTPStatusError(res.status, `HTTP ${res.status} with no body — ${res.url}`);
+    let bytes = 0;
+    const counter = new TransformStream({
+        transform(chunk, controller) {
+            bytes += chunk.byteLength;
+            controller.enqueue(chunk);
+        },
+    });
+    const tmp = dest + ".tmp";
+    await pipeline(Readable.fromWeb(res.body.pipeThrough(counter)), openWriteStream(tmp));
+    await movePath(tmp, dest);
+    return bytes;
+}
+/**
+ * The per-file entries of a {@link SourceCollectionManifest} keyed by file name, or an empty map when the manifest is
+ * missing or not in the collection shape, so a re-run after an interruption fetches only what is missing. The
+ * single-file modules' `loadManifestEntries` reads a bare array and is not this.
+ */
+export async function loadCollectionFiles(path) {
+    const parsed = await readManifest(path);
+    const entries = new Map();
+    for (const entry of Array.isArray(parsed?.files) ? parsed.files : []) {
+        entries.set(entry.filename, entry);
+    }
+    return entries;
+}
+/**
+ * Run one transfer up to `1 + retries` times, pausing `retryDelayMs` between attempts. A government portal drops a
+ * multi-hundred-megabyte connection often enough that a collection fetch which lets the first `TypeError: terminated`
+ * propagate loses the whole run to one file; the per-file loop calls the transfer through this and records the failure
+ * only once the attempts are spent. The last error is rethrown so the caller can name the file it lost.
+ */
+export async function withRetries(transfer, options = {}) {
+    const { retries = 3, retryDelayMs = DEFAULT_RETRY_DELAY_MS, report, label = "transfer" } = options;
+    let lastError;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        if (attempt > 0) {
+            report?.(`  retry ${attempt}/${retries} after ${retryDelayMs}ms — ${label}`);
+            await sleep(retryDelayMs);
+        }
+        try {
+            return await transfer();
+        }
+        catch (error) {
+            lastError = error;
+        }
+    }
+    throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}
+/**
+ * The byte count already at `path`, or 0 when nothing is there — the resume point of an interrupted range download.
+ */
+async function bytesOnDisk(path) {
+    const stat = await tryStat(path);
+    return stat?.size ?? 0;
+}
+/**
+ * Download `url` to `dest` in ranges, resuming from whatever the `.tmp` sibling already holds. For a host that drops a
+ * long connection every few megabytes but answers `Range` with 206 (the Korean address portal does both, measured at
+ * 0.9–12 MB per connection against a 181 MB file), a whole-body transfer never finishes and a plain retry starts over;
+ * this one asks for the remainder each time and keeps what landed. The total comes from the first `Content-Range`, and
+ * the loop gives up after `maxConnections` connections so a host that keeps answering 206 with no bytes cannot spin.
+ * Answers the byte count written; throws when the host answers anything but 206 for a range.
+ */
+export async function resumableDownload(options) {
+    const { url, dest, headers = {}, maxConnections = 400, retryDelayMs = DEFAULT_RETRY_DELAY_MS, report } = options;
+    const tmp = dest + ".tmp";
+    let have = await bytesOnDisk(tmp);
+    let total;
+    for (let connection = 0; connection < maxConnections; connection++) {
+        if (total !== undefined && have >= total)
+            break;
+        let res;
+        try {
+            res = await fetch(url, { headers: { ...headers, range: `bytes=${have}-` }, signal: AbortSignal.timeout(600_000) });
+        }
+        catch {
+            await sleep(retryDelayMs);
+            continue;
+        }
+        if (res.status === HTTP_RANGE_NOT_SATISFIABLE && have > 0) {
+            // Nothing past `have`: the file on disk is already the whole body (a parallel filler or an earlier run
+            // landed it), and `Content-Range: bytes */<total>` says how long it is.
+            const whole = /\*\/(\d+)/.exec(res.headers.get("content-range") ?? "")?.[1];
+            total = whole ? Number(whole) : have;
+            await res.body?.cancel();
+            break;
+        }
+        if (res.status !== HTTP_PARTIAL_CONTENT) {
+            throw new HTTPStatusError(res.status, `HTTP ${res.status} for a range request — ${url}`);
+        }
+        const range = /bytes \d+-\d+\/(\d+)/.exec(res.headers.get("content-range") ?? "")?.[1];
+        if (range) {
+            total = Number(range);
+        }
+        const before = have;
+        try {
+            // Append: the stream opens the file for appending so a partial body extends what earlier connections left.
+            await pipeline(Readable.fromWeb(res.body), openWriteStream(tmp, { flags: "a" }));
+        }
+        catch {
+            // The connection dropped mid-body; the bytes that landed are on disk, and the next range starts after them.
+        }
+        have = await bytesOnDisk(tmp);
+        if (have === before) {
+            await sleep(retryDelayMs);
+        }
+        else {
+            report?.(`  ${(have / 1024 / 1024).toFixed(1)} MB${total ? ` of ${(total / 1024 / 1024).toFixed(1)} MB` : ""}`);
+        }
+    }
+    if (total === undefined || have < total) {
+        throw new Error(`resumable download stalled at ${have} of ${total ?? "?"} bytes — ${url}`);
+    }
+    await movePath(tmp, dest);
+    return have;
+}
+/**
+ * The `Set-Cookie` values of a response folded into one `Cookie` header value, so a second request to the same portal
+ * carries the session the first one opened.
+ */
+export function cookieHeader(res) {
+    return res.headers
+        .getSetCookie()
+        .map((cookie) => cookie.split(";")[0])
+        .join("; ");
+}
+/**
+ * The file name a `Content-Disposition: attachment` header names, RFC 5987 form (`filename*=utf-8''…`) first, plain
+ * `filename="…"` second, or `fallback` when the header carries neither.
+ */
+export function attachmentFilename(res, fallback) {
+    const disposition = res.headers.get("content-disposition") ?? "";
+    const extended = /filename\*=(?:utf-8|UTF-8)'[^']*'([^;]+)/.exec(disposition)?.[1];
+    if (extended)
+        return decodeURIComponent(extended.trim());
+    const plain = /filename="?([^";]+)"?/.exec(disposition)?.[1];
+    if (plain)
+        return decodeURIComponent(plain.trim());
+    return fallback;
+}
 //# sourceMappingURL=download.js.map

package/out/tools/fetch/download.js.map

@@ -1 +1 @@
-{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../lib/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC1E,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/E,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AAGnD;;GAEG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAA;AAuC1C;;GAEG;AACH;;;;;;GAMG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAChC,MAAM,CAAQ;IAEvB,YAAY,MAAc,EAAE,OAAe;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACrB,CAAC;CACD;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,sBAAsB,IAAI,CAAC,MAAM,IAAI,qBAAqB,IAAI,MAAM,IAAI,qBAAqB,CAAC,CAAA;AACjH,CAAC;AAqBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAwB;IAC5D,MAAM,EACL,GAAG,EACH,IAAI,EACJ,SAAS,GAAG,OAAO,EACnB,OAAO,GAAG,CAAC,EACX,YAAY,GAAG,sBAAsB,EACrC,OAAO,EACP,MAAM,GACN,GAAG,OAAO,CAAA;IAEX,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,SAAS,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,GAAG,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,qGAAqG;YACrG,kGAAkG;YAClG,+FAA+F;YAC/F,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,iEAAiE;YACjE,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,MAAM,GAAG,EAAE,CAAC,CAAA;YAE9F,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/C,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YACnD,MAAM,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YAElC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,8DAA8D;YAC9D,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AASD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,IAA2B;IAC1F,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3B,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC3C,CAAC,CAAA;YAEF,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAA;gBAEjE,OAAO,GAAG,CAAC,MAAM,CAAA;YAClB,CAAC;YAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,GAAG,CAAC,MAAM,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAI,IAAY;IACjD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAE1C,+FAA+F;IAC/F,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAE5D,OAAO,cAAc,CAAI,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAI,IAAY,EAAE,GAAyB;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAM,IAAI,CAAC,CAAA;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,QAAiB;IAClE,MAAM,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,IAAI,CAAC,CAAA;AACzE,CAAC"}
+{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../lib/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACnF,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AACzF,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AAGnD;;GAEG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;;GAGG;AACH,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAEhC;;GAEG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAEtC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAA;AAuC1C;;GAEG;AACH;;;;;;GAMG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAChC,MAAM,CAAQ;IAEvB,YAAY,MAAc,EAAE,OAAe;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACrB,CAAC;CACD;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,sBAAsB,IAAI,CAAC,MAAM,IAAI,qBAAqB,IAAI,MAAM,IAAI,qBAAqB,CAAC,CAAA;AACjH,CAAC;AAqBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAwB;IAC5D,MAAM,EACL,GAAG,EACH,IAAI,EACJ,SAAS,GAAG,OAAO,EACnB,OAAO,GAAG,CAAC,EACX,YAAY,GAAG,sBAAsB,EACrC,OAAO,EACP,MAAM,GACN,GAAG,OAAO,CAAA;IAEX,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,SAAS,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,GAAG,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,qGAAqG;YACrG,kGAAkG;YAClG,+FAA+F;YAC/F,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,iEAAiE;YACjE,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,MAAM,GAAG,EAAE,CAAC,CAAA;YAE9F,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/C,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YACnD,MAAM,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YAElC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,8DAA8D;YAC9D,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AASD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,IAA2B;IAC1F,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3B,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC3C,CAAC,CAAA;YAEF,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAA;gBAEjE,OAAO,GAAG,CAAC,MAAM,CAAA;YAClB,CAAC;YAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,GAAG,CAAC,MAAM,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAI,IAAY;IACjD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAE1C,+FAA+F;IAC/F,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAE5D,OAAO,cAAc,CAAI,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAI,IAAY,EAAE,GAAyB;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAM,IAAI,CAAC,CAAA;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,QAAiB;IAClE,MAAM,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,IAAI,CAAC,CAAA;AACzE,CAAC;AAiBD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAa,EAAE,IAAY;IACjE,IAAI,CAAC,GAAG,CAAC,IAAI;QAAE,MAAM,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,EAAE,CAAC,CAAA;IACpG,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,OAAO,GAAG,IAAI,eAAe,CAAyB;QAC3D,SAAS,CAAC,KAAK,EAAE,UAAU;YAC1B,KAAK,IAAI,KAAK,CAAC,UAAU,CAAA;YACzB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;KACD,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,IAAI,GAAG,MAAM,CAAA;IACzB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAA;IACrF,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAEzB,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAY;IACrD,MAAM,MAAM,GAAG,MAAM,YAAY,CAA2B,IAAI,CAAC,CAAA;IACjE,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B,CAAA;IAEjD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IACnC,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,QAA0B,EAC1B,UAAwG,EAAE;IAE1G,MAAM,EAAE,OAAO,GAAG,CAAC,EAAE,YAAY,GAAG,sBAAsB,EAAE,MAAM,EAAE,KAAK,GAAG,UAAU,EAAE,GAAG,OAAO,CAAA;IAClG,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,WAAW,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,KAAK,EAAE,CAAC,CAAA;YAC5E,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,CAAC;YACJ,OAAO,MAAM,QAAQ,EAAE,CAAA;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhC,OAAO,IAAI,EAAE,IAAI,IAAI,CAAC,CAAA;AACvB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAOvC;IACA,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,cAAc,GAAG,GAAG,EAAE,YAAY,GAAG,sBAAsB,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAChH,MAAM,GAAG,GAAG,IAAI,GAAG,MAAM,CAAA;IACzB,IAAI,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAA;IACjC,IAAI,KAAyB,CAAA;IAE7B,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,cAAc,EAAE,UAAU,EAAE,EAAE,CAAC;QACpE,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,IAAI,KAAK;YAAE,MAAK;QAE/C,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,SAAS,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QACnH,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;YAEzB,SAAQ;QACT,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,0BAA0B,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YAC3D,uGAAuG;YACvG,wEAAwE;YACxE,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC3E,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;YACpC,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAA;YAExB,MAAK;QACN,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,oBAAoB,EAAE,CAAC;YACzC,MAAM,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,0BAA0B,GAAG,EAAE,CAAC,CAAA;QACzF,CAAC;QAED,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAEtF,IAAI,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QACtB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAA;QAEnB,IAAI,CAAC;YACJ,2GAA2G;YAC3G,MAAM,QAAQ,CACb,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAA8C,CAAC,EACpE,eAAe,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CACpC,CAAA;QACF,CAAC;QAAC,MAAM,CAAC;YACR,4GAA4G;QAC7G,CAAC;QAED,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAA;QAE7B,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACrB,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;aAAM,CAAC;YACP,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAChH,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,OAAO,KAAK,IAAI,GAAG,YAAY,GAAG,EAAE,CAAC,CAAA;IAC3F,CAAC;IAED,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAEzB,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAa;IACzC,OAAO,GAAG,CAAC,OAAO;SAChB,YAAY,EAAE;SACd,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SACrC,IAAI,CAAC,IAAI,CAAC,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAa,EAAE,QAAgB;IACjE,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAA;IAChE,MAAM,QAAQ,GAAG,0CAA0C,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAElF,IAAI,QAAQ;QAAE,OAAO,kBAAkB,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;IACxD,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAE5D,IAAI,KAAK;QAAE,OAAO,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IAElD,OAAO,QAAQ,CAAA;AAChB,CAAC"}

package/out/tools/fetch/gcis-tw.d.ts

@@ -0,0 +1,23 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch Taiwan's company and business registers (公司登記資料, 商業登記資料) from the Ministry of
+ *   Economic Affairs' commerce open-data platform, data.gcis.nat.gov.tw. The platform publishes the
+ *   full registers as CSV files split by region (the six special municipalities each on their own,
+ *   the rest of the island in four bands) and by industry letter, 148 files in all, each row carrying
+ *   the registered address (公司地址) and, for companies, the tax office's business address
+ *   (營業地址) as free text. This is the NOISY source for Taiwanese addresses: a person typed these.
+ *
+ *   License: 政府資料開放授權條款－第1版 (Open Government Data License, Taiwan, v1.0). The platform's
+ *   own dialog states the condition that binds: attribution in the form it prescribes, or the grant is
+ *   void from the start ("未盡顯名標示義務者，視為自始未取得開放資料之授權"). The manifest records the
+ *   prescribed wording per file so the corpus build and the model card can carry it.
+ *
+ *   Invoke via `mailwoman corpus fetch gcis-tw --out-root <path>`.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export type FetchGCISTWOptions = BaseFetchOptions;
+export declare function fetchGCISTW(options: FetchGCISTWOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=gcis-tw.d.ts.map

package/out/tools/fetch/gcis-tw.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gcis-tw.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/gcis-tw.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AASH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAA4C,MAAM,uBAAuB,CAAA;AAgBrH,MAAM,MAAM,kBAAkB,GAAG,gBAAgB,CAAA;AA+CjD,wBAAsB,WAAW,CAAC,OAAO,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAoFrH"}

package/out/tools/fetch/gcis-tw.js

@@ -0,0 +1,135 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch Taiwan's company and business registers (公司登記資料, 商業登記資料) from the Ministry of
+ *   Economic Affairs' commerce open-data platform, data.gcis.nat.gov.tw. The platform publishes the
+ *   full registers as CSV files split by region (the six special municipalities each on their own,
+ *   the rest of the island in four bands) and by industry letter, 148 files in all, each row carrying
+ *   the registered address (公司地址) and, for companies, the tax office's business address
+ *   (營業地址) as free text. This is the NOISY source for Taiwanese addresses: a person typed these.
+ *
+ *   License: 政府資料開放授權條款－第1版 (Open Government Data License, Taiwan, v1.0). The platform's
+ *   own dialog states the condition that binds: attribution in the form it prescribes, or the grant is
+ *   void from the start ("未盡顯名標示義務者，視為自始未取得開放資料之授權"). The manifest records the
+ *   prescribed wording per file so the corpus build and the model card can carry it.
+ *
+ *   Invoke via `mailwoman corpus fetch gcis-tw --out-root <path>`.
+ */
+import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters";
+import { pathExists } from "@mailwoman/core/fs/readers";
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { sleep } from "@mailwoman/core/utils/sleep";
+import { join } from "path-ts";
+import { loadCollectionFiles, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download";
+const SLUG = "gcis-tw";
+const PORTAL = "https://data.gcis.nat.gov.tw";
+const CATALOG_URL = `${PORTAL}/od/datacategory`;
+const LICENSE = "政府資料開放授權條款－第1版 (Open Government Data License, Taiwan, v1.0) — http://data.gov.tw/license";
+const ATTRIBUTION = "提供機關／經濟部商業發展署 [<dataset title>] — the 顯名聲明 each file's dialog prescribes";
+const PACE_MS = 300;
+/**
+ * The bulk register files: `<region><register>資料-<industry letter><industry>`. The per-industry API datasets and the
+ * monthly new-registration lists share the catalog and are not these.
+ */
+const BULK_TITLE = /(公司登記資料|商業登記資料)-[A-Z]/;
+/**
+ * A file name a shell and a manifest can carry: the title with its punctuation folded to underscores.
+ */
+function filenameFor(title) {
+    return `${title
+        .replaceAll(/[()（）、，/\\\s]+/g, "_")
+        .replaceAll(/_+/g, "_")
+        .replaceAll(/^_|_$/g, "")}.csv`;
+}
+async function listBulkDatasets() {
+    const res = await fetch(CATALOG_URL, { headers: { accept: "text/html" } });
+    if (!res.ok)
+        throw new Error(`gcis-tw: the catalog answered HTTP ${res.status}`);
+    const html = await res.text();
+    const datasets = [];
+    for (const [, path, rawTitle] of html.matchAll(/<a[^>]*href="(\/od\/detail[^"]*)"[^>]*>([^<]+)<\/a>/g)) {
+        const title = rawTitle?.trim();
+        if (!path || !title || !BULK_TITLE.test(title))
+            continue;
+        datasets.push({ title, detailPath: path.replace(/;jsessionid=[^?]*/, "") });
+    }
+    return datasets;
+}
+/**
+ * The `/od/file?oid=…` link a dataset's detail page hands its download dialog.
+ */
+async function fileURLFor(dataset) {
+    const res = await fetch(`${PORTAL}${dataset.detailPath}`, { headers: { accept: "text/html" } });
+    if (!res.ok)
+        return undefined;
+    const html = await res.text();
+    const path = /showDialog\('(\/od\/file\?oid=[^']+)'\)/.exec(html)?.[1];
+    return path ? `${PORTAL}${path}` : undefined;
+}
+export async function fetchGCISTW(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const manifestPath = join(destDir, "MANIFEST.json");
+    const datasets = await listBulkDatasets();
+    report?.(`=== ${SLUG}: ${datasets.length} bulk register files in the catalog`);
+    const previous = await loadCollectionFiles(manifestPath);
+    const files = new Map();
+    let fetched = 0;
+    let skipped = 0;
+    const failedCodes = [];
+    for (const dataset of datasets) {
+        const filename = filenameFor(dataset.title);
+        const dest = join(destDir, filename);
+        const before = previous.get(filename);
+        if (before && (await pathExists(dest))) {
+            files.set(filename, before);
+            skipped++;
+            continue;
+        }
+        await sleep(PACE_MS);
+        const url = await fileURLFor(dataset);
+        if (!url) {
+            report?.(`  ✗ no file link on the detail page for ${dataset.title}`);
+            failedCodes.push(dataset.title);
+            continue;
+        }
+        report?.(`--- ${dataset.title}`);
+        let bytes;
+        try {
+            bytes = await withRetries(async () => {
+                const res = await fetch(url, { headers: { accept: "*/*" }, signal: AbortSignal.timeout(1_800_000) });
+                if (!res.ok)
+                    throw new Error(`HTTP ${res.status} for ${dataset.title}`);
+                return streamBodyToFile(res, dest);
+            }, { report, label: dataset.title });
+        }
+        catch (error) {
+            report?.(`  ✗ ${error.message}`);
+            failedCodes.push(dataset.title);
+            continue;
+        }
+        if (bytes < BYTES_PER_KIB) {
+            report?.(`  ✗ ${bytes} bytes — an error page, not the register`);
+            failedCodes.push(dataset.title);
+            continue;
+        }
+        const sha = await sha256File(dest);
+        files.set(filename, { source_url: url, downloaded_at: new Date().toISOString(), filename, sha256: sha, bytes });
+        fetched++;
+        report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
+        const manifest = {
+            source: SLUG,
+            source_url: CATALOG_URL,
+            license: LICENSE,
+            attribution: ATTRIBUTION,
+            downloaded_at: new Date().toISOString(),
+            files: [...files.values()],
+        };
+        await writeManifest(manifestPath, manifest);
+    }
+    return { fetched, skipped, failed: failedCodes.length, failedCodes };
+}
+//# sourceMappingURL=gcis-tw.js.map

package/out/tools/fetch/gcis-tw.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gcis-tw.js","sourceRoot":"","sources":["../../../lib/tools/fetch/gcis-tw.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEzG,MAAM,IAAI,GAAG,SAAS,CAAA;AACtB,MAAM,MAAM,GAAG,8BAA8B,CAAA;AAC7C,MAAM,WAAW,GAAG,GAAG,MAAM,kBAAkB,CAAA;AAC/C,MAAM,OAAO,GAAG,0FAA0F,CAAA;AAC1G,MAAM,WAAW,GAAG,0EAA0E,CAAA;AAC9F,MAAM,OAAO,GAAG,GAAG,CAAA;AAEnB;;;GAGG;AACH,MAAM,UAAU,GAAG,uBAAuB,CAAA;AAS1C;;GAEG;AACH,SAAS,WAAW,CAAC,KAAa;IACjC,OAAO,GAAG,KAAK;SACb,UAAU,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAClC,UAAU,CAAC,KAAK,EAAE,GAAG,CAAC;SACtB,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAA;AACjC,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC9B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAA;IAE1E,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;IAChF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;IAC7B,MAAM,QAAQ,GAAc,EAAE,CAAA;IAE9B,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,sDAAsD,CAAC,EAAE,CAAC;QACxG,MAAM,KAAK,GAAG,QAAQ,EAAE,IAAI,EAAE,CAAA;QAE9B,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,SAAQ;QACxD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,EAAE,CAAC,CAAA;IAC5E,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,OAAgB;IACzC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAA;IAE/F,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,OAAO,SAAS,CAAA;IAC7B,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;IAC7B,MAAM,IAAI,GAAG,yCAAyC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAEtE,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAA2B,EAAE,MAA+B;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,QAAQ,GAAG,MAAM,gBAAgB,EAAE,CAAA;IACzC,MAAM,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,MAAM,qCAAqC,CAAC,CAAA;IAE9E,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;IACxD,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAA;IAC/C,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAErC,IAAI,MAAM,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACxC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;YAE3B,OAAO,EAAE,CAAA;YAET,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,CAAC,OAAO,CAAC,CAAA;QACpB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;QAErC,IAAI,CAAC,GAAG,EAAE,CAAC;YACV,MAAM,EAAE,CAAC,2CAA2C,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;YACpE,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAE/B,SAAQ;QACT,CAAC;QAED,MAAM,EAAE,CAAC,OAAO,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;QAChC,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,WAAW,CACxB,KAAK,IAAI,EAAE;gBACV,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;gBAEpG,IAAI,CAAC,GAAG,CAAC,EAAE;oBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,QAAQ,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;gBAEvE,OAAO,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACnC,CAAC,EACD,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAChC,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAQ,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC3C,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAE/B,SAAQ;QACT,CAAC;QAED,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,OAAO,KAAK,0CAA0C,CAAC,CAAA;YAChE,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAE/B,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAClC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAA;QAE/G,OAAO,EAAE,CAAA;QACT,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAErE,MAAM,QAAQ,GAA6B;YAC1C,MAAM,EAAE,IAAI;YACZ,UAAU,EAAE,WAAW;YACvB,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,WAAW;YACxB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;SAC1B,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,CAAA;AACrE,CAAC"}

package/out/tools/fetch/houjin-jp.d.ts

@@ -0,0 +1,24 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the National Tax Agency's corporate-number register (法人番号公表サイト 全件データ): every
+ *   corporation assigned a number, about 5,000,000 rows, with the head-office address as one string
+ *   (国内所在地) plus the prefecture and municipality split out and the postcode. The nationwide CSV
+ *   in Unicode is one zip of about 255 MB. This is the NOISY source for Japanese addresses; the
+ *   Overture rows the JP corpus is built from are the LABEL half.
+ *
+ *   The agency states the three published fields may be used freely by anyone ("どなたでも自由にご利用
+ *   いただくことができます"), with no attribution condition on the download page.
+ *
+ *   The file sits behind a form: the page hands out a per-session token, and a POST with that token,
+ *   `event=download` and the file number of the nationwide Unicode CSV answers the zip. The file
+ *   number is read off the page rather than pinned, because the agency re-issues the files monthly.
+ *
+ *   Invoke via `mailwoman corpus fetch houjin-jp --out-root <path>`.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export type FetchHoujinJPOptions = BaseFetchOptions;
+export declare function fetchHoujinJP(options: FetchHoujinJPOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=houjin-jp.d.ts.map

package/out/tools/fetch/houjin-jp.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"houjin-jp.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/houjin-jp.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAMH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAA4B,MAAM,uBAAuB,CAAA;AAWrG,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,CAAA;AAenD,wBAAsB,aAAa,CAClC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAgFvB"}

package/out/tools/fetch/houjin-jp.js

@@ -0,0 +1,102 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the National Tax Agency's corporate-number register (法人番号公表サイト 全件データ): every
+ *   corporation assigned a number, about 5,000,000 rows, with the head-office address as one string
+ *   (国内所在地) plus the prefecture and municipality split out and the postcode. The nationwide CSV
+ *   in Unicode is one zip of about 255 MB. This is the NOISY source for Japanese addresses; the
+ *   Overture rows the JP corpus is built from are the LABEL half.
+ *
+ *   The agency states the three published fields may be used freely by anyone ("どなたでも自由にご利用
+ *   いただくことができます"), with no attribution condition on the download page.
+ *
+ *   The file sits behind a form: the page hands out a per-session token, and a POST with that token,
+ *   `event=download` and the file number of the nationwide Unicode CSV answers the zip. The file
+ *   number is read off the page rather than pinned, because the agency re-issues the files monthly.
+ *
+ *   Invoke via `mailwoman corpus fetch houjin-jp --out-root <path>`.
+ */
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { attachmentFilename, cookieHeader, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download";
+const SLUG = "houjin-jp";
+const PAGE_URL = "https://www.houjin-bangou.nta.go.jp/download/zenken/";
+const FORM_URL = `${PAGE_URL}index.html`;
+const TOKEN_FIELD = "jp.go.nta.houjin_bangou.framework.web.common.CNSFWTokenProcessor.request.token";
+const LICENSE = "国税庁法人番号公表サイト — the three published fields are free for anyone to use (利用規約)";
+const ATTRIBUTION = "国税庁 法人番号公表サイト (National Tax Agency, Corporate Number Publication Site)";
+const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch";
+/**
+ * The nationwide row of the "CSV形式・Unicode" table carries the file number in its `doDownload(N)` handler.
+ */
+function nationwideUnicodeFileNumber(html) {
+    const start = html.indexOf('id="csv-unicode"');
+    if (start === -1)
+        return undefined;
+    const section = html.slice(start, html.indexOf('id="xml-unicode"', start));
+    const row = /全国[\s\S]{0,600}?doDownload\((\d+)\)/.exec(section);
+    return row?.[1];
+}
+export async function fetchHoujinJP(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const page = await fetch(PAGE_URL, { headers: { "user-agent": USER_AGENT, accept: "text/html" } });
+    if (!page.ok)
+        throw new Error(`houjin-jp: the download page answered HTTP ${page.status}`);
+    const html = await page.text();
+    const token = new RegExp(`${TOKEN_FIELD.replaceAll(".", "\\.")}" value="([^"]+)"`).exec(html)?.[1];
+    const fileNumber = nationwideUnicodeFileNumber(html);
+    if (!token || !fileNumber) {
+        report?.(`  ✗ the page carried ${token ? "a token" : "no token"} and ${fileNumber ? `file ${fileNumber}` : "no nationwide Unicode CSV row"}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: ["zenkoku-csv-unicode"] };
+    }
+    report?.(`=== ${SLUG}: nationwide CSV (Unicode), file ${fileNumber}`);
+    let filename = `zenkoku_all_${fileNumber}.zip`;
+    let bytes;
+    try {
+        bytes = await withRetries(async () => {
+            // A fresh page per attempt: the token is bound to the session cookie, and both may have lapsed.
+            const attemptPage = await fetch(PAGE_URL, { headers: { "user-agent": USER_AGENT, accept: "text/html" } });
+            const attemptToken = new RegExp(`${TOKEN_FIELD.replaceAll(".", "\\.")}" value="([^"]+)"`).exec(await attemptPage.text())?.[1];
+            if (!attemptToken)
+                throw new Error("the download page carried no token");
+            const res = await fetch(FORM_URL, {
+                method: "POST",
+                headers: {
+                    "user-agent": USER_AGENT,
+                    cookie: cookieHeader(attemptPage),
+                    referer: PAGE_URL,
+                    "content-type": "application/x-www-form-urlencoded",
+                },
+                body: new URLSearchParams({ [TOKEN_FIELD]: attemptToken, event: "download", selDlFileNo: fileNumber }),
+                signal: AbortSignal.timeout(1_800_000),
+            });
+            if (!res.ok || !(res.headers.get("content-type") ?? "").includes("octet-stream")) {
+                throw new Error(`HTTP ${res.status} ${res.headers.get("content-type") ?? ""}`);
+            }
+            filename = attachmentFilename(res, filename);
+            return streamBodyToFile(res, join(destDir, filename));
+        }, { report, label: filename });
+    }
+    catch (error) {
+        report?.(`  ✗ ${error.message}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: ["zenkoku-csv-unicode"] };
+    }
+    const dest = join(destDir, filename);
+    const sha = await sha256File(dest);
+    report?.(`  ✓ ${filename} ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
+    const manifest = {
+        source: SLUG,
+        source_url: PAGE_URL,
+        license: LICENSE,
+        attribution: ATTRIBUTION,
+        downloaded_at: new Date().toISOString(),
+        files: [{ source_url: FORM_URL, downloaded_at: new Date().toISOString(), filename, sha256: sha, bytes }],
+    };
+    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=houjin-jp.js.map

package/out/tools/fetch/houjin-jp.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"houjin-jp.js","sourceRoot":"","sources":["../../../lib/tools/fetch/houjin-jp.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEtH,MAAM,IAAI,GAAG,WAAW,CAAA;AACxB,MAAM,QAAQ,GAAG,sDAAsD,CAAA;AACvE,MAAM,QAAQ,GAAG,GAAG,QAAQ,YAAY,CAAA;AACxC,MAAM,WAAW,GAAG,gFAAgF,CAAA;AACpG,MAAM,OAAO,GAAG,6EAA6E,CAAA;AAC7F,MAAM,WAAW,GAAG,wEAAwE,CAAA;AAC5F,MAAM,UAAU,GAAG,wDAAwD,CAAA;AAI3E;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAY;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAA;IAE9C,IAAI,KAAK,KAAK,CAAC,CAAC;QAAE,OAAO,SAAS,CAAA;IAClC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAC,CAAA;IAC1E,MAAM,GAAG,GAAG,qCAAqC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAE/D,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAA;IAElG,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;IAC1F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAClG,MAAM,UAAU,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAA;IAEpD,IAAI,CAAC,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;QAC3B,MAAM,EAAE,CACP,wBAAwB,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,QAAQ,UAAU,CAAC,CAAC,CAAC,QAAQ,UAAU,EAAE,CAAC,CAAC,CAAC,+BAA+B,EAAE,CACnI,CAAA;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAA;IACnF,CAAC;IAED,MAAM,EAAE,CAAC,OAAO,IAAI,oCAAoC,UAAU,EAAE,CAAC,CAAA;IAErE,IAAI,QAAQ,GAAG,eAAe,UAAU,MAAM,CAAA;IAC9C,IAAI,KAAa,CAAA;IAEjB,IAAI,CAAC;QACJ,KAAK,GAAG,MAAM,WAAW,CACxB,KAAK,IAAI,EAAE;YACV,gGAAgG;YAChG,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAA;YAEzG,MAAM,YAAY,GAAG,IAAI,MAAM,CAAC,GAAG,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAC7F,MAAM,WAAW,CAAC,IAAI,EAAE,CACxB,EAAE,CAAC,CAAC,CAAC,CAAA;YAEN,IAAI,CAAC,YAAY;gBAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;YAExE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;gBACjC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACR,YAAY,EAAE,UAAU;oBACxB,MAAM,EAAE,YAAY,CAAC,WAAW,CAAC;oBACjC,OAAO,EAAE,QAAQ;oBACjB,cAAc,EAAE,mCAAmC;iBACnD;gBACD,IAAI,EAAE,IAAI,eAAe,CAAC,EAAE,CAAC,WAAW,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;gBACtG,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;aACtC,CAAC,CAAA;YAEF,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YAC/E,CAAC;YAED,QAAQ,GAAG,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;YAE5C,OAAO,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAA;QACtD,CAAC,EACD,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAC3B,CAAA;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,OAAQ,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;QAE3C,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAA;IACnF,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IACpC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,OAAO,QAAQ,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;IAEjF,MAAM,QAAQ,GAA6B;QAC1C,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,QAAQ;QACpB,OAAO,EAAE,OAAO;QAChB,WAAW,EAAE,WAAW;QACxB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;KACxG,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/tools/fetch/index.d.ts

@@ -57,6 +57,18 @@
  *   - `state-hi-schools` — Hawaii DOE school directory (original XLSX; HIDOE + PCS sheets). Tier A
  *       (state PD-equivalent).
  *   - `tiger-full` — US Census TIGER 2024 ADDRFEAT, all US counties. Tier A (US PD).
+ *   - `juso-kr` — the Korean road-name address register (주소DB, plus the English road-name DB) from
+ *       business.juso.go.kr, the LABEL source for Korean. Tier B ("이용허락범위 제한 없음", attribution).
+ *   - `localdata-kr` — Korea's local-government permit registry, one CSV per category, each row carrying
+ *       the lot-number AND road-name address of one premises plus a coordinate. The NOISY source for
+ *       Korean. Tier B (same label as `juso-kr`).
+ *   - `gcis-tw` — Taiwan's company and business registers from the economic ministry's open-data
+ *       platform, 148 files by region and industry. The NOISY source for Taiwanese. Tier B (Open
+ *       Government Data License v1.0 — the prescribed attribution is a condition of the grant).
+ *   - `acra-sg` — Singapore's ACRA corporate-entity register, 27 CSVs with the address FIELDED. The
+ *       NOISY source for Singapore. Tier B (Singapore Open Data Licence 1.0).
+ *   - `houjin-jp` — the National Tax Agency's corporate-number register, the nationwide Unicode CSV.
+ *       The NOISY source for Japanese. Tier A-equivalent (free use, no attribution condition stated).
  *
  *   License tiers per `docs/licensing-strategy.md` (or the playpen knowledge base mirror at
  *   `docs/docs/projects/mailwoman/licensing-strategy.md`). `openaddresses` is a **Tier-mixed**
@@ -96,11 +108,16 @@
  *      MANIFEST is well-formed.
  *   5. Add the source's adapter (or extend an existing one) under `corpus/src/adapters/`.
  */
+import { fetchACRASG } from "#tools/fetch/acra-sg";
 import { fetchBan } from "#tools/fetch/ban";
+import { fetchGCISTW } from "#tools/fetch/gcis-tw";
 import { fetchGeonamesDumps } from "#tools/fetch/geonames-dump";
 import { fetchGeonamesPostal } from "#tools/fetch/geonames-postal";
+import { fetchHoujinJP } from "#tools/fetch/houjin-jp";
 import { fetchHRSA } from "#tools/fetch/hrsa";
 import { fetchIMLSPLS } from "#tools/fetch/imls-pls";
+import { fetchJusoKR } from "#tools/fetch/juso-kr";
+import { fetchLocaldataKR } from "#tools/fetch/localdata-kr";
 import { fetchNAD } from "#tools/fetch/nad";
 import { fetchNPPES } from "#tools/fetch/nppes";
 import { fetchOpenAddresses } from "#tools/fetch/openaddresses";
@@ -109,11 +126,16 @@ import { fetchStateHISchools } from "#tools/fetch/state-hi-schools";
 import { fetchStateSources } from "#tools/fetch/state-sources";
 import { fetchTigerFull } from "#tools/fetch/tiger-full";
 import { fetchWikidataSubVenue } from "#tools/fetch/wikidata-subvenue";
+export * from "#tools/fetch/acra-sg";
 export * from "#tools/fetch/ban";
+export * from "#tools/fetch/gcis-tw";
 export * from "#tools/fetch/geonames-dump";
 export * from "#tools/fetch/geonames-postal";
+export * from "#tools/fetch/houjin-jp";
 export * from "#tools/fetch/hrsa";
 export * from "#tools/fetch/imls-pls";
+export * from "#tools/fetch/juso-kr";
+export * from "#tools/fetch/localdata-kr";
 export * from "#tools/fetch/nad";
 export * from "#tools/fetch/nppes";
 export * from "#tools/fetch/openaddresses";
@@ -126,7 +148,12 @@ export * from "#tools/fetch/wikidata-subvenue";
  * The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
  */
 export declare const FETCH_SOURCES: {
+    readonly "acra-sg": typeof fetchACRASG;
     readonly ban: typeof fetchBan;
+    readonly "gcis-tw": typeof fetchGCISTW;
+    readonly "houjin-jp": typeof fetchHoujinJP;
+    readonly "juso-kr": typeof fetchJusoKR;
+    readonly "localdata-kr": typeof fetchLocaldataKR;
     readonly nad: typeof fetchNAD;
     readonly "geonames-dump": typeof fetchGeonamesDumps;
     readonly "geonames-postal": typeof fetchGeonamesPostal;

package/out/tools/fetch/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAA;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAA;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAA;AAEtE,cAAc,kBAAkB,CAAA;AAChC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,mBAAmB,CAAA;AACjC,cAAc,uBAAuB,CAAA;AACrC,cAAc,kBAAkB,CAAA;AAChC,cAAc,oBAAoB,CAAA;AAClC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,0BAA0B,CAAA;AACxC,cAAc,+BAA+B,CAAA;AAC7C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,gCAAgC,CAAA;AAE9C;;GAEG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;CAchB,CAAA;AAEV,MAAM,MAAM,aAAa,GAAG,MAAM,OAAO,aAAa,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6GG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAA;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAA;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAA;AAEtE,cAAc,sBAAsB,CAAA;AACpC,cAAc,kBAAkB,CAAA;AAChC,cAAc,sBAAsB,CAAA;AACpC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,wBAAwB,CAAA;AACtC,cAAc,mBAAmB,CAAA;AACjC,cAAc,uBAAuB,CAAA;AACrC,cAAc,sBAAsB,CAAA;AACpC,cAAc,2BAA2B,CAAA;AACzC,cAAc,kBAAkB,CAAA;AAChC,cAAc,oBAAoB,CAAA;AAClC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,0BAA0B,CAAA;AACxC,cAAc,+BAA+B,CAAA;AAC7C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,gCAAgC,CAAA;AAE9C;;GAEG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;CAmBhB,CAAA;AAEV,MAAM,MAAM,aAAa,GAAG,MAAM,OAAO,aAAa,CAAA"}