@mailwoman/corpus 9.4.0 → 10.1.0

package/out/it/tools/fetch/anac.js

@@ -0,0 +1,272 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download ANAC's OCDS contract releases for the `it-anac` adapter.
+ *
+ * The Open Contracting Partnership's Data Registry serves publication 117 at
+ * `https://data.open-contracting.org/en/publication/117/download?name=<edition>.jsonl.gz`, where an
+ * edition is a year, `undated`, or `full` for every year in one file. Each file is JSON Lines, one
+ * OCDS release per line, which is the shape `#it/adapters/anac/adapter` reads.
+ *
+ * The acquisition belongs under a `tools/` root. `@mailwoman/corpus` is one of the
+ * `TOOLING_PACKAGES` in `dependency-cruiser.config.mjs`, which keep their tooling under `lib/`, so
+ * this module sits beside `#fi/tools/fetch/ryhti` rather than in an `sdk/` root the workspace does
+ * not declare.
+ *
+ * Three measured properties of the host decide what this module does. Measured on 2026-10-02:
+ *
+ * 1. **The host honors `Range`.** It answers `accept-ranges: bytes`, so an interrupted transfer
+ *    resumes through {@linkcode resumableDownload} rather than restarting. That matters for the
+ *    `full` edition, whose `content-length` was 183,974,254.
+ * 2. **An edition's length and modification time are both served by `HEAD`.** `full.jsonl.gz`
+ *    answered `last-modified` `Sat, 19 Sep 2026 03:30:49 GMT` and `2025.jsonl.gz` answered `Sat, 19
+ *    Sep 2026 03:31:23 GMT` with `content-length` 3,963,957. The re-run check compares the pair
+ *    against the manifest, which costs one small request per edition instead of the body.
+ * 3. **This module moves bytes and decodes none of them.** The compressed body is streamed to disk
+ *    and then decompressed stream to stream, so no chunk boundary is interpreted as a character
+ *    boundary. ANAC's data carries accented Italian place names, and a `chunk.toString("utf8")` per
+ *    gzip chunk writes U+FFFD at every boundary that splits a multi-byte character.
+ *
+ * ANAC updates the publication monthly, on the second of the month, which is why the current year's
+ * edition is re-fetched on a `HEAD` disagreement rather than treated as final.
+ *
+ * The registry publishes this release under CC BY 4.0, which the address-source register elected.
+ * The manifest records the license and the attribution, and the adapter records the license on every
+ * row.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { gunzipChunks } from "@mailwoman/core/fs/compression";
+import { ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { tryStat } from "@mailwoman/core/fs/readers";
+import { openReadStream, openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { PathBuilder } from "path-ts";
+import { IT_ANAC_ADAPTER_ID, IT_ANAC_ATTRIBUTION, IT_ANAC_LICENSE } from "#it/adapters/anac/adapter";
+import { loadCollectionFiles, resumableDownload, writeManifest } from "#tools/fetch/download";
+/**
+ * The publication the registry serves this release under.
+ *
+ * Italy has four publications on the registry and id 117 is ANAC's.
+ * Id 87 is OpenTender's Italian publication.
+ *
+ * It runs to March 2024 and comes from a different body.
+ * Reading the country alone would take the wrong publication.
+ */
+export const IT_ANAC_PUBLICATION_ID = 117;
+/**
+ * The download endpoint for one edition.
+ */
+export function anacEditionURL(edition) {
+    return `https://data.open-contracting.org/en/publication/${IT_ANAC_PUBLICATION_ID}/download?name=${edition}.jsonl.gz`;
+}
+/**
+ * The directory the editions are written under, which is the adapter's `inputPath`.
+ */
+const SLUG = IT_ANAC_ADAPTER_ID;
+/**
+ * The edition fetched when a caller names none.
+ *
+ * `full` is every year in one file, which is what a corpus build reads.
+ * A caller measuring one year names that year instead.
+ */
+export const IT_ANAC_DEFAULT_EDITIONS = ["full"];
+/**
+ * Read what the host states about one edition.
+ *
+ * Through `APIClient`, which is the interface for a repeated request with a small response.
+ * The bodies are not taken this way: `APIClient` buffers a response, and the `full` edition is 184 MB.
+ *
+ * A header the host does not send is reported as `null` rather than raised on,
+ * and the caller then downloads rather than reading the absence as a match.
+ */
+export async function readANACEditionHead(client, edition) {
+    const response = await client.fetch({ method: "head", url: anacEditionURL(edition) });
+    const headers = response.headers;
+    const length = headers?.["content-length"];
+    return {
+        lastModified: headers?.["last-modified"] ?? null,
+        contentLength: length !== undefined && /^\d+$/u.test(length) ? Number(length) : null,
+        contentType: headers?.["content-type"] ?? null,
+    };
+}
+/**
+ * The directory `#it/adapters/anac/adapter` reads under a fetch root.
+ *
+ * The adapter takes the directory rather than one file, because a build reading
+ * several years reads every edition in it.
+ */
+export function itANACInputPath(outRoot) {
+    return outRoot(SLUG);
+}
+/**
+ * Whether an edition on disk is still the one the manifest describes and the host still serves.
+ *
+ * The `HEAD` decides first, because ANAC republishes the publication monthly
+ * and the host states a new length and modification time when it does.
+ * The file's own length is then checked, so a truncated local copy is re-fetched even
+ * where the host has not changed.
+ *
+ * Exported so a test can exercise the decision without a transfer: the alternative is
+ * letting {@linkcode downloadITANAC} reach the host to find out which branch it took.
+ */
+export async function isANACEditionCurrent(recorded, head, path, verifyDigest) {
+    if (!recorded)
+        return false;
+    // A host that states neither cannot be compared against, so the edition is downloaded.
+    if (head.lastModified === null && head.contentLength === null)
+        return false;
+    if (head.lastModified !== null && head.lastModified !== recorded.last_modified)
+        return false;
+    if (head.contentLength !== null && head.contentLength !== recorded.compressed_bytes)
+        return false;
+    const stat = await tryStat(path);
+    if (!stat || stat.size !== recorded.bytes)
+        return false;
+    if (!verifyDigest)
+        return true;
+    return (await sha256File(path)) === recorded.sha256;
+}
+/**
+ * Download each requested edition into `options.outputDir` and leave them beside one manifest.
+ *
+ * Re-runnable per edition: an edition whose `HEAD` matches the manifest's recorded
+ * modification time and compressed length, and whose `.jsonl` is still on disk at
+ * the recorded length, makes no request for the body.
+ */
+export async function downloadITANAC(client, options) {
+    const { report } = options;
+    const destDir = PathBuilder.from(options.outputDir);
+    const editions = options.editions?.length ? options.editions : IT_ANAC_DEFAULT_EDITIONS;
+    await makeDirectories(destDir);
+    const manifestPath = destDir("MANIFEST.json");
+    const recorded = (await loadCollectionFiles(manifestPath));
+    const written = new Map(recorded);
+    const summary = { fetched: 0, skipped: 0, failed: 0, failedCodes: [] };
+    report?.(`=== ${SLUG}`);
+    for (const edition of editions) {
+        const filename = `${edition}.jsonl`;
+        const editionPath = destDir(filename);
+        const compressedPath = destDir(`${filename}.gz`);
+        let head;
+        try {
+            head = await readANACEditionHead(client, edition);
+        }
+        catch (error) {
+            report?.(`  ✗ ${edition}: ${error instanceof Error ? error.message : String(error)}`);
+            summary.failed++;
+            summary.failedCodes.push(edition);
+            continue;
+        }
+        report?.(`  ${edition}: ${head.contentLength === null ? "no content-length" : `${ByteFormatter.formatIEC(head.contentLength)} compressed`}` +
+            `, last-modified ${head.lastModified ?? "absent"}`);
+        const current = !options.force &&
+            (await isANACEditionCurrent(recorded.get(filename), head, editionPath, options.verifyDigest ?? false));
+        if (current) {
+            report?.(`  ✓ ${edition} already current — no download.`);
+            summary.skipped++;
+            continue;
+        }
+        let compressedBytes;
+        try {
+            compressedBytes = await resumableDownload({
+                url: anacEditionURL(edition),
+                dest: compressedPath,
+                headers: { accept: "application/gzip, application/x-gzip, */*" },
+                retryDelayMs: options.retryDelayMs,
+                report,
+            });
+        }
+        catch (error) {
+            report?.(`  ✗ ${edition} download failed: ${error instanceof Error ? error.message : String(error)}`);
+            await removePathIfPresent(compressedPath);
+            summary.failed++;
+            summary.failedCodes.push(edition);
+            continue;
+        }
+        // The delivered count rather than the stated one.
+        // ANAC republishes monthly, so a `HEAD` taken before the transfer can describe
+        // a different edition than the body that arrived.
+        if (head.contentLength !== null && head.contentLength !== compressedBytes) {
+            report?.(`  ! ${edition}: the HEAD stated ${head.contentLength} compressed bytes and ${compressedBytes} arrived, ` +
+                `so the edition was republished between the two requests`);
+        }
+        try {
+            await decompressToJSONL(compressedPath, editionPath);
+        }
+        catch (error) {
+            report?.(`  ✗ ${edition} decompress failed: ${error instanceof Error ? error.message : String(error)}`);
+            await removePathIfPresent(compressedPath);
+            summary.failed++;
+            summary.failedCodes.push(edition);
+            continue;
+        }
+        const stat = await tryStat(editionPath);
+        if (!stat) {
+            report?.(`  ✗ ${edition}: the decompressed edition is not at ${editionPath.toString()}`);
+            summary.failed++;
+            summary.failedCodes.push(edition);
+            continue;
+        }
+        if (!options.keepCompressed) {
+            await removePathIfPresent(compressedPath);
+        }
+        written.set(filename, {
+            source_url: anacEditionURL(edition),
+            downloaded_at: new Date().toISOString(),
+            filename,
+            sha256: await sha256File(editionPath),
+            bytes: stat.size,
+            last_modified: head.lastModified,
+            compressed_bytes: compressedBytes,
+        });
+        report?.(`  ✓ ${edition}: ${ByteFormatter.formatIEC(stat.size)} decompressed`);
+        summary.fetched++;
+    }
+    const manifest = {
+        source: SLUG,
+        source_url: `https://data.open-contracting.org/en/publication/${IT_ANAC_PUBLICATION_ID}`,
+        license: IT_ANAC_LICENSE,
+        attribution: IT_ANAC_ATTRIBUTION,
+        downloaded_at: new Date().toISOString(),
+        files: [...written.values()].toSorted((left, right) => left.filename.localeCompare(right.filename)),
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  MANIFEST written to ${manifestPath.toString()}`);
+    return summary;
+}
+/**
+ * Download ANAC's OCDS editions into `<outRoot>/it-anac/`.
+ *
+ * The registry entry point.
+ * `#it/adapters/anac/adapter` is pointed at {@linkcode itANACInputPath},
+ * the directory itself, so it reads every edition inside it.
+ */
+export async function fetchITANAC(options, report) {
+    await using client = new APIClient({ displayName: SLUG, retry: true });
+    // Awaited rather than returned: `await using` disposes the client when this scope exits,
+    // and a disposed `APIClient` refuses every later request.
+    return await downloadITANAC(client, {
+        outputDir: options.outRoot(SLUG),
+        editions: options.editions,
+        keepCompressed: options.keepCompressed,
+        verifyDigest: options.verifyDigest,
+        force: options.force,
+        retryDelayMs: options.retryDelayMs,
+        signal: options.signal,
+        report,
+    });
+}
+/**
+ * Decompress one gzip file to another path, stream to stream.
+ *
+ * Exported because a test exercises it directly: an accented Italian place name
+ * split across two gzip chunks must arrive whole.
+ */
+export async function decompressToJSONL(source, destination) {
+    const decompressed = gunzipChunks(openReadStream(source));
+    await pipeline(Readable.fromWeb(decompressed), openWriteStream(destination));
+}
+//# sourceMappingURL=anac.js.map

package/out/it/tools/fetch/anac.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"anac.js","sourceRoot":"","sources":["../../../../lib/it/tools/fetch/anac.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAA;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAChG,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAEpG,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAE7F;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAEzC;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC7C,OAAO,oDAAoD,sBAAsB,kBAAkB,OAAO,WAAW,CAAA;AACtH,CAAC;AAED;;GAEG;AACH,MAAM,IAAI,GAAG,kBAAkB,CAAA;AAE/B;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAsB,CAAC,MAAM,CAAC,CAAA;AA8BnE;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,MAAgC,EAAE,OAAe;IAC1F,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CAAU,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC9F,MAAM,OAAO,GAAG,QAAQ,CAAC,OAA6C,CAAA;IACtE,MAAM,MAAM,GAAG,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAA;IAE1C,OAAO;QACN,YAAY,EAAE,OAAO,EAAE,CAAC,eAAe,CAAC,IAAI,IAAI;QAChD,aAAa,EAAE,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;QACpF,WAAW,EAAE,OAAO,EAAE,CAAC,cAAc,CAAC,IAAI,IAAI;KAC9C,CAAA;AACF,CAAC;AAqCD;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,OAAoC;IACnE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;AACrB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,QAAyC,EACzC,IAAqB,EACrB,IAAqB,EACrB,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAA;IAE3B,uFAAuF;IACvF,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI;QAAE,OAAO,KAAK,CAAA;IAE3E,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ,CAAC,aAAa;QAAE,OAAO,KAAK,CAAA;IAE5F,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,aAAa,KAAK,QAAQ,CAAC,gBAAgB;QAAE,OAAO,KAAK,CAAA;IAEjG,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK;QAAE,OAAO,KAAK,CAAA;IAEvD,IAAI,CAAC,YAAY;QAAE,OAAO,IAAI,CAAA;IAE9B,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAA;AACpD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,MAAgC,EAChC,OAA4B;IAE5B,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAC1B,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,wBAAwB,CAAA;IAEvF,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IAC7C,MAAM,QAAQ,GAAG,CAAC,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAqC,CAAA;IAC9F,MAAM,OAAO,GAAG,IAAI,GAAG,CAA8B,QAAQ,CAAC,CAAA;IAC9D,MAAM,OAAO,GAAiB,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IAEpF,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,GAAG,OAAO,QAAQ,CAAA;QACnC,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;QACrC,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,QAAQ,KAAK,CAAC,CAAA;QAEhD,IAAI,IAAqB,CAAA;QAEzB,IAAI,CAAC;YACJ,IAAI,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAClD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAO,OAAO,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YAErF,OAAO,CAAC,MAAM,EAAE,CAAA;YAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEjC,SAAQ;QACT,CAAC;QAED,MAAM,EAAE,CACP,KAAK,OAAO,KAAK,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE;YACjI,mBAAmB,IAAI,CAAC,YAAY,IAAI,QAAQ,EAAE,CACnD,CAAA;QAED,MAAM,OAAO,GACZ,CAAC,OAAO,CAAC,KAAK;YACd,CAAC,MAAM,oBAAoB,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC,CAAC,CAAA;QAEvG,IAAI,OAAO,EAAE,CAAC;YACb,MAAM,EAAE,CAAC,OAAO,OAAO,iCAAiC,CAAC,CAAA;YAEzD,OAAO,CAAC,OAAO,EAAE,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,eAAuB,CAAA;QAE3B,IAAI,CAAC;YACJ,eAAe,GAAG,MAAM,iBAAiB,CAAC;gBACzC,GAAG,EAAE,cAAc,CAAC,OAAO,CAAC;gBAC5B,IAAI,EAAE,cAAc;gBACpB,OAAO,EAAE,EAAE,MAAM,EAAE,2CAA2C,EAAE;gBAChE,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,MAAM;aACN,CAAC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAO,OAAO,qBAAqB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACrG,MAAM,mBAAmB,CAAC,cAAc,CAAC,CAAA;YAEzC,OAAO,CAAC,MAAM,EAAE,CAAA;YAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEjC,SAAQ;QACT,CAAC;QAED,kDAAkD;QAClD,+EAA+E;QAC/E,kDAAkD;QAClD,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;YAC3E,MAAM,EAAE,CACP,OAAO,OAAO,qBAAqB,IAAI,CAAC,aAAa,yBAAyB,eAAe,YAAY;gBACxG,yDAAyD,CAC1D,CAAA;QACF,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,iBAAiB,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAO,OAAO,uBAAuB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACvG,MAAM,mBAAmB,CAAC,cAAc,CAAC,CAAA;YAEzC,OAAO,CAAC,MAAM,EAAE,CAAA;YAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEjC,SAAQ;QACT,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,CAAA;QAEvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,MAAM,EAAE,CAAC,OAAO,OAAO,wCAAwC,WAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;YAExF,OAAO,CAAC,MAAM,EAAE,CAAA;YAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEjC,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAC7B,MAAM,mBAAmB,CAAC,cAAc,CAAC,CAAA;QAC1C,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE;YACrB,UAAU,EAAE,cAAc,CAAC,OAAO,CAAC;YACnC,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ;YACR,MAAM,EAAE,MAAM,UAAU,CAAC,WAAW,CAAC;YACrC,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,aAAa,EAAE,IAAI,CAAC,YAAY;YAChC,gBAAgB,EAAE,eAAe;SACjC,CAAC,CAAA;QAEF,MAAM,EAAE,CAAC,OAAO,OAAO,KAAK,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;QAE9E,OAAO,CAAC,OAAO,EAAE,CAAA;IAClB,CAAC;IAED,MAAM,QAAQ,GAA6B;QAC1C,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,oDAAoD,sBAAsB,EAAE;QACxF,OAAO,EAAE,eAAe;QACxB,WAAW,EAAE,mBAAmB;QAChC,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;KACnG,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,yBAAyB,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAE5D,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAA2B,EAAE,MAA+B;IAC7F,YAAY,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtE,yFAAyF;IACzF,0DAA0D;IAC1D,OAAO,MAAM,cAAc,CAAC,MAAM,EAAE;QACnC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM;KACN,CAAC,CAAA;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,WAA4B;IAC5F,MAAM,YAAY,GAAG,YAAY,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAA;IAEzD,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAsD,CAAC,EAAE,eAAe,CAAC,WAAW,CAAC,CAAC,CAAA;AACvH,CAAC"}

package/out/jp/adapters/wof-admin/adapter.d.ts

@@ -0,0 +1,68 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `wof-admin-jp`: Japanese admin-hierarchy adapter.
+ *
+ *   prototype — not yet wired into the Stage 3 training corpus. Demonstrates the JP address shape and
+ *   synthesizes BIO-labeled training rows from the global WOF SQLite.
+ *
+ *   JP addresses differ from US/EU in three ways:
+ *
+ *   1. **Reversed ordering** — region → county → locality → block → house_number "東京都世田谷区南町1-2-3" not
+ *        "1-2-3 Minamicho, Setagaya, Tokyo"
+ *   2. **No street names** — most JP addresses use a block/sub-block grid system. The "neighborhood"
+ *        placetype (丁目) is the closest analog to a street but is actually a grid cell.
+ *   3. **Postcode-first convention** — addresses are often prefixed with `〒NNN-nnnn`.
+ *
+ *   Schema mapping to ComponentTags (subset of Stage 3 + JP-specific Phase 6 tags):
+ *
+ *   | JP concept | WOF placetype | ComponentTag (Phase 6) |
+ *   |-----------------|--------------------|-----------------------| | 都道府県 (prefecture) | region |
+ *   region (or prefecture) | | 市区町村 (city/ward) | county/locality | locality (or municipality) | |
+ *   丁目 (chome) | neighborhood | block (Phase 6 tag) | | 番地 (banchi) | (synth) | sub_block | | 号
+ *   (gō) | (synth) | house_number | | 〒 (postcode prefix) | — | postcode |
+ *
+ *   This adapter currently emits only the admin chain (region → locality → block). House numbers and
+ *   sub-blocks require a different data source (JP postcode database or real-world address dumps
+ *   from mlit/JapanPost).
+ */
+import { type CanonicalRow, type CorpusAdapter } from "#types";
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export declare const WOF_ADMIN_JP_ADAPTER_ID = "wof-admin-jp";
+interface PlaceRow {
+    id: number;
+    name: string;
+    placetype: string;
+    parent_id: number;
+    country: string;
+}
+/**
+ * Synthesize a JP address from a hierarchy chain.
+ *
+ * Format: `〒<postcode>? <region><locality><neighbourhood>?`
+ *
+ * No house numbers yet — needs mlit data.
+ */
+export declare function synthesizeJpAddress(chain: PlaceRow[], jpnNames: Map<number, string>): {
+    raw: string;
+    components: CanonicalRow["components"];
+} | null;
+/**
+ * Build the JP adapter.
+ *
+ * Reads from the unified global WOF SQLite and walks admin chains from neighborhoods.
+ * Yields canonical rows.
+ */
+export declare function createWOFAdminJpAdapter(): CorpusAdapter;
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export declare const wofAdminJpAdapter: CorpusAdapter;
+export {};
+//# sourceMappingURL=adapter.d.ts.map

package/out/jp/adapters/wof-admin/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../lib/jp/adapters/wof-admin/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAQH,OAAO,EAAoC,KAAK,YAAY,EAAE,KAAK,aAAa,EAAiB,MAAM,QAAQ,CAAA;AAO/G;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,iBAAiB,CAAA;AAErD,UAAU,QAAQ;IACjB,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CACf;AAuCD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAClC,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC3B;IACF,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;CACtC,GAAG,IAAI,CAyBP;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,IAAI,aAAa,CA4EvD;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,eAA4B,CAAA"}

package/out/jp/adapters/wof-admin/adapter.js

@@ -0,0 +1,172 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `wof-admin-jp`: Japanese admin-hierarchy adapter.
+ *
+ *   prototype — not yet wired into the Stage 3 training corpus. Demonstrates the JP address shape and
+ *   synthesizes BIO-labeled training rows from the global WOF SQLite.
+ *
+ *   JP addresses differ from US/EU in three ways:
+ *
+ *   1. **Reversed ordering** — region → county → locality → block → house_number "東京都世田谷区南町1-2-3" not
+ *        "1-2-3 Minamicho, Setagaya, Tokyo"
+ *   2. **No street names** — most JP addresses use a block/sub-block grid system. The "neighborhood"
+ *        placetype (丁目) is the closest analog to a street but is actually a grid cell.
+ *   3. **Postcode-first convention** — addresses are often prefixed with `〒NNN-nnnn`.
+ *
+ *   Schema mapping to ComponentTags (subset of Stage 3 + JP-specific Phase 6 tags):
+ *
+ *   | JP concept | WOF placetype | ComponentTag (Phase 6) |
+ *   |-----------------|--------------------|-----------------------| | 都道府県 (prefecture) | region |
+ *   region (or prefecture) | | 市区町村 (city/ward) | county/locality | locality (or municipality) | |
+ *   丁目 (chome) | neighborhood | block (Phase 6 tag) | | 番地 (banchi) | (synth) | sub_block | | 号
+ *   (gō) | (synth) | house_number | | 〒 (postcode prefix) | — | postcode |
+ *
+ *   This adapter currently emits only the admin chain (region → locality → block). House numbers and
+ *   sub-blocks require a different data source (JP postcode database or real-world address dumps
+ *   from mlit/JapanPost).
+ */
+import { isPresent } from "@mailwoman/core/objects";
+import { allRows, getRow } from "@mailwoman/core/utils";
+import { DatabaseClient } from "@mailwoman/sqlite/client";
+import { SourceRegister } from "#registers";
+import { AddressRole, SurfaceOrigin } from "#types";
+/**
+ * Ancestry hops walked upward from a WOF record before giving up — deeper than any real JP admin chain.
+ */
+const MAX_ANCESTRY_DEPTH = 6;
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const WOF_ADMIN_JP_ADAPTER_ID = "wof-admin-jp";
+/**
+ * Walk parent chain up to 6 levels, over a place table already resident in memory.
+ *
+ * `resolve` is the override for a parent outside the preloaded set (a chain that leaves the country);
+ * it is a point query, so keeping the common case out of it is the whole point.
+ * The GeoJSON adapters' in-memory analogue is `buildAncestorNameIndex` (`#utils`), and the two stay
+ * separate because this one's rows come from SQLite and a chain can leave the preloaded country set.
+ *
+ * That analogue stores the nearest ancestor's name for each tag.
+ * This function receives one `PlaceRow` per level and reads the chain directly.
+ */
+function chainOf(byID, resolve, startID) {
+    const out = [];
+    let id = startID;
+    for (let i = 0; i < MAX_ANCESTRY_DEPTH && id > 0; i++) {
+        const row = byID.get(id) ?? resolve(id);
+        if (!row)
+            break;
+        out.push(row);
+        id = row.parent_id;
+    }
+    return out;
+}
+/**
+ * Pick the best display name for a place: prefer Japanese variant, fall back to English.
+ */
+function pickName(row, jpnNames) {
+    return jpnNames.get(row.id) ?? row.name;
+}
+/**
+ * Synthesize a JP address from a hierarchy chain.
+ *
+ * Format: `〒<postcode>? <region><locality><neighbourhood>?`
+ *
+ * No house numbers yet — needs mlit data.
+ */
+export function synthesizeJpAddress(chain, jpnNames) {
+    const region = chain.find((r) => r.placetype === "region");
+    const locality = chain.find((r) => r.placetype === "locality" || r.placetype === "county");
+    if (!region || !locality)
+        return null;
+    const neighbourhood = chain.find((r) => r.placetype === "neighbourhood");
+    const regionName = pickName(region, jpnNames);
+    const localityName = pickName(locality, jpnNames);
+    const neighbourhoodName = neighbourhood ? pickName(neighbourhood, jpnNames) : null;
+    const components = {
+        region: regionName,
+        locality: localityName,
+        country: "JP",
+    };
+    if (neighbourhoodName) {
+        components.dependent_locality = neighbourhoodName;
+    }
+    const raw = [regionName, localityName, neighbourhoodName].filter(isPresent).join("");
+    return { raw, components };
+}
+/**
+ * Build the JP adapter.
+ *
+ * Reads from the unified global WOF SQLite and walks admin chains from neighborhoods.
+ * Yields canonical rows.
+ */
+export function createWOFAdminJpAdapter() {
+    return {
+        id: WOF_ADMIN_JP_ADAPTER_ID,
+        defaultLicense: "CC-BY-4.0",
+        addressRole: AddressRole.Premise,
+        register: SourceRegister.WhosOnFirst,
+        surface: SurfaceOrigin.Rendered,
+        description: "Japanese admin hierarchy from WOF (synthesized addresses without house numbers).",
+        async *rows(opts) {
+            if (opts.country && opts.country !== "JP") {
+                throw new Error(`wof-admin-jp adapter: only JP supported, got country=${opts.country}`);
+            }
+            using db = new DatabaseClient(opts.inputPath, { readOnly: true });
+            const jpnNamesStmt = db.prepare(`SELECT id, name FROM names WHERE language = 'jpn'`);
+            const jpnNames = new Map();
+            for (const row of jpnNamesStmt.all()) {
+                if (!jpnNames.has(row.id)) {
+                    jpnNames.set(row.id, row.name);
+                }
+            }
+            // One read of the JP place table instead of a fresh `prepare` + up to six point queries per seed.
+            // There are tens of thousands of neighborhood seeds.
+            const byID = new Map();
+            for (const row of allRows(db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE country='JP'`))) {
+                byID.set(row.id, row);
+            }
+            const outsideStmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`);
+            const resolveOutside = (id) => {
+                const row = getRow(outsideStmt, id);
+                if (row) {
+                    byID.set(id, row);
+                }
+                return row;
+            };
+            const seeds = [...byID.values()].filter((row) => row.placetype === "neighbourhood");
+            let emitted = 0;
+            for (const seed of seeds) {
+                if (opts.signal?.aborted)
+                    break;
+                if (opts.limit !== undefined && emitted >= opts.limit)
+                    break;
+                const chain = chainOf(byID, resolveOutside, seed.id);
+                const synth = synthesizeJpAddress(chain, jpnNames);
+                if (!synth)
+                    continue;
+                yield {
+                    raw: synth.raw,
+                    components: synth.components,
+                    country: "JP",
+                    locale: "ja-JP",
+                    source: WOF_ADMIN_JP_ADAPTER_ID,
+                    source_id: `${WOF_ADMIN_JP_ADAPTER_ID}-${seed.id}`,
+                    corpus_version: "",
+                    license: "CC-BY-4.0",
+                };
+                emitted++;
+            }
+        },
+    };
+}
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const wofAdminJpAdapter = createWOFAdminJpAdapter();
+//# sourceMappingURL=adapter.js.map

package/out/jp/adapters/wof-admin/adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../lib/jp/adapters/wof-admin/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,WAAW,EAA8D,aAAa,EAAE,MAAM,QAAQ,CAAA;AAE/G;;GAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAE5B;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,cAAc,CAAA;AAUrD;;;;;;;;;;GAUG;AACH,SAAS,OAAO,CACf,IAA2B,EAC3B,OAA6C,EAC7C,OAAe;IAEf,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,IAAI,EAAE,GAAG,OAAO,CAAA;IAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,kBAAkB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,EAAE,CAAC,CAAA;QAEvC,IAAI,CAAC,GAAG;YAAE,MAAK;QACf,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACb,EAAE,GAAG,GAAG,CAAC,SAAS,CAAA;IACnB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CAAC,GAAa,EAAE,QAA6B;IAC7D,OAAO,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,CAAA;AACxC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAClC,KAAiB,EACjB,QAA6B;IAK7B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAA;IAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAA;IAE1F,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,eAAe,CAAC,CAAA;IAExE,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;IAC7C,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IACjD,MAAM,iBAAiB,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAElF,MAAM,UAAU,GAA+B;QAC9C,MAAM,EAAE,UAAU;QAClB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,IAAI;KACb,CAAA;IAED,IAAI,iBAAiB,EAAE,CAAC;QACvB,UAAU,CAAC,kBAAkB,GAAG,iBAAiB,CAAA;IAClD,CAAC;IAED,MAAM,GAAG,GAAG,CAAC,UAAU,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEpF,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC3B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB;IACtC,OAAO;QACN,EAAE,EAAE,uBAAuB;QAC3B,cAAc,EAAE,WAAW;QAC3B,WAAW,EAAE,WAAW,CAAC,OAAO;QAChC,QAAQ,EAAE,cAAc,CAAC,WAAW;QACpC,OAAO,EAAE,aAAa,CAAC,QAAQ;QAC/B,WAAW,EAAE,kFAAkF;QAE/F,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,wDAAwD,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;YACxF,CAAC;YAED,MAAM,EAAE,GAAG,IAAI,cAAc,CAAc,IAAI,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;YAE9E,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC,mDAAmD,CAAC,CAAA;YACpF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;YAE1C,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,GAAG,EAAoC,EAAE,CAAC;gBACxE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAC3B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAA;gBAC/B,CAAC;YACF,CAAC;YAED,kGAAkG;YAClG,qDAAqD;YACrD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAoB,CAAA;YAExC,KAAK,MAAM,GAAG,IAAI,OAAO,CACxB,EAAE,CAAC,OAAO,CAAC,4EAA4E,CAAC,CACxF,EAAE,CAAC;gBACH,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;YACtB,CAAC;YAED,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,sEAAsE,CAAC,CAAA;YAEtG,MAAM,cAAc,GAAG,CAAC,EAAU,EAAwB,EAAE;gBAC3D,MAAM,GAAG,GAAG,MAAM,CAAW,WAAW,EAAE,EAAE,CAAC,CAAA;gBAE7C,IAAI,GAAG,EAAE,CAAC;oBACT,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;gBAClB,CAAC;gBAED,OAAO,GAAG,CAAA;YACX,CAAC,CAAA;YAED,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,eAAe,CAAC,CAAA;YAEnF,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAK;gBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,MAAK;gBAE5D,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,CAAA;gBACpD,MAAM,KAAK,GAAG,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;gBAElD,IAAI,CAAC,KAAK;oBAAE,SAAQ;gBAEpB,MAAM;oBACL,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,uBAAuB;oBAC/B,SAAS,EAAE,GAAG,uBAAuB,IAAI,IAAI,CAAC,EAAE,EAAE;oBAClD,cAAc,EAAE,EAAE;oBAClB,OAAO,EAAE,WAAW;iBACpB,CAAA;gBAED,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,uBAAuB,EAAE,CAAA"}

package/out/jp/tools/fetch/houjin.d.ts

@@ -0,0 +1,27 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the National Tax Agency's corporate-number register (法人番号公表サイト 全件データ): every
+ *   corporation assigned a number, about 5,000,000 rows, with the head-office address as one string
+ *   (国内所在地) plus the prefecture and municipality split out and the postcode. The nationwide CSV
+ *   in Unicode is one zip of about 255 MB.
+ *
+ *   The agency issues the corporate number, so the identifier is an `identity` assertion. The head-office
+ *   string beside it is an `observation` the corporation filed, on `registered-office` rows. The Overture
+ *   rows the JP corpus is built from assert `address` on `premise` rows. That distinction matters when the two are mixed.
+ *
+ *   The agency states the three published fields may be used freely by anyone ("どなたでも自由にご利用
+ *   いただくことができます"), with no attribution condition on the download page.
+ *
+ *   The file sits behind a form. The page hands out a per-session token. A post with that token,
+ *   `event=download` and the file number of the nationwide Unicode CSV answers the zip. The file
+ *   number is read off the page rather than pinned, because the agency re-issues the files monthly.
+ *
+ *   Invoke via `mailwoman corpus fetch houjin-jp --out-root <path>`.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export type FetchHoujinJPOptions = BaseFetchOptions;
+export declare function fetchHoujinJP(options: FetchHoujinJPOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=houjin.d.ts.map

package/out/jp/tools/fetch/houjin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"houjin.d.ts","sourceRoot":"","sources":["../../../../lib/jp/tools/fetch/houjin.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAKH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAA4B,MAAM,uBAAuB,CAAA;AAWrG,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,CAAA;AAenD,wBAAsB,aAAa,CAClC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAkFvB"}