@mailwoman/corpus 10.0.0 → 10.2.0

package/out/lu/tools/fetch/bd-adresses.js

@@ -0,0 +1,223 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download the Administration du cadastre et de la topographie's BD-Adresses CSV for the
+ * `bd-adresses` adapter.
+ *
+ * The acquisition belongs under a `tools/` root. `@mailwoman/corpus` is one of the
+ * `TOOLING_PACKAGES` in `dependency-cruiser.config.mjs`, which keep their tooling under `lib/`, so
+ * this module sits beside `#fi/tools/fetch/ryhti` rather than in an `sdk/` root the workspace does
+ * not declare.
+ *
+ * `data.public.lu` states `cc-zero` for this dataset. The address-source register elects
+ * `CC0-1.0` for it under `lu-property-building-1`. CC0 asks for no attribution, and crediting the ACT
+ * remains good practice, so the manifest records the publisher beside the license.
+ *
+ * Four measured properties decide what this module does:
+ *
+ * 1. **The download URL holds the edition's own timestamp.** The dataset is republished weekly
+ *    and the CSV resource's `url` reads
+ *    `https://download.data.public.lu/resources/adresses-georeferencees-bd-adresses/20260928-023119/addresses.csv`,
+ *    where the path segment is the publication time. A hardcoded URL therefore pins one edition and
+ *    404s on the next. The module reads {@linkcode LU_BD_ADRESSES_DATASET_URL} and takes the `url`
+ *    of the resource whose `format` is `csv`. The resource's own `latest`,
+ *    `https://data.public.lu/fr/datasets/r/<resource id>`, answers http 302 to the same dated path,
+ *    and the dated path is the one downloaded, so the bytes and the checksum recorded beside them
+ *    belong to one edition.
+ * 2. **The publisher states an md5 for the file.** The resource records
+ *    `checksum: {type: "md5", value: "…"}` and a `filesize`, measured as
+ *    `c2df2f3c18b846144bb5d5dc57fc14ca` and 28,186,970 on the 2026-09-28 edition. The transfer is
+ *    checked against both, so a truncated body is a reported failure rather than a short CSV that
+ *    parses. A publisher that one day states a digest of another type is reported rather than
+ *    skipped silently.
+ * 3. **The file opens with a UTF-8 byte-order mark**, `EF BB BF`. The portal's `filesize` counts it.
+ *    A reader that discards the mark therefore reports three bytes fewer than the portal does.
+ *    The module writes the body to disk unaltered and checks the digest over the bytes as delivered.
+ * 4. **The re-run check is the stated checksum against the manifest.** The dataset API answer is
+ *    roughly 15 KB, so comparing it costs one small request instead of 28 MB.
+ *
+ * The CSV is the only one of the three bulk files this reads. The GeoJSON is 80,836,067 bytes and
+ * the shapefile 19,681,289 for the same records, and `#lu/adapters/bd-adresses/adapter` hands
+ * `opts.inputPath` to `CSVSpliterator.fromAsync`.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { tryStat } from "@mailwoman/core/fs/readers/stat";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { md5File, sha256File } from "@mailwoman/core/hash";
+import { PathBuilder } from "path-ts";
+import { BD_ADRESSES_ADAPTER_ID, BD_ADRESSES_DEFAULT_LICENSE, BD_ADRESSES_REQUIRED_COLUMNS, } from "#lu/adapters/bd-adresses/adapter";
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download";
+import { assertHeaderColumns, readDelimitedHeader } from "#tools/fetch/header";
+/**
+ * The dataset record that states the current edition of each bulk file.
+ */
+export const LU_BD_ADRESSES_DATASET_URL = "https://data.public.lu/api/1/datasets/adresses-georeferencees-bd-adresses/";
+/**
+ * The directory the download is written under, the adapter's `inputPath`'s parent.
+ */
+const SLUG = BD_ADRESSES_ADAPTER_ID;
+/**
+ * The file the adapter reads.
+ *
+ * It is the publisher's own name for the resource.
+ */
+export const LU_BD_ADRESSES_CSV_FILENAME = "addresses.csv";
+/**
+ * The publisher the manifest credits.
+ */
+export const LU_BD_ADRESSES_ATTRIBUTION = "Administration du cadastre et de la topographie";
+/**
+ * The semicolon the publisher delimits with.
+ */
+export const LU_BD_ADRESSES_DELIMITER = ";";
+/**
+ * The CSV resource of a dataset record.
+ *
+ * @throws When the record holds no `csv` resource with a URL, because the publisher
+ * having moved the file is a reported failure rather than an empty transfer.
+ */
+export function readBDAdressesResource(record) {
+    const resource = (record.resources ?? []).find((entry) => entry.format === "csv" && Boolean(entry.url));
+    if (!resource?.url) {
+        const formats = (record.resources ?? []).map((entry) => entry.format ?? "?").join(", ");
+        throw new Error(`${LU_BD_ADRESSES_DATASET_URL} names no csv resource with a url. Its resources read ${formats || "(none)"}.`);
+    }
+    const checksum = resource.checksum;
+    return {
+        url: resource.url,
+        filesize: resource.filesize ?? null,
+        // A digest of another type is read as none rather than compared as an md5,
+        // so a `sha1` value is never reported as an md5 mismatch.
+        md5: checksum?.type === "md5" && checksum.value ? checksum.value : null,
+        lastModified: resource.last_modified ?? null,
+    };
+}
+/**
+ * Download the current edition of `addresses.csv` into `options.outputDir`.
+ *
+ * `client` is the caller's, so a test supplies its own and {@linkcode fetchBDAdresses}
+ * decides the retry policy.
+ */
+export async function downloadBDAdresses(client, options) {
+    const { report } = options;
+    const destDir = PathBuilder.from(options.outputDir);
+    const csvPath = destDir(LU_BD_ADRESSES_CSV_FILENAME);
+    const manifestPath = destDir("MANIFEST.json");
+    await makeDirectories(destDir);
+    report?.(`=== ${SLUG} / ${LU_BD_ADRESSES_CSV_FILENAME}`);
+    let resource;
+    try {
+        const { data } = await client.fetch({
+            method: "GET",
+            url: LU_BD_ADRESSES_DATASET_URL,
+            responseType: "json",
+        });
+        resource = readBDAdressesResource(data);
+    }
+    catch (error) {
+        report?.(`  ✗ the dataset record could not be read: ${error instanceof Error ? error.message : String(error)}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    report?.(`  Current edition ${resource.url}`);
+    const existing = await readManifest(manifestPath);
+    const present = await tryStat(csvPath);
+    if (!options.force &&
+        present &&
+        existing &&
+        resource.md5 &&
+        existing.publisher_md5 === resource.md5 &&
+        existing.source_url === resource.url) {
+        report?.(`  Already at md5 ${resource.md5}, so nothing was transferred`);
+        return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
+    }
+    let bytes;
+    try {
+        ;
+        ({ bytes } = await downloadToFile({
+            url: resource.url,
+            dest: csvPath,
+            retries: options.retries,
+            retryDelayMs: options.retryDelayMs,
+            headers: { accept: "text/csv, */*" },
+            report,
+        }));
+    }
+    catch (error) {
+        report?.(`  ✗ download failed: ${error instanceof Error ? error.message : String(error)}`);
+        await removePathIfPresent(csvPath);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    report?.(`  Downloaded ${ByteFormatter.formatIEC(bytes)} (${bytes} bytes)`);
+    // The delivered count against the stated one.
+    // The byte-order mark the file opens with is part of what the portal counts,
+    // so the two agree on a complete transfer.
+    if (resource.filesize !== null && resource.filesize !== bytes) {
+        report?.(`  ✗ the dataset record states ${resource.filesize} bytes and ${bytes} arrived, ` +
+            `so the transfer is not the edition the record describes`);
+        await removePathIfPresent(csvPath);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    const md5 = await md5File(csvPath);
+    if (resource.md5 && resource.md5 !== md5) {
+        report?.(`  ✗ the dataset record states md5 ${resource.md5} and the bytes hash to ${md5}`);
+        await removePathIfPresent(csvPath);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    let columns;
+    try {
+        columns = await readDelimitedHeader(csvPath, LU_BD_ADRESSES_DELIMITER);
+        assertHeaderColumns(columns, BD_ADRESSES_REQUIRED_COLUMNS, `${SLUG}: ${LU_BD_ADRESSES_CSV_FILENAME}`);
+    }
+    catch (error) {
+        report?.(`  ✗ ${error instanceof Error ? error.message : String(error)}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    const manifest = {
+        source_url: resource.url,
+        downloaded_at: new Date().toISOString(),
+        filename: LU_BD_ADRESSES_CSV_FILENAME,
+        sha256: await sha256File(csvPath),
+        bytes,
+        license: BD_ADRESSES_DEFAULT_LICENSE,
+        attribution: LU_BD_ADRESSES_ATTRIBUTION,
+        dataset_url: LU_BD_ADRESSES_DATASET_URL,
+        publisher_md5: resource.md5,
+        publisher_filesize: resource.filesize,
+        last_modified: resource.lastModified,
+        columns,
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)} over ${columns.length} columns  sha256=${manifest.sha256}`);
+    report?.(`  MANIFEST written to ${manifestPath.toString()}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+/**
+ * Download BD-Adresses into `<outRoot>/bd-adresses/`.
+ *
+ * The registry entry point.
+ * `#lu/adapters/bd-adresses/adapter` is pointed at {@linkcode bdAdressesInputPath},
+ * the `addresses.csv` inside that directory, rather than at the directory itself.
+ */
+export async function fetchBDAdresses(options, report) {
+    await using client = new APIClient({ displayName: SLUG, retry: true });
+    // Awaited rather than returned: `await using` disposes the client when this scope exits,
+    // and a disposed `APIClient` refuses every later request.
+    return await downloadBDAdresses(client, {
+        outputDir: options.outRoot(SLUG),
+        force: options.force,
+        retries: options.retries,
+        retryDelayMs: options.retryDelayMs,
+        signal: options.signal,
+        report,
+    });
+}
+/**
+ * The file `#lu/adapters/bd-adresses/adapter` reads, under a fetch run's `outRoot`.
+ */
+export function bdAdressesInputPath(outRoot) {
+    return outRoot(SLUG, LU_BD_ADRESSES_CSV_FILENAME);
+}
+//# sourceMappingURL=bd-adresses.js.map

package/out/lu/tools/fetch/bd-adresses.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bd-adresses.js","sourceRoot":"","sources":["../../../../lib/lu/tools/fetch/bd-adresses.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D,OAAO,EACN,sBAAsB,EACtB,2BAA2B,EAC3B,4BAA4B,GAC5B,MAAM,kCAAkC,CAAA;AAEzC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AACnF,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AAE9E;;GAEG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,4EAA4E,CAAA;AAEtH;;GAEG;AACH,MAAM,IAAI,GAAG,sBAAsB,CAAA;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,eAAe,CAAA;AAE1D;;GAEG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,iDAAiD,CAAA;AAE3F;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAA;AA0D3C;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAqB;IAC3D,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;IAEvG,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEvF,MAAM,IAAI,KAAK,CACd,GAAG,0BAA0B,yDAAyD,OAAO,IAAI,QAAQ,GAAG,CAC5G,CAAA;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAA;IAElC,OAAO;QACN,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,IAAI;QACnC,2EAA2E;QAC3E,0DAA0D;QAC1D,GAAG,EAAE,QAAQ,EAAE,IAAI,KAAK,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QACvE,YAAY,EAAE,QAAQ,CAAC,aAAa,IAAI,IAAI;KAC5C,CAAA;AACF,CAAC;AAcD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,MAAgC,EAChC,OAAkC;IAElC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAC1B,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,OAAO,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAA;IACpD,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IAE7C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,2BAA2B,EAAE,CAAC,CAAA;IAExD,IAAI,QAA4B,CAAA;IAEhC,IAAI,CAAC;QACJ,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAgB;YAClD,MAAM,EAAE,KAAK;YACb,GAAG,EAAE,0BAA0B;YAC/B,YAAY,EAAE,MAAM;SACpB,CAAC,CAAA;QAEF,QAAQ,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,6CAA6C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAE/G,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,qBAAqB,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAA;IAE7C,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAqB,YAAY,CAAC,CAAA;IACrE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;IAEtC,IACC,CAAC,OAAO,CAAC,KAAK;QACd,OAAO;QACP,QAAQ;QACR,QAAQ,CAAC,GAAG;QACZ,QAAQ,CAAC,aAAa,KAAK,QAAQ,CAAC,GAAG;QACvC,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,GAAG,EACnC,CAAC;QACF,MAAM,EAAE,CAAC,oBAAoB,QAAQ,CAAC,GAAG,8BAA8B,CAAC,CAAA;QAExE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IAC9D,CAAC;IAED,IAAI,KAAa,CAAA;IAEjB,IAAI,CAAC;QACJ,CAAC;QAAA,CAAC,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;YAClC,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,OAAO,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;YACpC,MAAM;SACN,CAAC,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,wBAAwB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1F,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;QAElC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,gBAAgB,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAA;IAE3E,8CAA8C;IAC9C,6EAA6E;IAC7E,2CAA2C;IAC3C,IAAI,QAAQ,CAAC,QAAQ,KAAK,IAAI,IAAI,QAAQ,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;QAC/D,MAAM,EAAE,CACP,iCAAiC,QAAQ,CAAC,QAAQ,cAAc,KAAK,YAAY;YAChF,yDAAyD,CAC1D,CAAA;QAED,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;QAElC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;IAElC,IAAI,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC;QAC1C,MAAM,EAAE,CAAC,qCAAqC,QAAQ,CAAC,GAAG,0BAA0B,GAAG,EAAE,CAAC,CAAA;QAC1F,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;QAElC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,IAAI,OAA0B,CAAA;IAE9B,IAAI,CAAC;QACJ,OAAO,GAAG,MAAM,mBAAmB,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;QAEtE,mBAAmB,CAAC,OAAO,EAAE,4BAA4B,EAAE,GAAG,IAAI,KAAK,2BAA2B,EAAE,CAAC,CAAA;IACtG,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAEzE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,QAAQ,GAAuB;QACpC,UAAU,EAAE,QAAQ,CAAC,GAAG;QACxB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,2BAA2B;QACrC,MAAM,EAAE,MAAM,UAAU,CAAC,OAAO,CAAC;QACjC,KAAK;QACL,OAAO,EAAE,2BAA2B;QACpC,WAAW,EAAE,0BAA0B;QACvC,WAAW,EAAE,0BAA0B;QACvC,aAAa,EAAE,QAAQ,CAAC,GAAG;QAC3B,kBAAkB,EAAE,QAAQ,CAAC,QAAQ;QACrC,aAAa,EAAE,QAAQ,CAAC,YAAY;QACpC,OAAO;KACP,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,MAAM,oBAAoB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3G,MAAM,EAAE,CAAC,yBAAyB,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAE5D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,OAA+B,EAC/B,MAA+B;IAE/B,YAAY,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtE,yFAAyF;IACzF,0DAA0D;IAC1D,OAAO,MAAM,kBAAkB,CAAC,MAAM,EAAE;QACvC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM;KACN,CAAC,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAoB;IACvD,OAAO,OAAO,CAAC,IAAI,EAAE,2BAA2B,CAAC,CAAA;AAClD,CAAC"}

package/out/lv/adapters/varis/adapter.d.ts

@@ -0,0 +1,284 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `lv-varis`: Latvia's State Address Register (`Valsts adrešu reģistrs`), read from the open data
+ * Valsts zemes dienests publishes as the data.gov.lv dataset `varis-atvertie-dati`.
+ *
+ * Input is the directory `fetchVaris` writes: six of the dataset's CSV tables, each under the file
+ * name the portal serves it as, `aw_eka.csv` and the five tables its parent codes point into. Every table is
+ * UTF-8 with a byte-order mark, comma-separated, and quotes every field, doubling a quote inside a
+ * value. The column metadata the publisher posts beside each table declares `ISO-8859-1`; the bytes
+ * are UTF-8, and the reader decodes them as UTF-8. Over the 2026-09-30 edition no value in
+ * `aw_eka.csv` holds a line break, 0 of 610,876 records.
+ *
+ * ## One row per building or land-unit address
+ *
+ * `aw_eka.csv` holds the addresses of buildings and of land units intended for building, one record
+ * per address. Of its 610,876 records, `STATUSS` is `EKS` (existing) on 550,652, `DEL` (removed) on
+ * 32,448 and `ERR` (erroneous) on 27,776. Only `EKS` records become rows. The two other states are
+ * counted under {@linkcode VarisRefusal}.
+ *
+ * Each record names its immediate parent by `VKUR_CD` and `VKUR_TIPS`, and the parent names its own
+ * the same way, up to the country. The type codes are the register's: `104` city (`pilsēta`), `105`
+ * parish (`pagasts`), `106` village (`ciems`), `107` street (`iela`), `113` municipality (`novads`)
+ * and `101` Latvia. No table read lists its code `100000000` as a row. Over the 550,652 active records the
+ * chains take seven shapes, written from the parent upward. The rows column counts what the adapter
+ * emits after the refusals below:
+ *
+ * | Chain                         | Records |    Rows | {@linkcode VarisAddressType}  |
+ * | ----------------------------- | ------: | ------: | ----------------------------- |
+ * | parish, municipality          | 116,225 | 116,225 | `ParishNamedHouse`          |
+ * | street, city, municipality    | 113,906 | 113,905 | `TownStreet`                 |
+ * | street, city                  | 111,218 | 111,217 | `StateCityStreet`           |
+ * | village, parish, municipality | 108,164 | 108,164 | `VillageNamedHouse`         |
+ * | street, village, parish, mun. |  98,192 |  98,179 | `VillageStreet`              |
+ * | city, municipality            |   2,814 |   2,814 | `CityNamedHouse`            |
+ * | city                          |     133 |     133 | `CityNamedHouse`            |
+ *
+ * No active record has an unresolved or inactive ancestor, 0 of 550,652.
+ *
+ * ## The written form is the publisher's
+ *
+ * `STD` holds the register's own full address string: `Krišjāņa Barona iela 1, Rīga, LV-1050` or
+ * `"Riņņi", Vecates pag., Valmieras nov., LV-4211`. Each
+ * part is the `NOSAUKUMS` of one object in the chain, in chain order, joined by a comma and a space,
+ * with the postcode `ATRIB` last. A building on a street writes the street name and the house number
+ * as one part, and a building with its own name writes it inside ASCII double quotes. The adapter composes
+ * that string from the chain and compares it with `STD`. Over the 550,637 active records the adapter
+ * does not refuse first, the two are equal on 550,637. A record where they differ is refused, so every
+ * row's `raw` is `STD` unchanged and every component's position in it is known. The row's surface is
+ * `attested`. `alignRow` aligns the 550,637 rows at edit distance 0, quarantines none, and places
+ * every component in the order above.
+ *
+ * The comparison reads values as stored. 25 building names end in a space, and `STD` keeps it inside the
+ * quotes, `"Vilidža "`, so the reader turns off the CSV reader's trimming, composes from the stored
+ * values and trims only the labels.
+ *
+ * The codex renderer cannot produce this order. libaddressinput's Latvian layout prints the region
+ * line before the locality, `Siguldas nov., Sigulda, LV-2150`, where the register writes `Sigulda,
+ * Siguldas nov., LV-2150`, so `formatAddressRow` is not called.
+ *
+ * ## Which tag each part takes
+ *
+ * - The street's `NOSAUKUMS` is `street`, type word included: `Brīvības iela`, `Vidusceļš`. No Latvian
+ *   street-type split exists in this repository, and the register stores the name whole.
+ * - The house number is `house_number`, its building-block suffix included: `195 k-1` is house 195,
+ *   block (`korpuss`) 1, one address in the register. Over the 323,316 street addresses, 249,783 are
+ *   bare digits and 56,374 digits and a letter. 15 hold no digit, such as `Bauskas iela Ozoli`, where
+ *   a house name stands in the number's place. Those are refused, because the name is not a number
+ *   and labeling it a venue would place a venue after a street. No other Latvian row does that.
+ * - A building's name is `venue`. SCHEMA.mdx defines `venue` as a place with a name and keeps
+ *   `building_name` for Japan, and the Estonian register adapter labels a farm name `venue` for the same
+ *   reason. The quotes are punctuation and lie outside the span. 236 names hold a quote of their own,
+ *   such as `Stacija "Biksti"`, which the register writes `"Stacija "Biksti""`.
+ * - A city and a parish are `locality`. Both are direct parts of a municipality, so they are the same
+ *   tier, and a rural address without a village still writes the parish it lies in. The
+ *   parish is written with the register's abbreviation, `Vecates pag.`, and the span includes it, as a
+ *   `subregion` of `Multnomah County` includes `County`.
+ * - A village is `dependent_locality`. It lies inside a parish, and the Estonian adapter labels a
+ *   village inside a rural municipality the same way.
+ * - A municipality is `region`, abbreviation included: `Valmieras nov.`. The register holds no tier
+ *   above it: the parent of all 35 active municipalities is Latvia. The 7 active cities whose parent is
+ *   Latvia, the state cities such as Rīga and Daugavpils, write no municipality, so their rows have no
+ *   region.
+ * - `ATRIB` is `postcode`, prefix included: `LV-4211`.
+ *
+ * ## Premises and apartments
+ *
+ * The register's premises table, `aw_dziv.csv`, holds 868,669 active apartment and premises addresses
+ * in 57,285 buildings, more than the 550,652 building addresses. A tenth of those buildings hold 50.1%
+ * of the apartments, and one building holds 681. Rows for every apartment would make a few large blocks
+ * the commonest Latvian address, so the adapter reads no premises.
+ *
+ * ## Personal data, identity and license
+ *
+ * The register records places and names no owner or occupant, so no row has a natural person's
+ * name. A building name is the name of the property, as on a map.
+ *
+ * The row id is the register's own object code, `lv-varis-<KODS>`. Five `STD` strings are each written
+ * by two codes, so two rows can share a `raw` while their ids differ.
+ *
+ * data.gov.lv publishes the dataset under `CC-BY-4.0`, the `license_id` its CKAN record names for the
+ * publisher Valsts zemes dienests, and the column metadata posted beside `aw_dziv.csv` names
+ * `https://creativecommons.org/licenses/by/4.0/` as `dc:license`. CC BY 4.0 requires credit, a link to the license and
+ * a statement that changes were made, so every row records the license and
+ * {@linkcode LV_VARIS_ATTRIBUTION} is the credit the model card states.
+ *
+ * The adapter honors `opts.limit` and `opts.signal`. `opts.country` is optional and accepts only `LV`.
+ */
+import type { PathBuilderLike } from "path-ts";
+import { type CanonicalRow, type CorpusAdapter } from "#types";
+/**
+ * Registry id for this adapter, stamped into every row it emits.
+ */
+export declare const LV_VARIS_ADAPTER_ID = "lv-varis";
+/**
+ * The one jurisdiction this adapter emits.
+ */
+export declare const LV_VARIS_COUNTRIES: readonly string[];
+/**
+ * The license data.gov.lv names for the dataset.
+ */
+export declare const LV_VARIS_LICENSE = "CC-BY-4.0";
+/**
+ * The credit CC BY 4.0 obliges.
+ */
+export declare const LV_VARIS_ATTRIBUTION = "Valsts zemes dienests, Valsts adre\u0161u re\u0123istra atv\u0113rtie dati, via data.gov.lv";
+/**
+ * The register tables the adapter reads, by the file name the portal serves each under.
+ */
+export declare const VARIS_TABLES: {
+    readonly Building: "aw_eka.csv";
+    readonly Street: "aw_iela.csv";
+    readonly Village: "aw_ciems.csv";
+    readonly Parish: "aw_pagasts.csv";
+    readonly Municipality: "aw_novads.csv";
+    readonly City: "aw_pilseta.csv";
+};
+export type VarisTable = (typeof VARIS_TABLES)[keyof typeof VARIS_TABLES];
+/**
+ * The register's addressing-object type codes, `TIPS_CD` and `VKUR_TIPS`.
+ */
+export declare const VarisObjectType: {
+    readonly Country: "101";
+    readonly City: "104";
+    readonly Parish: "105";
+    readonly Village: "106";
+    readonly Street: "107";
+    readonly Building: "108";
+    readonly Municipality: "113";
+};
+export type VarisObjectType = (typeof VarisObjectType)[keyof typeof VarisObjectType];
+/**
+ * Why a building record did not become a row.
+ */
+export declare const VarisRefusal: {
+    /**
+     * `STATUSS` is `DEL`: the address was removed.
+     */
+    readonly Removed: "row:status-removed";
+    /**
+     * `STATUSS` is `ERR`: the register marks the address erroneous.
+     */
+    readonly Erroneous: "row:status-erroneous";
+    /**
+     * `STATUSS` holds a value the register's documentation does not list.
+     */
+    readonly UnknownStatus: "row:status-unknown";
+    /**
+     * A code in the parent chain names no object in the tables read.
+     */
+    readonly ParentUnresolved: "row:parent-unresolved";
+    /**
+     * An object in the parent chain is not active.
+     */
+    readonly AncestorInactive: "row:ancestor-inactive";
+    /**
+     * The chain holds an object type in a place no address shape allows.
+     */
+    readonly UnexpectedHierarchy: "row:unexpected-hierarchy";
+    /**
+     * The record has no `LV-` postcode.
+     */
+    readonly PostcodeAbsent: "row:postcode-absent";
+    /**
+     * A building on a street has a name with no digit where its number belongs.
+     */
+    readonly NamedHouseOnStreet: "row:named-house-on-street";
+    /**
+     * The string composed from the chain differs from the register's `STD`.
+     */
+    readonly StandardFormMismatch: "row:standard-form-mismatch";
+};
+export type VarisRefusal = (typeof VarisRefusal)[keyof typeof VarisRefusal];
+/**
+ * The shape of an address, read from its parent chain.
+ */
+export declare const VarisAddressType: {
+    /**
+     * A street address in one of the state cities.
+     * Those cities lie under no municipality.
+     */
+    readonly StateCityStreet: "state-city-street";
+    /**
+     * A street address in a town of a municipality.
+     */
+    readonly TownStreet: "town-street";
+    /**
+     * A street address in a village.
+     */
+    readonly VillageStreet: "village-street";
+    /**
+     * A house with its own name in a city, on no street.
+     */
+    readonly CityNamedHouse: "city-named-house";
+    /**
+     * A house with its own name in a village.
+     */
+    readonly VillageNamedHouse: "village-named-house";
+    /**
+     * A house with its own name in a parish, in no village.
+     */
+    readonly ParishNamedHouse: "parish-named-house";
+};
+export type VarisAddressType = (typeof VarisAddressType)[keyof typeof VarisAddressType];
+/**
+ * The columns read from every table.
+ */
+export interface VarisRecord {
+    KODS: string;
+    TIPS_CD: string;
+    STATUSS: string;
+    VKUR_CD: string;
+    VKUR_TIPS: string;
+    NOSAUKUMS: string;
+    ATRIB: string;
+    STD: string;
+}
+/**
+ * One object a building's parent chain passes through.
+ */
+export interface VarisPlace {
+    type: string;
+    name: string;
+    parentCode: string;
+    active: boolean;
+}
+/**
+ * The places of the five parent tables, keyed by object code.
+ */
+export type VarisPlaces = ReadonlyMap<string, VarisPlace>;
+export type VarisReading = {
+    readonly admitted: CanonicalRow;
+    readonly addressType: VarisAddressType;
+} | {
+    readonly refused: VarisRefusal;
+};
+/**
+ * The object code the register gives Latvia, the parent of every municipality and state city.
+ * No table read holds it as a row.
+ */
+export declare const VARIS_COUNTRY_CODE = "100000000";
+/**
+ * Reads one building record into a row or the reason it is refused.
+ */
+export declare function readVarisBuilding(record: VarisRecord, places: VarisPlaces): VarisReading;
+/**
+ * Streams one table's records, each value as stored.
+ *
+ * The reader turns trimming off because `STD` keeps a stored trailing space inside a building name's quotes.
+ * The composition check compares the two.
+ */
+export declare function readVarisTable(path: PathBuilderLike): AsyncIterable<VarisRecord>;
+/**
+ * Reads the five parent tables into one map keyed by object code.
+ */
+export declare function readVarisPlaces(directory: PathBuilderLike): Promise<Map<string, VarisPlace>>;
+export declare function createVarisAdapter(): CorpusAdapter;
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export declare const varisAdapter: CorpusAdapter;
+//# sourceMappingURL=adapter.d.ts.map

package/out/lv/adapters/varis/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../lib/lv/adapters/varis/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6GG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAM9C,OAAO,EAGN,KAAK,YAAY,EACjB,KAAK,aAAa,EAGlB,MAAM,QAAQ,CAAA;AAEf;;GAEG;AACH,eAAO,MAAM,mBAAmB,aAAa,CAAA;AAE7C;;GAEG;AACH,eAAO,MAAM,kBAAkB,EAAE,SAAS,MAAM,EAAW,CAAA;AAE3D;;GAEG;AACH,eAAO,MAAM,gBAAgB,cAAc,CAAA;AAE3C;;GAEG;AACH,eAAO,MAAM,oBAAoB,gGAAiF,CAAA;AAElH;;GAEG;AACH,eAAO,MAAM,YAAY;aACxB,QAAQ,EAAE,YAAY;aACtB,MAAM,EAAE,aAAa;aACrB,OAAO,EAAE,cAAc;aACvB,MAAM,EAAE,gBAAgB;aACxB,YAAY,EAAE,eAAe;aAC7B,IAAI,EAAE,gBAAgB;CACb,CAAA;AAEV,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,OAAO,YAAY,CAAC,CAAA;AAEzE;;GAEG;AACH,eAAO,MAAM,eAAe;aAC3B,OAAO,EAAE,KAAK;aACd,IAAI,EAAE,KAAK;aACX,MAAM,EAAE,KAAK;aACb,OAAO,EAAE,KAAK;aACd,MAAM,EAAE,KAAK;aACb,QAAQ,EAAE,KAAK;aACf,YAAY,EAAE,KAAK;CACV,CAAA;AAEV,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,OAAO,eAAe,CAAC,CAAA;AAOpF;;GAEG;AACH,eAAO,MAAM,YAAY;IACxB;;OAEG;aACH,OAAO,EAAE,oBAAoB;IAC7B;;OAEG;aACH,SAAS,EAAE,sBAAsB;IACjC;;OAEG;aACH,aAAa,EAAE,oBAAoB;IACnC;;OAEG;aACH,gBAAgB,EAAE,uBAAuB;IACzC;;OAEG;aACH,gBAAgB,EAAE,uBAAuB;IACzC;;OAEG;aACH,mBAAmB,EAAE,0BAA0B;IAC/C;;OAEG;aACH,cAAc,EAAE,qBAAqB;IACrC;;OAEG;aACH,kBAAkB,EAAE,2BAA2B;IAC/C;;OAEG;aACH,oBAAoB,EAAE,4BAA4B;CACzC,CAAA;AAEV,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,OAAO,YAAY,CAAC,CAAA;AAE3E;;GAEG;AACH,eAAO,MAAM,gBAAgB;IAC5B;;;OAGG;aACH,eAAe,EAAE,mBAAmB;IACpC;;OAEG;aACH,UAAU,EAAE,aAAa;IACzB;;OAEG;aACH,aAAa,EAAE,gBAAgB;IAC/B;;OAEG;aACH,cAAc,EAAE,kBAAkB;IAClC;;OAEG;aACH,iBAAiB,EAAE,qBAAqB;IACxC;;OAEG;aACH,gBAAgB,EAAE,oBAAoB;CAC7B,CAAA;AAEV,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,OAAO,gBAAgB,CAAC,CAAA;AAEvF;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACX;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,OAAO,CAAA;CACf;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;AAEzD,MAAM,MAAM,YAAY,GACrB;IAAE,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAA;CAAE,GAC3E;IAAE,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAA;CAAE,CAAA;AASrC;;;GAGG;AACH,eAAO,MAAM,kBAAkB,cAAc,CAAA;AAiC7C;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,GAAG,YAAY,CA+ExF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,eAAe,GAAG,aAAa,CAAC,WAAW,CAAC,CAEhF;AAED;;GAEG;AACH,wBAAsB,eAAe,CAAC,SAAS,EAAE,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAsBlG;AAED,wBAAgB,kBAAkB,IAAI,aAAa,CAsClD;AAED;;GAEG;AACH,eAAO,MAAM,YAAY,eAAuB,CAAA"}