@mailwoman/corpus 9.4.0 → 10.1.0

package/out/cz/tools/fetch/cuzk.d.ts

@@ -0,0 +1,272 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Harvest ČÚZK's INSPIRE Addresses archives into the directory `#cz/adapters/cuzk/adapter` reads.
+ *
+ * ČÚZK serves the Addresses (AD) theme as a two-level ATOM service rather than as a WFS. The
+ * service document at `https://atom.cuzk.gov.cz/AD/AD.xml` holds one entry per municipality, each
+ * pointing at a dataset feed, and each dataset feed offers that municipality's addresses as a
+ * zipped GML in two projections. Measured 2026-10-02: the service document answered HTTP 200 with
+ * 8,373,817 bytes holding 6,258 entries, sha256
+ * `9b1d51a721ab601fa74307cce5ce20f74d12dd29512f87be29cd1072f0a2d83b`.
+ *
+ * The acquisition belongs under a `tools/` root. `@mailwoman/corpus` is one of the
+ * `TOOLING_PACKAGES` in `dependency-cruiser.config.mjs`, which keep their tooling under `lib/`, so
+ * this module sits beside `#fr/tools/fetch/ban` rather than in an `sdk/` root the workspace does
+ * not declare.
+ *
+ * Every one of the service document's `rights` elements reads `žádné podmínky neplatí`, INSPIRE's
+ * controlled value for no conditions applying to access and use, and the address-source register
+ * elects it on that basis. `#cz/adapters/cuzk/adapter` records the same value on every row.
+ *
+ * Four properties of this service decide the harvest's shape, and each was measured rather than
+ * assumed:
+ *
+ * 1. **The service document carries each municipality's file modification time.** An entry's
+ *    `<updated>` equals the archive's `Last-Modified` to the second, checked on six municipalities
+ *    on 2026-10-02 — `584061` reads `2026-06-04T02:19:33+02:00` against
+ *    `Thu, 04 Jun 2026 00:19:33 GMT`, and `531529`, `551481`, `545031`, `587044` and `584282`
+ *    agree the same way. So one 8.4 MB request states the freshness of all 6,258 archives, and a
+ *    re-run requests only the municipalities whose stated time moved. The alternative — a
+ *    conditional request per municipality — costs 6,258 round trips to learn the same thing.
+ * 2. **The download URL is composed from a base the feed itself declares.** The service document's
+ *    own `rel="next"` links are `https://services.cuzk.gov.cz/gml/inspire/ad/epsg-4258` and
+ *    `…/epsg-5514`, and a dataset feed's `alternate` link for the same municipality is that base
+ *    plus `/<code>.zip`. Composing from the declared base rather than reading 6,258 dataset feeds
+ *    halves the harvest's requests. {@linkcode HarvestCzCuzkOptions.resolveThroughDatasetFeed}
+ *    reads the publisher's own href instead, for a run that will not compose a URL.
+ * 3. **The `type` attribute describes the data rather than the file.** A dataset feed's link
+ *    advertises `application/gml+xml` while the host serves `application/zip`, so the GML is inside
+ *    the archive and the advertised type cannot decide what to write. The archive is stored as it
+ *    arrives: the adapter opens it with `adm-zip`, because at least one archive's central directory
+ *    records its member's size as the ZIP64 sentinel `0xFFFFFFFF`.
+ * 4. **A dataset feed's `length` is accurate here, and is still not used.** `584061.zip` claims
+ *    24,846 bytes and the host's `content-length` says 24,846. Denmark's feed understates its file
+ *    by 23 times, so every byte count this module records is counted off the delivered body.
+ *
+ * The archives are small — a systematic sample of 50 of the 6,258 municipalities on 2026-10-02 ran
+ * from 5,236 to 1,642,586 bytes with a median of 16,591 — so the harvest is bound by round trips
+ * rather than by bandwidth, and it is dispatched serially through one `APIClient`. ČÚZK publishes
+ * no rate limit. {@linkcode CZ_CUZK_REQUEST_INTERVAL_MS} holds the harvest to four requests per
+ * second against one government host, which is under half of the ten per second the same 50
+ * requests measured, and a caller that has asked ČÚZK for more raises it.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { type PathBuilderLike } from "path-ts";
+import type { AtomFeed } from "#tools/fetch/atom";
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download";
+/**
+ * The ATOM service document listing one dataset feed per municipality.
+ */
+export declare const CZ_CUZK_SERVICE_FEED_URL = "https://atom.cuzk.gov.cz/AD/AD.xml";
+/**
+ * The attribution the model card must carry for this source.
+ */
+export declare const CZ_CUZK_ATTRIBUTION = "\u010Cesk\u00FD \u00FA\u0159ad zem\u011Bm\u011B\u0159ick\u00FD a katastr\u00E1ln\u00ED (\u010C\u00DAZK), INSPIRE t\u00E9ma Adresy";
+/**
+ * The two projections the service offers, keyed by the path segment the feed's own base URL ends in.
+ *
+ * The adapter reads either.
+ * `epsg-4258` is ETRS89, in degrees, and is the projection the adapter's
+ * component joins were verified against.
+ */
+export declare const CZ_CUZK_PROJECTIONS: {
+    readonly "epsg-4258": "ETRS89";
+    readonly "epsg-5514": "S-JTSK";
+};
+/**
+ * One of {@linkcode CZ_CUZK_PROJECTIONS}.
+ */
+export type CzCuzkProjection = keyof typeof CZ_CUZK_PROJECTIONS;
+/**
+ * The minimum spacing between two requests to `services.cuzk.gov.cz`, in milliseconds.
+ *
+ * Four requests per second.
+ * A systematic sample of 50 archives on 2026-10-02 answered a serial `HEAD` in 100 ms each,
+ * so this leaves the host more than half the rate it demonstrated.
+ */
+export declare const CZ_CUZK_REQUEST_INTERVAL_MS = 250;
+/**
+ * One municipality's dataset, as the service document states it.
+ */
+export interface CzMunicipalityDataset {
+    /**
+     * The municipality code, which is both the `obec` code and the archive's file name stem.
+     */
+    code: string;
+    /**
+     * The entry's title, `INSPIRE - adresní místa - obec: Unkovice [584061]`.
+     */
+    title: string;
+    /**
+     * The `<updated>` the service document states, which is this archive's modification time.
+     */
+    updated: string;
+    /**
+     * The dataset feed for this municipality, which lists the same data in both projections.
+     */
+    datasetFeedURL: string;
+    /**
+     * The archive, composed from the base the feed declares for the requested projection.
+     */
+    downloadURL: string;
+    /**
+     * The file name the archive is written under, which is what the adapter globs.
+     */
+    filename: string;
+}
+/**
+ * The municipality code of one entry, read from its INSPIRE identifier and checked against its title.
+ *
+ * Both spellings are read because the code is the file name the archive is composed
+ * and stored under, so a disagreement between the publisher's two statements of it
+ * would silently store one municipality's addresses under another's name.
+ *
+ * @throws When either statement is missing or the two disagree, which reads as a change
+ * in the publisher's layout rather than as a municipality without a code.
+ */
+export declare function municipalityCodeOf(entry: {
+    title: string;
+    identifierCode: string | null;
+}): string;
+/**
+ * The base URL the feed declares for one projection.
+ *
+ * Matched on the href's last path segment rather than on the link's `title`,
+ * because the segment is the EPSG code itself while the title is a Czech-language label.
+ *
+ * @throws When the feed declares no base for the requested projection, naming the ones it does
+ * declare, so a publisher that moves its files reports itself instead of composing a dead URL.
+ */
+export declare function projectionBaseURL(feed: AtomFeed, projection: CzCuzkProjection): string;
+/**
+ * The municipalities the service document lists, with the archive URL for one projection.
+ *
+ * @throws When the feed lists no entry, when two entries claim one municipality code — which
+ * would make one archive overwrite the other — or when an entry carries no dataset feed link.
+ */
+export declare function readCzCuzkServiceFeed(feed: AtomFeed, projection: CzCuzkProjection): readonly CzMunicipalityDataset[];
+/**
+ * The archive link a dataset feed states for one projection.
+ *
+ * Read only when {@linkcode HarvestCzCuzkOptions.resolveThroughDatasetFeed} is set.
+ * The feed holds one entry per projection, each with one `alternate` link, and the projection is
+ * identified by the href's own path segment for the same reason {@linkcode projectionBaseURL} uses it.
+ *
+ * @throws When the dataset feed states no link for the requested projection.
+ */
+export declare function datasetFeedArchiveURL(feed: AtomFeed, projection: CzCuzkProjection, context: string): string;
+/**
+ * One archive, as the harvest's manifest records it.
+ *
+ * {@linkcode SourceManifest}'s five fields plus what a re-run needs: the modification time
+ * the service document stated for this municipality, and the one the host served it under.
+ */
+export interface CzCuzkArchiveEntry extends SourceManifest {
+    municipality_code: string;
+    /**
+     * The `<updated>` the service document stated when this archive was fetched.
+     *
+     * A later run that reads the same value for this municipality leaves the file alone.
+     * This is the field that makes the harvest resumable without a request per municipality.
+     */
+    feed_updated: string;
+    /**
+     * The `Last-Modified` the host served the archive under, or `null` where it served none.
+     *
+     * Recorded rather than compared: {@linkcode CzCuzkArchiveEntry.feed_updated}
+     * answers the same question without a request.
+     */
+    last_modified: string | null;
+}
+/**
+ * The harvest's `MANIFEST.json`.
+ */
+export interface CzCuzkHarvestManifest extends SourceCollectionManifest {
+    /**
+     * The projection the archives in this directory are in.
+     *
+     * Recorded because the two projections share a file name, so a directory holding
+     * both would be read by the adapter as two copies of every address.
+     */
+    projection: CzCuzkProjection;
+    /**
+     * How many municipalities the service document listed when this manifest was written.
+     *
+     * The denominator for `files`, read from the feed on each run rather than fixed here.
+     */
+    municipalities_listed: number;
+    files: CzCuzkArchiveEntry[];
+}
+export interface HarvestCzCuzkOptions {
+    /**
+     * Where the archives and the manifest are written, which is the adapter's `inputPath`.
+     */
+    outputDir: PathBuilderLike;
+    /**
+     * Which projection to harvest.
+     * Defaults to `epsg-4258`.
+     */
+    projection?: CzCuzkProjection;
+    /**
+     * Harvest only these municipality codes, for a probe or for a repair of named municipalities.
+     *
+     * A code the service document does not list is reported as a failure rather than ignored.
+     */
+    municipalities?: readonly string[];
+    /**
+     * Stop after this many municipalities, counting the ones already on disk.
+     *
+     * A bounded run rather than all 6,258.
+     */
+    limit?: number;
+    /**
+     * Take each archive's URL from its own dataset feed instead of composing it
+     * from the base the service document declares.
+     *
+     * Costs one extra request per municipality and asks the publisher rather than composing.
+     */
+    resolveThroughDatasetFeed?: boolean;
+    /**
+     * Re-read the sha256 of every archive already on disk instead of comparing its byte count.
+     *
+     * The default compares the recorded byte count against the file's size, which is one `stat`.
+     * This re-hashes, which reads every archive in the directory.
+     */
+    verifyDigests?: boolean;
+    signal?: AbortSignal;
+    report?: (line: string) => void;
+}
+/**
+ * Harvest the municipalities the service document lists into `options.outputDir`.
+ *
+ * The client is injected so the harvest is testable without the network and so a caller
+ * decides the pacing. {@linkcode fetchCzCuzk} is the registry entry point and supplies both.
+ *
+ * @returns What was fetched, what was already current, and which municipality codes failed.
+ */
+export declare function harvestCzCuzk(client: Pick<APIClient, "fetch">, options: HarvestCzCuzkOptions): Promise<FetchSummary>;
+export interface FetchCzCuzkOptions extends BaseFetchOptions, Pick<HarvestCzCuzkOptions, "projection" | "municipalities" | "limit" | "resolveThroughDatasetFeed" | "verifyDigests" | "signal"> {
+    /**
+     * The minimum spacing between two requests, in milliseconds.
+     *
+     * Defaults to {@linkcode CZ_CUZK_REQUEST_INTERVAL_MS}.
+     */
+    minRequestIntervalMs?: number;
+}
+/**
+ * Harvest ČÚZK's INSPIRE Addresses archives into `<outRoot>/cz-cuzk/`.
+ *
+ * Re-runnable: a municipality whose recorded modification time still matches the service
+ * document's and whose archive is still on disk at the recorded length costs no request.
+ *
+ * A full harvest is 6,259 requests — the service document plus one per municipality —
+ * and an estimated 396 MB, from a mean of 63,337 bytes over a systematic sample
+ * of 50 of the 6,258 archives on 2026-10-02.
+ * The sample ran from 5,236 to 1,642,586 bytes, so the total is an estimate from
+ * 50 measurements rather than a measurement of all 6,258.
+ */
+export declare function fetchCzCuzk(options: FetchCzCuzkOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=cuzk.d.ts.map

package/out/cz/tools/fetch/cuzk.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cuzk.d.ts","sourceRoot":"","sources":["../../../../lib/cz/tools/fetch/cuzk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AAKH,OAAO,EAAE,SAAS,EAA+B,MAAM,qBAAqB,CAAA;AAM5E,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAG3D,OAAO,KAAK,EAAE,QAAQ,EAAY,MAAM,mBAAmB,CAAA;AAE3D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAGrH;;GAEG;AACH,eAAO,MAAM,wBAAwB,uCAAuC,CAAA;AAO5E;;GAEG;AACH,eAAO,MAAM,mBAAmB,sIAAqE,CAAA;AAErG;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB;aAC/B,WAAW,EAAE,QAAQ;aACrB,WAAW,EAAE,QAAQ;CACZ,CAAA;AAEV;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,MAAM,OAAO,mBAAmB,CAAA;AAE/D;;;;;;GAMG;AACH,eAAO,MAAM,2BAA2B,MAAM,CAAA;AAqB9C;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,cAAc,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AAaD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,MAAM,CA0BlG;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,GAAG,MAAM,CAYtF;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,GAAG,SAAS,qBAAqB,EAAE,CA+CpH;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAW3G;AAED;;;;;GAKG;AACH,MAAM,WAAW,kBAAmB,SAAQ,cAAc;IACzD,iBAAiB,EAAE,MAAM,CAAA;IACzB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB;;;;;OAKG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,wBAAwB;IACtE;;;;;OAKG;IACH,UAAU,EAAE,gBAAgB,CAAA;IAC5B;;;;OAIG;IACH,qBAAqB,EAAE,MAAM,CAAA;IAC7B,KAAK,EAAE,kBAAkB,EAAE,CAAA;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,SAAS,EAAE,eAAe,CAAA;IAC1B;;;OAGG;IACH,UAAU,CAAC,EAAE,gBAAgB,CAAA;IAC7B;;;;OAIG;IACH,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAClC;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;;;OAKG;IACH,yBAAyB,CAAC,EAAE,OAAO,CAAA;IACnC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AA2BD;;;;;;;GAOG;AACH,wBAAsB,aAAa,CAClC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,oBAAoB,GAC3B,OAAO,CAAC,YAAY,CAAC,CA+GvB;AA6GD,MAAM,WAAW,kBAChB,SACC,gBAAgB,EAChB,IAAI,CACH,oBAAoB,EACpB,YAAY,GAAG,gBAAgB,GAAG,OAAO,GAAG,2BAA2B,GAAG,eAAe,GAAG,QAAQ,CACpG;IACF;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAmBrH"}