@mailwoman/corpus 10.0.0 → 10.1.0

package/out/es/tools/fetch/bizkaia.d.ts

@@ -0,0 +1,278 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Harvests the INSPIRE Addresses theme the Diputación Foral de Bizkaia publishes into the
+ *   directory `#es/adapters/bizkaia/adapter` reads.
+ *
+ *   ## Two services, because the archives carry addresses alone
+ *
+ *   The ATOM download service at `https://apli.bizkaia.eus/apps/Danok/INSPIRE/addresses.xml` lists
+ *   one entry per municipality, each offering that municipality's addresses as a zipped GML through a
+ *   `rel="enclosure"` link. Those archives hold no `ad:ThoroughfareName`, `ad:PostalDescriptor` or
+ *   `ad:AdminUnitName` feature: every `ad:component` reference an address writes is a stored-query
+ *   URL on the publisher's WFS, which the ATOM service does not republish. An adapter handed the
+ *   archives alone therefore reads every address in the province without its street, its postcode or
+ *   its municipality, which is why this harvest acquires both services into one directory.
+ *
+ *   Each component type is read in one request, which is the only way this service serves it. Its
+ *   capabilities document states `ImplementsResultPaging` as `FALSE` and `CountDefault` as `1000`, so
+ *   a reader that paged a type would receive the first page again. The harvest therefore asks
+ *   `RESULTTYPE=hits` for the type's count and then asks for that many features in one `GetFeature`,
+ *   and {@linkcode fetchBizkaiaComponentDocument} raises where the document returns fewer than the
+ *   count. A truncated component document is the failure that would otherwise reach the adapter as
+ *   an unresolved reference per address.
+ *
+ *   A component document's own bytes are not reproducible. Its `wfs:FeatureCollection` carries a
+ *   `timeStamp` attribute holding the moment of the request, so two identical harvests differ in
+ *   that element and therefore in their digests. The manifest records each document's byte count and
+ *   sha256 as what arrived, never as a value a later run compares against.
+ *
+ *   ## What decides a skip
+ *
+ *   The service document states one `<updated>`, repeated on all of its entries, rather than a
+ *   modification time per municipality: the 112 entries served on 2026-10-03 all read
+ *   `2026-10-01T02:40:20Z`, which is the feed document's own modification time. One request therefore
+ *   states the freshness of every archive the service offers, as ČÚZK's does, and a re-run requests
+ *   only the municipalities whose archive is missing or no longer the recorded length. Whether that
+ *   value moves whenever an archive is replaced is inferred from one observation rather than
+ *   established: on 2026-10-03 the feed read `2026-10-01T02:40:20Z` and `ES.BFA.AD.001.zip` was
+ *   served with `Last-Modified: Thu, 01 Oct 2026 00:01:15 GMT`. Each archive's own `last-modified` is
+ *   recorded beside its entry so a later run can check that inference against the record.
+ *
+ *   The component documents state no version at all. The WFS answers them under `Cache-Control:
+ *   no-cache` with no `Last-Modified`, so a run holds no value to compare and every run that
+ *   harvests them downloads them again. {@linkcode HarvestESBizkaiaOptions.components} is how a run that wants
+ *   the archives alone declines the 14 MB.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { type PathBuilderLike } from "path-ts";
+import type { AtomFeed } from "#tools/fetch/atom";
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download";
+/**
+ * The ATOM download service listing one archive per municipality.
+ */
+export declare const ES_BIZKAIA_SERVICE_URL = "https://apli.bizkaia.eus/apps/Danok/INSPIRE/addresses.xml";
+/**
+ * The WFS the municipality archives reference their components on.
+ *
+ * Read from the `xlink:href` the archives themselves write, which addresses this
+ * endpoint through the `GetFeatureById` stored query.
+ */
+export declare const ES_BIZKAIA_WFS_URL = "https://geo.bizkaia.eus/arcgisserverinspire/rest/services/Catastro/Annex1/MapServer/exts/InspireFeatureDownload/service";
+/**
+ * The attribution the publisher's own `<rights>` element requires, quoted as it writes it.
+ */
+export declare const ES_BIZKAIA_ATTRIBUTION = "\u00AB\u00A9Bizkaiko Foru Aldundia\u00BB";
+/**
+ * The component feature types the harvest saves from the WFS.
+ *
+ * The four the adapter indexes.
+ * `ad:AddressAreaName` is harvested although the province published none when this was written, because
+ * the schema admits one and a type left unasked is indistinguishable from a type that answered empty.
+ */
+export declare const ES_BIZKAIA_COMPONENT_TYPES: readonly ["ad:ThoroughfareName", "ad:PostalDescriptor", "ad:AdminUnitName", "ad:AddressAreaName"];
+/**
+ * One of {@linkcode ES_BIZKAIA_COMPONENT_TYPES}.
+ */
+export type ESBizkaiaComponentType = (typeof ES_BIZKAIA_COMPONENT_TYPES)[number];
+/**
+ * The minimum spacing between two requests, in milliseconds.
+ *
+ * Four requests per second against one government host.
+ * The publisher states no rate limit.
+ */
+export declare const ES_BIZKAIA_REQUEST_INTERVAL_MS = 250;
+/**
+ * The file name a component type is written under, `ad:ThoroughfareName` → `ad-ThoroughfareName.xml`.
+ *
+ * The colon is replaced because the adapter globs these documents out of a directory
+ * and a prefixed name is awkward to address from a shell.
+ * The `.xml` extension is what puts the document inside the adapter's own glob.
+ */
+export declare function bizkaiaComponentFilename(type: string): string;
+/**
+ * One municipality's archive, as the service document states it.
+ */
+export interface ESBizkaiaMunicipalityDataset {
+    /**
+     * The five-digit municipality code the entry's title states, `48001`.
+     */
+    code: string;
+    /**
+     * The entry's title, `48001-ABADIÑO Addresses`.
+     */
+    title: string;
+    /**
+     * The `<updated>` the entry states.
+     * Every entry of this service repeats the feed's own value.
+     */
+    updated: string;
+    /**
+     * The archive's URL, as the entry's `rel="enclosure"` link writes it.
+     */
+    downloadURL: string;
+    /**
+     * The file name the archive is written under, which is what the adapter globs.
+     */
+    filename: string;
+}
+/**
+ * The municipalities the service document lists.
+ *
+ * The file name comes from the entry's INSPIRE identifier and the municipality code from
+ * its title, and the two are checked against each other, because the file name is what
+ * the archive is stored under: a disagreement between the publisher's two statements of
+ * a municipality would store one municipality's addresses under another's name.
+ *
+ * @throws When the feed lists no entry, when an entry states neither spelling of its municipality,
+ * when the two spellings disagree, when two entries claim one file name — which would make
+ * one archive overwrite the other — or when an entry carries no `rel="enclosure"` link.
+ */
+export declare function readESBizkaiaServiceFeed(feed: AtomFeed): readonly ESBizkaiaMunicipalityDataset[];
+/**
+ * One municipality's archive, as the harvest's manifest records it.
+ */
+export interface ESBizkaiaArchiveEntry extends SourceManifest {
+    municipality_code: string;
+    /**
+     * The `<updated>` the service document stated when this archive was fetched.
+     */
+    feed_updated: string;
+    /**
+     * The `Last-Modified` the host served the archive under, or `null` where it served none.
+     *
+     * Recorded rather than compared.
+     * It is the record against which the service document's single `<updated>` can
+     * later be checked as a per-municipality signal.
+     */
+    last_modified: string | null;
+}
+/**
+ * One saved WFS component document, as the harvest's manifest records it.
+ */
+export interface ESBizkaiaComponentEntry extends SourceManifest {
+    type_name: string;
+    /**
+     * The count the service reported for this type under `RESULTTYPE=hits`.
+     */
+    feature_count: number;
+    /**
+     * The `numberReturned` the saved document states, which the harvest requires to
+     * equal {@linkcode ESBizkaiaComponentEntry.feature_count}.
+     */
+    features_returned: number;
+}
+/**
+ * The harvest's `MANIFEST.json`.
+ */
+export interface ESBizkaiaHarvestManifest extends SourceCollectionManifest {
+    wfs_url: string;
+    /**
+     * The `<updated>` the service document stated on this run.
+     */
+    feed_updated: string | null;
+    /**
+     * How many municipalities the service document listed when this manifest was written.
+     *
+     * The denominator for `files`, read from the feed on each run rather than fixed here.
+     */
+    municipalities_listed: number;
+    files: ESBizkaiaArchiveEntry[];
+    /**
+     * The saved WFS documents, kept apart from `files` because they carry no
+     * municipality and no stated version.
+     */
+    components: ESBizkaiaComponentEntry[];
+}
+export interface HarvestESBizkaiaOptions {
+    /**
+     * Where the archives, the component documents and the manifest are written,
+     * which is the adapter's `inputPath`.
+     */
+    outputDir: PathBuilderLike;
+    /**
+     * Harvest only these five-digit municipality codes, for a probe or for a repair of named ones.
+     *
+     * A code the service document does not list is reported as a failure rather than ignored.
+     */
+    municipalities?: readonly string[];
+    /**
+     * Stop after this many municipalities, counting the ones already on disk.
+     */
+    limit?: number;
+    /**
+     * Save the WFS component documents as well as the archives.
+     * Defaults to true.
+     *
+     * An adapter run needs them: an archive references its street, its postcode
+     * and its municipality and carries none of the three.
+     */
+    components?: boolean;
+    /**
+     * Download every selected archive again, whatever the manifest records.
+     */
+    force?: boolean;
+    /**
+     * Re-read the sha256 of every archive already on disk instead of comparing its byte count.
+     *
+     * The default compares the recorded byte count against the file's size, which is one `stat`.
+     */
+    verifyDigests?: boolean;
+    signal?: AbortSignal;
+    report?: (line: string) => void;
+}
+/**
+ * Reads one component feature type whole and writes it beside the archives.
+ *
+ * Two requests per type: `RESULTTYPE=hits` for the count, then one `GetFeature` for that many features.
+ * `COUNT` is asked one above the stated count, so a service whose hits response
+ * undercounts its own matches answers with more features than it claimed
+ * rather than with exactly the number the harvest asked for.
+ *
+ * @throws When the document's `numberReturned` is missing, unreadable,
+ * or not the count the service reported.
+ * A document holding fewer features than the service matched would reach the adapter as
+ * an unresolved component reference on every address that referenced a missing feature.
+ */
+export declare function fetchBizkaiaComponentDocument(client: Pick<APIClient, "fetch">, typeName: string, options: {
+    dest: PathBuilderLike;
+    signal?: AbortSignal;
+}): Promise<ESBizkaiaComponentEntry>;
+/**
+ * Harvest the municipality archives and the WFS component documents into `options.outputDir`.
+ *
+ * The client is injected so the harvest is testable without the network and so a caller decides
+ * the pacing. {@linkcode fetchESBizkaia} is the registry entry point and supplies both.
+ *
+ * @returns What was fetched, what was already current, and which municipality codes
+ * or component types failed.
+ */
+export declare function harvestESBizkaia(client: Pick<APIClient, "fetch">, options: HarvestESBizkaiaOptions): Promise<FetchSummary>;
+/**
+ * The path `#es/adapters/bizkaia/adapter` reads, given the root a fetch wrote under.
+ *
+ * The directory rather than a file: the adapter reads every archive and every saved WFS
+ * document under it, and it is the component documents that carry the streets.
+ */
+export declare function esBizkaiaInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike;
+/**
+ * Per-invocation options for the registry entry.
+ */
+export interface FetchESBizkaiaOptions extends BaseFetchOptions, Pick<HarvestESBizkaiaOptions, "municipalities" | "limit" | "components" | "force" | "verifyDigests" | "signal"> {
+    /**
+     * The minimum spacing between two requests, in milliseconds.
+     *
+     * Defaults to {@linkcode ES_BIZKAIA_REQUEST_INTERVAL_MS}.
+     */
+    minRequestIntervalMs?: number;
+}
+/**
+ * Harvest Bizkaia's INSPIRE Addresses into `<outRoot>/es-bizkaia/`.
+ *
+ * A full harvest is the service document, one request per municipality it lists,
+ * and two requests per component type.
+ */
+export declare function fetchESBizkaia(options: FetchESBizkaiaOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=bizkaia.d.ts.map

package/out/es/tools/fetch/bizkaia.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bizkaia.d.ts","sourceRoot":"","sources":["../../../../lib/es/tools/fetch/bizkaia.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAKH,OAAO,EAAE,SAAS,EAAoD,MAAM,qBAAqB,CAAA;AAOjG,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAG3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AAEjD,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAIrH;;GAEG;AACH,eAAO,MAAM,sBAAsB,8DAA8D,CAAA;AAEjG;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,4HAC2F,CAAA;AAE1H;;GAEG;AACH,eAAO,MAAM,sBAAsB,6CAA8B,CAAA;AAEjE;;;;;;GAMG;AACH,eAAO,MAAM,0BAA0B,YACtC,qBAAqB,EACrB,qBAAqB,EACrB,kBAAkB,EAClB,oBAAoB,CACX,CAAA;AAEV;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,0BAA0B,CAAC,CAAC,MAAM,CAAC,CAAA;AAEhF;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,MAAM,CAAA;AA+BjD;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,QAAQ,GAAG,SAAS,4BAA4B,EAAE,CAkEhG;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC5D,iBAAiB,EAAE,MAAM,CAAA;IACzB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB;;;;;;OAMG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,cAAc;IAC9D,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;;OAGG;IACH,iBAAiB,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAyB,SAAQ,wBAAwB;IACzE,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B;;;;OAIG;IACH,qBAAqB,EAAE,MAAM,CAAA;IAC7B,KAAK,EAAE,qBAAqB,EAAE,CAAA;IAC9B;;;OAGG;IACH,UAAU,EAAE,uBAAuB,EAAE,CAAA;CACrC;AAED,MAAM,WAAW,uBAAuB;IACvC;;;OAGG;IACH,SAAS,EAAE,eAAe,CAAA;IAC1B;;;;OAIG;IACH,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAClC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IACf;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AA8BD;;;;;;;;;;;;GAYG;AACH,wBAAsB,6BAA6B,CAClD,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,GACtD,OAAO,CAAC,uBAAuB,CAAC,CA8DlC;AAED;;;;;;;;GAQG;AACH,wBAAsB,gBAAgB,CACrC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,uBAAuB,GAC9B,OAAO,CAAC,YAAY,CAAC,CAkJvB;AAsED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC,GAAG,eAAe,CAExF;AAED;;GAEG;AACH,MAAM,WAAW,qBAChB,SACC,gBAAgB,EAChB,IAAI,CAAC,uBAAuB,EAAE,gBAAgB,GAAG,OAAO,GAAG,YAAY,GAAG,OAAO,GAAG,eAAe,GAAG,QAAQ,CAAC;IAChH;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAmBvB"}