@mailwoman/corpus 10.0.0 → 10.1.0

package/out/es/tools/fetch/catastro.d.ts

@@ -0,0 +1,272 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Harvest the INSPIRE Addresses archives of the Dirección General del Catastro into the directory
+ * `#es/adapters/catastro/adapter` reads.
+ *
+ * The service is a two-level ATOM download service. The national service document at
+ * `https://www.catastro.hacienda.gob.es/INSPIRE/Addresses/ES.SDGC.AD.atom.xml` lists one entry per
+ * territorial office, each entry links that office's province feed, and a province feed lists one
+ * zipped GML per municipality:
+ *
+ *     ES.SDGC.AD.atom.xml  →  ES.SDGC.ad.atom_<pp>.xml  →  A.ES.SDGC.AD.<ppmmm>.zip
+ *
+ * Measured 2026-10-03: the service document answered http 200 with 681,783 bytes holding 55
+ * entries, sha256 `4342d37b6d5527eb7b8bebadc97966c90f1a041068ae9398b90183f2176898b9`.
+ *
+ * Five properties of this service decide the harvest's shape.
+ *
+ * 1. **Three of the entries belong to other publishers.** Fifty-two are titled
+ *    `Territorial office <pp> <name>` and are the Dirección General del Catastro's own provinces.
+ *    The other three are titled `Provincial Council of Bizkaia`, `… of Gipuzkoa` and
+ *    `… of Navarra`, which publish their own cadastres through this feed and state their own terms.
+ *    Each of those has its own address-source register row, its own license and its own adapter, so
+ *    this harvest selects the territorial offices by a positive match on their title rather than by
+ *    excluding three names. A feed that renames its offices then matches zero titles and raises.
+ * 2. **A province feed declares ISO-8859-1 and means it, while the national feed declares UTF-8.**
+ *    `ES.SDGC.ad.atom_15.xml` writes A Coruña's directory name with the byte `0xD1` for `Ñ`.
+ *    Decoding that feed as UTF-8 yields U+FFFD, and the URL built from the replacement character
+ *    answers an html error page under http 200 rather than the archive. So both levels are read as
+ *    bytes and decoded through {@linkcode decodeDeclaredXML}, which reads each document's own
+ *    declaration.
+ * 3. **An archive URL cannot be composed, and the href it is read from is not request-ready.** The
+ *    directory segment is the municipality's own name, so Madrid sits under `28900-MADRID` while
+ *    the composable `28079` answers an html page. The publisher writes that segment with literal
+ *    spaces and with non-ASCII letters, and {@linkcode archiveRequestURL} percent-encodes both.
+ * 4. **The municipality code is stated twice, and the two are checked against each other.** An
+ *    entry's title reads `55101-CEUTA addresses` and its archive is named
+ *    `A.ES.SDGC.AD.55101.zip`. The file keeps the publisher's own name rather than one built from
+ *    the code, so the code decides which entry is read and never where the file lands. A
+ *    disagreement between the publisher's two statements of it is a change in the feed's layout
+ *    rather than a municipality without a code.
+ * 5. **Every entry of every feed states one `<updated>`, the edition's publication date.** All 55
+ *    national entries and all 87 entries of `ES.SDGC.ad.atom_02.xml` read
+ *    `2026-08-21T00:00:00Z`, so the value dates the publication rather than the file. It is still
+ *    the freshness signal a re-run compares, because a new edition moves it. A skip requires the
+ *    feed to state a value: where it states none, both sides read an empty string and an equality
+ *    test would keep an archive of unknown age for as long as the feed stayed silent.
+ *
+ * A full harvest is one request for the service document, 52 for the province feeds and one per
+ * municipality. The archives are small enough to buffer: Ceuta is 354,647 bytes, A Coruña 907,574
+ * and Madrid 7,104,627, measured 2026-10-03. The harvest is dispatched serially through one
+ * `APIClient` at {@linkcode ES_CATASTRO_REQUEST_INTERVAL_MS}, and
+ * {@linkcode HarvestESCatastroOptions.provinces} and {@linkcode HarvestESCatastroOptions.limit}
+ * bound a run that is not harvesting the whole country.
+ *
+ * The archives stay compressed. `#es/adapters/catastro/adapter` reads the GML member through
+ * `inspireGMLChunks`, which inflates from the archive, and Ceuta's member is 11,861,975 bytes
+ * against its archive's 354,647.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { type PathBuilderLike } from "path-ts";
+import type { AtomFeed } from "#tools/fetch/atom";
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download";
+/**
+ * The national ATOM service document, which lists one province feed per territorial office.
+ */
+export declare const ES_CATASTRO_SERVICE_FEED_URL = "https://www.catastro.hacienda.gob.es/INSPIRE/Addresses/ES.SDGC.AD.atom.xml";
+/**
+ * The attribution the publisher's own rights statement requires.
+ *
+ * A province feed's `<rights>` reads `This service can be used free of charge in every instance, as
+ * long as that the D. G. of the Cadastre (Ministry of Finance) is mentioned as author and owner of
+ * the information`, so a model card carrying this source credits the office in its own language.
+ */
+export declare const ES_CATASTRO_ATTRIBUTION = "Direcci\u00F3n General del Catastro (Ministerio de Hacienda)";
+/**
+ * The minimum spacing between two requests to `catastro.hacienda.gob.es`, in milliseconds.
+ *
+ * Four requests per second against one government host.
+ * The publisher states no rate limit, and a national harvest is roughly 8,100 requests,
+ * so the harvest is paced rather than parallel.
+ */
+export declare const ES_CATASTRO_REQUEST_INTERVAL_MS = 250;
+/**
+ * The document's text, decoded from the encoding its own declaration states.
+ *
+ * The national feed declares UTF-8 and a province feed declares ISO-8859-1, so a reader
+ * that fixed either encoding would mojibake the other publisher's accented place names.
+ * A document that declares no encoding is decoded as UTF-8, which is what XML's own default states.
+ */
+export declare function decodeDeclaredXML(bytes: Uint8Array): string;
+/**
+ * The URL an href is requested at.
+ *
+ * A province feed writes a municipality's directory segment as the publisher spells it,
+ * with literal spaces and with non-ASCII letters: `…/15/15900-A CORUÑA/A.ES.SDGC.AD.15900.zip`.
+ * `URL` percent-encodes both, which is what the host serves the archive at.
+ */
+export declare function archiveRequestURL(href: string): string;
+/**
+ * One territorial office, as the national service document states it.
+ */
+export interface ESCatastroProvince {
+    /**
+     * The two-digit province code, which is also the first two digits of each of its municipality codes.
+     */
+    code: string;
+    /**
+     * The office name the entry's title states, `Albacete`.
+     */
+    name: string;
+    title: string;
+    /**
+     * This office's province feed, which lists one archive per municipality.
+     */
+    provinceFeedURL: string;
+    /**
+     * The entry's `<updated>`, which dates the national edition.
+     */
+    updated: string;
+}
+/**
+ * One municipality's archive, as its province feed states it.
+ */
+export interface ESCatastroMunicipalityDataset {
+    provinceCode: string;
+    /**
+     * The five-digit cadastral municipality code.
+     */
+    code: string;
+    /**
+     * The municipality name the entry's title states, `CEUTA`.
+     */
+    name: string;
+    title: string;
+    /**
+     * The archive, percent-encoded as the host serves it.
+     */
+    archiveURL: string;
+    /**
+     * The `<updated>` the province feed states for this municipality.
+     */
+    updated: string;
+    /**
+     * The file name the archive is written under, which is the name the publisher gave it
+     * and the name the adapter globs.
+     */
+    filename: string;
+}
+/**
+ * The territorial offices the national service document lists.
+ *
+ * @throws When the document holds no entry, when it holds no territorial-office entry at all,
+ * when an office carries no province-feed link, or when two offices claim one province code.
+ * Each of those would otherwise write zero archives while reporting a completed run.
+ */
+export declare function readESCatastroServiceFeed(feed: AtomFeed): readonly ESCatastroProvince[];
+/**
+ * The municipalities one province feed lists.
+ *
+ * @throws When the feed holds no entry, when an entry carries no archive link,
+ * when the municipality code its title states disagrees with the one its archive
+ * is named for, or when two entries name one archive.
+ */
+export declare function readESCatastroProvinceFeed(feed: AtomFeed, province: Pick<ESCatastroProvince, "code" | "provinceFeedURL">): readonly ESCatastroMunicipalityDataset[];
+/**
+ * One archive, as the harvest's manifest records it.
+ */
+export interface ESCatastroArchiveEntry extends SourceManifest {
+    province_code: string;
+    municipality_code: string;
+    /**
+     * The `<updated>` the province feed stated when this archive was fetched.
+     *
+     * A later run that reads the same value for this municipality leaves the file alone.
+     * The publisher moves it when it publishes a new edition.
+     */
+    feed_updated: string;
+    /**
+     * The `Last-Modified` the host served the archive under, or `null` where it served none.
+     */
+    last_modified: string | null;
+}
+/**
+ * The harvest's `MANIFEST.json`.
+ */
+export interface ESCatastroHarvestManifest extends SourceCollectionManifest {
+    /**
+     * How many territorial offices the service document listed, read from the feed on each run.
+     */
+    provinces_listed: number;
+    /**
+     * How many of those offices this run read a province feed for.
+     *
+     * A bounded run reads fewer, so this is the denominator for `municipalities_listed`
+     * rather than a count of the country.
+     */
+    provinces_harvested: number;
+    /**
+     * How many municipalities the province feeds this run read listed between them.
+     */
+    municipalities_listed: number;
+    files: ESCatastroArchiveEntry[];
+}
+export interface HarvestESCatastroOptions {
+    /**
+     * Where the archives and the manifest are written, which is the adapter's `inputPath`.
+     */
+    outputDir: PathBuilderLike;
+    /**
+     * Harvest only these two-digit province codes.
+     *
+     * A code the service document does not list is reported as a failure rather than ignored.
+     * Without this every one of the 52 province feeds is read, at one request each.
+     */
+    provinces?: readonly string[];
+    /**
+     * Harvest only these five-digit municipality codes, among the provinces selected.
+     *
+     * A code none of the selected province feeds lists is reported as a failure.
+     */
+    municipalities?: readonly string[];
+    /**
+     * Stop after this many municipalities, counting the ones already on disk.
+     */
+    limit?: number;
+    /**
+     * Re-read the sha256 of every archive already on disk instead of comparing its byte count.
+     *
+     * The default compares the recorded byte count against the file's size, which is one `stat`.
+     */
+    verifyDigests?: boolean;
+    signal?: AbortSignal;
+    report?: (line: string) => void;
+}
+/**
+ * Harvest the municipalities the selected province feeds list into `options.outputDir`.
+ *
+ * The client is injected so the harvest is testable without the network and so a caller decides
+ * the pacing. {@linkcode fetchESCatastro} is the registry entry point and supplies both.
+ *
+ * @returns What was fetched, what the feed states is already recorded, and which codes failed.
+ * @throws When the service document lists no territorial office, or
+ * when a selected province feed lists no municipality.
+ * Either would otherwise answer `{fetched: 0, skipped: 0, failed: 0}`, which a caller
+ * reads as a fetch that completed and found the publisher empty.
+ */
+export declare function harvestESCatastro(client: Pick<APIClient, "fetch">, options: HarvestESCatastroOptions): Promise<FetchSummary>;
+/**
+ * The path `#es/adapters/catastro/adapter` reads, given the root a fetch wrote under.
+ *
+ * The adapter takes one archive or a directory holding them, and a harvest writes a directory.
+ */
+export declare function esCatastroInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike;
+export interface FetchESCatastroOptions extends BaseFetchOptions, Pick<HarvestESCatastroOptions, "provinces" | "municipalities" | "limit" | "verifyDigests" | "signal"> {
+    /**
+     * The minimum spacing between two requests, in milliseconds.
+     *
+     * Defaults to {@linkcode ES_CATASTRO_REQUEST_INTERVAL_MS}.
+     */
+    minRequestIntervalMs?: number;
+}
+/**
+ * Harvest the Dirección General del Catastro's INSPIRE Addresses archives into `<outRoot>/es-catastro/`.
+ *
+ * Re-runnable: a municipality whose recorded publication date is still the one its province
+ * feed states, and whose archive is still on disk at the recorded length, costs no request.
+ */
+export declare function fetchESCatastro(options: FetchESCatastroOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=catastro.d.ts.map

package/out/es/tools/fetch/catastro.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"catastro.d.ts","sourceRoot":"","sources":["../../../../lib/es/tools/fetch/catastro.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AAEH,OAAO,EAAE,SAAS,EAA+B,MAAM,qBAAqB,CAAA;AAO5E,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAG3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AAEjD,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAIrH;;GAEG;AACH,eAAO,MAAM,4BAA4B,+EAA+E,CAAA;AAExH;;;;;;GAMG;AACH,eAAO,MAAM,uBAAuB,iEAA4D,CAAA;AAEhG;;;;;;GAMG;AACH,eAAO,MAAM,+BAA+B,MAAM,CAAA;AAiDlD;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAI3D;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,eAAe,EAAE,MAAM,CAAA;IACvB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC7C,YAAY,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,QAAQ,GAAG,SAAS,kBAAkB,EAAE,CAqEvF;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CACzC,IAAI,EAAE,QAAQ,EACd,QAAQ,EAAE,IAAI,CAAC,kBAAkB,EAAE,MAAM,GAAG,iBAAiB,CAAC,GAC5D,SAAS,6BAA6B,EAAE,CA0E1C;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,cAAc;IAC7D,aAAa,EAAE,MAAM,CAAA;IACrB,iBAAiB,EAAE,MAAM,CAAA;IACzB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,wBAAwB;IAC1E;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAA;IACxB;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAA;IAC3B;;OAEG;IACH,qBAAqB,EAAE,MAAM,CAAA;IAC7B,KAAK,EAAE,sBAAsB,EAAE,CAAA;CAC/B;AAED,MAAM,WAAW,wBAAwB;IACxC;;OAEG;IACH,SAAS,EAAE,eAAe,CAAA;IAC1B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B;;;;OAIG;IACH,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAClC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AA4ED;;;;;;;;;;;GAWG;AACH,wBAAsB,iBAAiB,CACtC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,wBAAwB,GAC/B,OAAO,CAAC,YAAY,CAAC,CAyJvB;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC,GAAG,eAAe,CAEzF;AAED,MAAM,WAAW,sBAChB,SACC,gBAAgB,EAChB,IAAI,CAAC,wBAAwB,EAAE,WAAW,GAAG,gBAAgB,GAAG,OAAO,GAAG,eAAe,GAAG,QAAQ,CAAC;IACtG;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACpC,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAkBvB"}