@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/fetch/wikidata-subvenue.d.ts

@@ -3,117 +3,48 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Fetch the multilingual sub-venue designator vocabulary from Wikidata (#35 wave 1).
- *
- *   Source : https://query.wikidata.org/sparql (the Wikidata Query Service).
- *   License: CC0. Wikidata's data is public-domain dedicated, so nothing rides on a derived slice.
- *            Tier A.
- *
- *   ## The pull is CLASS labels, not instance names — and that inversion is the whole design
- *
- *   The obvious read of "Wikidata for localized designators" is: fetch every airport terminal entity
- *   and read its name in each language. That turned out to be the wrong query. Wikidata
- *   knows 246 items that are `instance of / subclass of*` airport terminal, carrying 775 labels
- *   between them (measured 2026-08-04), and most of those labels are proper names that translate
- *   verbatim — `TWA Flight Center` is spelled `TWA Flight Center` in fifteen languages. The
- *   instance layer is thin and its localization is mostly a no-op.
- *
- *   The DESIGNATOR is the label of the CLASS. `wd:Q849706` ("airport terminal") is labelled `Terminal`
- *   in German, `terminal aéroportuaire` in French, `ターミナルビル` in Japanese, `航站楼` in Chinese —
- *   and `skos:altLabel` adds the aliases (`Abfertigungsgebäude`, `Flughafenterminal`, `aerostazione`).
- *   Eight concept ids yield 877 label+alias rows across 174 languages, which is the vocabulary the
- *   corpus task asked for and the instance query does not contain. {@link SUBVENUE_CONCEPTS} is that
- *   list of ids; {@link buildDesignatorLabelQuery} is that query.
- *
- *   Instance labels are fetched too ({@link buildTerminalInstanceQuery}), for a different job: they
- *   are ATTESTED USAGE — evidence of how a designator combines with a modifier or an identifier in
- *   running text. 775 rows is small, and it is a validation set, not a vocabulary.
- *
- *   ## What a caller must NOT do with the output
- *
- *   A class label is a CONCEPT NAME, not a designator as written in an address. Q849706's Spanish
- *   label is `terminal aeroportuaria` and its French is `terminal d'aéroport`; nobody writes either on
- *   an envelope, they write `Terminal`. Q247739's Spanish is `puerta de embarque` where the addressed
- *   form is `Puerta`. So this fetch produces CANDIDATE SURFACES that need a head-noun/curation pass
- *   before any of them reaches `neural/venue-structure.ts`'s designator vocabulary — the lexicon
- *   builder marks every one `curated: false` and the burden of promotion is on a human. Wiring the raw
- *   pull straight into the span proposer would admit multi-word phrases that match nothing and, worse,
- *   admit `hall` in a language where it names an ordinary room.
- *
- *   ## Why `APIClient` here when the OurAirports sibling uses `downloadToFile`
- *
- *   This is the API-request side of `AGENTS.md`'s split: small JSON bodies, several calls per run, and
- *   a host that publishes a rate policy and enforces it with 429s. Pacing, bounded `Retry-After`-aware
- *   retry, response caching and `ResourceError` mapping all earn their keep, so it extends
- *   {@link APIClient}. `ourairports.ts` is four static file transfers off a CDN and correctly does not.
- *
- *   WDQS also REQUIRES a descriptive `User-Agent` naming the tool and a contact — an anonymous or
- *   library-default agent is blocked outright by the Wikimedia user-agent policy. See
- *   {@link WIKIDATA_USER_AGENT}.
- *
- *   Invoke via `mailwoman corpus fetch wikidata-subvenue --out-root <path>`.
+ * Fetches CC0 sub-venue concept labels and airport-terminal names from the Wikidata Query Service.
+ * The labels are candidates that need curation before any recipe uses them as designators.
  */
 import { APIClient, type ClockLike } from "@mailwoman/core/api";
+import type { PathBuilderLike } from "path-ts";
 import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
 /**
- * The SPARQL endpoint. Public, no credential.
+ * The public SPARQL endpoint of the Wikidata Query Service.
  */
 export declare const WDQS_ENDPOINT = "https://query.wikidata.org/sparql";
 /**
- * The `User-Agent` every request carries.
- *
- * NOT decoration. The Wikimedia user-agent policy blocks requests whose agent is absent, generic, or a library default,
- * and WDQS enforces it — an unidentified client gets a 403 that no amount of retrying fixes. The policy asks for a tool
- * name, a URL, and a contact address, all three of which are here.
+ * The descriptive `User-Agent` that Wikidata requires, with the tool name, URL and contact address.
  */
 export declare const WIKIDATA_USER_AGENT = "mailwoman/1.0 (https://github.com/sister-software/mailwoman; teffen@sister.software) corpus-subvenue-fetch";
 /**
- * One Wikidata concept whose labels are a designator's multilingual surface set.
- *
- * `designatorID` is the mailwoman-side vocabulary term, matching `neural/venue-structure.ts`'s
- * `VENUE_STRUCTURE_DESIGNATORS` wherever the two overlap. `qid` was resolved by `wbsearchentities` and hand-checked
- * against the entity's English description (recorded below) on 2026-08-04 — a QID picked by search alone is how you end
- * up pulling the labels of a Bronx neighbourhood called Concourse.
- *
- * `wing` is ABSENT and that is a finding, not an oversight: Wikidata has no clean concept for "wing of a building".
- * `wbsearchentities` for "wing" returns a surname, two English villages, a rugby position and a drone company. Since
- * `wing` is the single most valuable designator in the arc — `West Wing` is the one modifier case that already parses,
- * and `East Wing` is the one that does not — its localized surfaces have to come from somewhere else. See the wave-1
- * report.
+ * A Wikidata concept whose labels are collected for one sub-venue designator.
  */
 export interface SubVenueConcept {
     designatorID: string;
     qid: string;
     /**
-     * The entity's English description, recorded so a future reader can tell at a glance whether the QID still names what
-     * we think it names.
+     * An English description that lets a reviewer confirm the QID.
      */
     gloss: string;
 }
 /**
- * The concept table. Eight ids, each verified against its English description on 2026-08-04.
+ * The concepts whose labels and aliases the fetcher collects.
  */
 export declare const SUBVENUE_CONCEPTS: readonly SubVenueConcept[];
 /**
- * Build the class-label query: `rdfs:label` and `skos:altLabel` for every concept, in every language, tagged with which
- * of the two it came from so the lexicon can rank a label above an alias.
- *
- * `VALUES` rather than a property path over the whole class tree — the concept list is closed and hand-verified, and a
- * `wdt:P279*` walk from `building` would drag in every structure type on earth.
+ * Builds a query for the labels and aliases of the concepts in every language.
  */
 export declare function buildDesignatorLabelQuery(concepts?: readonly SubVenueConcept[]): string;
 /**
- * Build the instance-label query — every item that is an `instance of` (through any `subclass of` chain) an airport
- * terminal, with all of its labels. Measured at 246 items / 775 labels on 2026-08-04, well inside WDQS's 60-second
- * budget.
+ * Builds a query for the labels of instances of the class and its subclasses.
  *
- * A caveat worth knowing before trusting a row: Wikidata's P31 on these is not clean. `Q1322696` (Kigali International
- * Airport) is typed as an airport terminal, so the result set mixes AIRPORTS in with terminals. The consumer filters;
- * this module fetches what the query returns.
+ * The subclass walk returns some unrelated items.
+ * Consumers must filter them.
  */
 export declare function buildTerminalInstanceQuery(classQID?: string): string;
 /**
- * The SPARQL JSON results shape, narrowed to the two column types these queries produce.
+ * The SPARQL JSON results shape.
  */
 export interface SPARQLResults {
     results: {
@@ -125,47 +56,48 @@ export interface SPARQLResults {
     };
 }
 /**
- * Whether a decoded body is a SPARQL results envelope. Used as the cache's write validator so an HTML error page served
- * under a 200 is never persisted for the next run to destructure into `undefined`.
+ * Reports whether a value has the SPARQL JSON results shape.
  */
 export declare function isSPARQLResults(value: unknown): value is SPARQLResults;
+/**
+ * Options for {@link createWikidataClient}.
+ */
 export interface CreateWikidataClientOptions {
     /**
-     * On-disk response-cache root. Defaults to a `http-cache` directory beside the fetch output.
+     * The directory of the on-disk response cache.
      */
-    cacheDir: string;
+    cacheDir: PathBuilderLike;
     /**
-     * Time source powering the pacer and the retry backoff. Defaults to the system clock; tests inject a fake so no suite
-     * ever sleeps a real second.
+     * The clock for pacing and retries.
+     * Tests can replace it.
      */
     clock?: ClockLike;
     /**
-     * Axios overrides, merged over this client's defaults. THE TEST INJECTION POINT — pass an `adapter` and no live call
-     * is made. Overriding `headers` wholesale would drop the required `User-Agent`, so don't.
+     * Axios overrides.
+     * Replacement headers must keep the required `User-Agent`.
      */
     axios?: ConstructorParameters<typeof APIClient>[0]["axios"];
 }
 /**
- * A Wikidata Query Service client: paced, retrying, disk-cached, and correctly identified.
+ * A paced, cached and retrying Wikidata Query Service client.
  */
 export declare class WikidataClient extends APIClient {
     /**
-     * Run one SPARQL query and return its results envelope.
+     * Runs one SPARQL query.
      */
     query(sparql: string): Promise<SPARQLResults>;
 }
 /**
- * Construct a {@link WikidataClient} with every default resolved.
+ * Creates a Wikidata client with the fetcher's pacing, retry, cache and header defaults.
  */
 export declare function createWikidataClient(options: CreateWikidataClientOptions): WikidataClient;
+/**
+ * Options for {@link fetchWikidataSubVenue}.
+ */
 export type FetchWikidataSubVenueOptions = BaseFetchOptions;
 /**
- * Run both queries and write their raw SPARQL JSON into `<outRoot>/wikidata-subvenue/`, with a `MANIFEST.json` carrying
- * the endpoint, the exact queries, the concept table, row counts and sha256s.
- *
- * The RAW envelope is written rather than a reshaped one on purpose: the lexicon build is a separate, pure step
- * (`sub-venue-lexicon.ts`) and keeping the fetch output byte-faithful to what WDQS served means a lexicon regeneration
- * never needs the network.
+ * Runs the label and terminal queries.
+ * It saves their raw JSON and writes a manifest.
  */
 export declare function fetchWikidataSubVenue(options: FetchWikidataSubVenueOptions, report?: (line: string) => void): Promise<FetchSummary>;
 //# sourceMappingURL=wikidata-subvenue.d.ts.map

package/out/tools/fetch/wikidata-subvenue.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"wikidata-subvenue.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/wikidata-subvenue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAM/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAK3E;;GAEG;AACH,eAAO,MAAM,aAAa,sCAAsC,CAAA;AAEhE;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,+GAC6E,CAAA;AAkC7G;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,eAAe;IAC/B,YAAY,EAAE,MAAM,CAAA;IACpB,GAAG,EAAE,MAAM,CAAA;IACX;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAA;CACb;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,EAAE,SAAS,eAAe,EASvD,CAAA;AAOD;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,GAAE,SAAS,eAAe,EAAsB,GAAG,MAAM,CAU1G;AAED;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,GAAE,MAA2B,GAAG,MAAM,CAMxF;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,OAAO,EAAE;QACR,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC,CAAA;KACrF,CAAA;CACD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,KAAK,CAAC,EAAE,SAAS,CAAA;IACjB;;;OAGG;IACH,KAAK,CAAC,EAAE,qBAAqB,CAAC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;CAC3D;AAED;;GAEG;AACH,qBAAa,cAAe,SAAQ,SAAS;IAC5C;;OAEG;IACU,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;CAQ1D;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,2BAA2B,GAAG,cAAc,CAkCzF;AAED,MAAM,MAAM,4BAA4B,GAAG,gBAAgB,CAAA;AA0C3D;;;;;;;GAOG;AACH,wBAAsB,qBAAqB,CAC1C,OAAO,EAAE,4BAA4B,EACrC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA+CvB"}
+{"version":3,"file":"wikidata-subvenue.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/wikidata-subvenue.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAK/D,OAAO,KAAK,EAAe,eAAe,EAAE,MAAM,SAAS,CAAA;AAE3D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAK3E;;GAEG;AACH,eAAO,MAAM,aAAa,sCAAsC,CAAA;AAEhE;;GAEG;AACH,eAAO,MAAM,mBAAmB,+GAC6E,CAAA;AAW7G;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,YAAY,EAAE,MAAM,CAAA;IACpB,GAAG,EAAE,MAAM,CAAA;IACX;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;CACb;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,EAAE,SAAS,eAAe,EASvD,CAAA;AAID;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,GAAE,SAAS,eAAe,EAAsB,GAAG,MAAM,CAU1G;AAED;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,GAAE,MAA2B,GAAG,MAAM,CAMxF;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,OAAO,EAAE;QACR,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC,CAAA;KACrF,CAAA;CACD;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,QAAQ,EAAE,eAAe,CAAA;IACzB;;;OAGG;IACH,KAAK,CAAC,EAAE,SAAS,CAAA;IACjB;;;OAGG;IACH,KAAK,CAAC,EAAE,qBAAqB,CAAC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;CAC3D;AAED;;GAEG;AACH,qBAAa,cAAe,SAAQ,SAAS;IAC5C;;OAEG;IACU,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAOzD;CACD;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,2BAA2B,GAAG,cAAc,CA4BzF;AAED;;GAEG;AACH,MAAM,MAAM,4BAA4B,GAAG,gBAAgB,CAAA;AA0C3D;;;GAGG;AACH,wBAAsB,qBAAqB,CAC1C,OAAO,EAAE,4BAA4B,EACrC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA+CvB"}

package/out/tools/fetch/wikidata-subvenue.js

@@ -3,105 +3,31 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Fetch the multilingual sub-venue designator vocabulary from Wikidata (#35 wave 1).
- *
- *   Source : https://query.wikidata.org/sparql (the Wikidata Query Service).
- *   License: CC0. Wikidata's data is public-domain dedicated, so nothing rides on a derived slice.
- *            Tier A.
- *
- *   ## The pull is CLASS labels, not instance names — and that inversion is the whole design
- *
- *   The obvious read of "Wikidata for localized designators" is: fetch every airport terminal entity
- *   and read its name in each language. That turned out to be the wrong query. Wikidata
- *   knows 246 items that are `instance of / subclass of*` airport terminal, carrying 775 labels
- *   between them (measured 2026-08-04), and most of those labels are proper names that translate
- *   verbatim — `TWA Flight Center` is spelled `TWA Flight Center` in fifteen languages. The
- *   instance layer is thin and its localization is mostly a no-op.
- *
- *   The DESIGNATOR is the label of the CLASS. `wd:Q849706` ("airport terminal") is labelled `Terminal`
- *   in German, `terminal aéroportuaire` in French, `ターミナルビル` in Japanese, `航站楼` in Chinese —
- *   and `skos:altLabel` adds the aliases (`Abfertigungsgebäude`, `Flughafenterminal`, `aerostazione`).
- *   Eight concept ids yield 877 label+alias rows across 174 languages, which is the vocabulary the
- *   corpus task asked for and the instance query does not contain. {@link SUBVENUE_CONCEPTS} is that
- *   list of ids; {@link buildDesignatorLabelQuery} is that query.
- *
- *   Instance labels are fetched too ({@link buildTerminalInstanceQuery}), for a different job: they
- *   are ATTESTED USAGE — evidence of how a designator combines with a modifier or an identifier in
- *   running text. 775 rows is small, and it is a validation set, not a vocabulary.
- *
- *   ## What a caller must NOT do with the output
- *
- *   A class label is a CONCEPT NAME, not a designator as written in an address. Q849706's Spanish
- *   label is `terminal aeroportuaria` and its French is `terminal d'aéroport`; nobody writes either on
- *   an envelope, they write `Terminal`. Q247739's Spanish is `puerta de embarque` where the addressed
- *   form is `Puerta`. So this fetch produces CANDIDATE SURFACES that need a head-noun/curation pass
- *   before any of them reaches `neural/venue-structure.ts`'s designator vocabulary — the lexicon
- *   builder marks every one `curated: false` and the burden of promotion is on a human. Wiring the raw
- *   pull straight into the span proposer would admit multi-word phrases that match nothing and, worse,
- *   admit `hall` in a language where it names an ordinary room.
- *
- *   ## Why `APIClient` here when the OurAirports sibling uses `downloadToFile`
- *
- *   This is the API-request side of `AGENTS.md`'s split: small JSON bodies, several calls per run, and
- *   a host that publishes a rate policy and enforces it with 429s. Pacing, bounded `Retry-After`-aware
- *   retry, response caching and `ResourceError` mapping all earn their keep, so it extends
- *   {@link APIClient}. `ourairports.ts` is four static file transfers off a CDN and correctly does not.
- *
- *   WDQS also REQUIRES a descriptive `User-Agent` naming the tool and a contact — an anonymous or
- *   library-default agent is blocked outright by the Wikimedia user-agent policy. See
- *   {@link WIKIDATA_USER_AGENT}.
- *
- *   Invoke via `mailwoman corpus fetch wikidata-subvenue --out-root <path>`.
+ * Fetches CC0 sub-venue concept labels and airport-terminal names from the Wikidata Query Service.
+ * The labels are candidates that need curation before any recipe uses them as designators.
  */
 import { APIClient } from "@mailwoman/core/api";
 import { buildDiskStorage } from "@mailwoman/core/api/disk-storage";
 import { makeDirectories, writeLocalFile } from "@mailwoman/core/fs/writers";
 import { sha256File } from "@mailwoman/core/hash";
-import { join } from "path-ts";
+import { prettyJSON } from "@mailwoman/core/json";
 import { writeManifest } from "#tools/fetch/download";
 const SLUG = "wikidata-subvenue";
 /**
- * The SPARQL endpoint. Public, no credential.
+ * The public SPARQL endpoint of the Wikidata Query Service.
  */
 export const WDQS_ENDPOINT = "https://query.wikidata.org/sparql";
 /**
- * The `User-Agent` every request carries.
- *
- * NOT decoration. The Wikimedia user-agent policy blocks requests whose agent is absent, generic, or a library default,
- * and WDQS enforces it — an unidentified client gets a 403 that no amount of retrying fixes. The policy asks for a tool
- * name, a URL, and a contact address, all three of which are here.
+ * The descriptive `User-Agent` that Wikidata requires, with the tool name, URL and contact address.
  */
 export const WIKIDATA_USER_AGENT = "mailwoman/1.0 (https://github.com/sister-software/mailwoman; teffen@sister.software) corpus-subvenue-fetch";
-/**
- * Minimum spacing between dispatches, in milliseconds.
- *
- * WDQS's published limit is expressed as processing-time budget rather than a request rate, and this run issues fewer
- * than a dozen queries total, so the number is chosen for politeness rather than to sit against a ceiling: one query
- * per second is far inside anything WDQS objects to, and at this volume the whole fetch still completes in seconds.
- *
- * Set as `minRequestIntervalMs` rather than `requestsPerMinute` deliberately — `AGENTS.md` records that
- * `requestsPerMinute` is a BUDGET model whose cooldown lets N requests go out back to back, so it does not deliver N
- * per minute and is not the check that holds a rate. The interval is.
- */
 const WDQS_MIN_REQUEST_INTERVAL_MS = 1000;
-/**
- * Per-attempt socket-inactivity timeout. WDQS's own query timeout is 60 seconds and it answers with a 500 when a query
- * exceeds it, so a client timeout below that would turn a server-side timeout into a client-side one and lose the error
- * body that says which query was too expensive.
- */
+// The timeout exceeds the service's 60-second query limit so that its own error response arrives first.
 const WDQS_REQUEST_TIMEOUT_MS = 90_000;
-/**
- * Total attempts (including the first) before giving up on a 429/5xx.
- */
 const WDQS_MAX_ATTEMPTS = 3;
-/**
- * How long a cached SPARQL response stays fresh. A week: the class labels this pulls change on the timescale at which
- * someone edits a Wikidata concept's German alias, which is to say rarely, and a re-run inside a working session should
- * not re-ask.
- */
 const WDQS_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
 /**
- * The concept table. Eight ids, each verified against its English description on 2026-08-04.
+ * The concepts whose labels and aliases the fetcher collects.
  */
 export const SUBVENUE_CONCEPTS = [
     { designatorID: "terminal", qid: "Q849706", gloss: "airport terminal — part of an airport" },
@@ -113,16 +39,9 @@ export const SUBVENUE_CONCEPTS = [
     { designatorID: "hall", qid: "Q240854", gloss: "hall — large enclosed room" },
     { designatorID: "satellite", qid: "Q15990706", gloss: "satellite terminal — detached airport building" },
 ];
-/**
- * The class whose instances are fetched for attested-usage evidence: airport terminal.
- */
 const TERMINAL_CLASS_QID = "Q849706";
 /**
- * Build the class-label query: `rdfs:label` and `skos:altLabel` for every concept, in every language, tagged with which
- * of the two it came from so the lexicon can rank a label above an alias.
- *
- * `VALUES` rather than a property path over the whole class tree — the concept list is closed and hand-verified, and a
- * `wdt:P279*` walk from `building` would drag in every structure type on earth.
+ * Builds a query for the labels and aliases of the concepts in every language.
  */
 export function buildDesignatorLabelQuery(concepts = SUBVENUE_CONCEPTS) {
     const values = concepts.map((c) => `wd:${c.qid}`).join(" ");
@@ -135,13 +54,10 @@ export function buildDesignatorLabelQuery(concepts = SUBVENUE_CONCEPTS) {
 }`;
 }
 /**
- * Build the instance-label query — every item that is an `instance of` (through any `subclass of` chain) an airport
- * terminal, with all of its labels. Measured at 246 items / 775 labels on 2026-08-04, well inside WDQS's 60-second
- * budget.
+ * Builds a query for the labels of instances of the class and its subclasses.
  *
- * A caveat worth knowing before trusting a row: Wikidata's P31 on these is not clean. `Q1322696` (Kigali International
- * Airport) is typed as an airport terminal, so the result set mixes AIRPORTS in with terminals. The consumer filters;
- * this module fetches what the query returns.
+ * The subclass walk returns some unrelated items.
+ * Consumers must filter them.
  */
 export function buildTerminalInstanceQuery(classQID = TERMINAL_CLASS_QID) {
     return `SELECT ?item ?lang ?label WHERE {
@@ -151,18 +67,17 @@ export function buildTerminalInstanceQuery(classQID = TERMINAL_CLASS_QID) {
 }`;
 }
 /**
- * Whether a decoded body is a SPARQL results envelope. Used as the cache's write validator so an HTML error page served
- * under a 200 is never persisted for the next run to destructure into `undefined`.
+ * Reports whether a value has the SPARQL JSON results shape.
  */
 export function isSPARQLResults(value) {
     return typeof value === "object" && value !== null && Array.isArray(value.results?.bindings);
 }
 /**
- * A Wikidata Query Service client: paced, retrying, disk-cached, and correctly identified.
+ * A paced, cached and retrying Wikidata Query Service client.
  */
 export class WikidataClient extends APIClient {
     /**
-     * Run one SPARQL query and return its results envelope.
+     * Runs one SPARQL query.
      */
     async query(sparql) {
         const url = new URL(WDQS_ENDPOINT);
@@ -172,7 +87,7 @@ export class WikidataClient extends APIClient {
     }
 }
 /**
- * Construct a {@link WikidataClient} with every default resolved.
+ * Creates a Wikidata client with the fetcher's pacing, retry, cache and header defaults.
  */
 export function createWikidataClient(options) {
     return new WikidataClient({
@@ -183,15 +98,11 @@ export function createWikidataClient(options) {
         caching: {
             storage: buildDiskStorage({
                 directory: options.cacheDir,
-                // `value.data` is the cached RESPONSE; `value.data.data` is its body. Passing the response here
-                // instead of the body is a silent-failure trap — the predicate returns false for every entry and
-                // every run re-fetches while logging "rejected by the configured validate() predicate". Caught
-                // on the first live run, 2026-08-04.
+                // The validator checks the cached response body inside the envelope.
                 validate: (value) => isSPARQLResults(value.data?.data),
             }),
             ttl: WDQS_CACHE_TTL_MS,
-            // The TTL is chosen against Wikidata's edit cadence; letting a CDN header override it would
-            // silently replace that reasoning with whatever varnish in front of WDQS happens to send.
+            // The configured TTL overrides the CDN cache headers.
             interpretHeader: false,
         },
         axios: {
@@ -201,20 +112,18 @@ export function createWikidataClient(options) {
             },
             timeout: WDQS_REQUEST_TIMEOUT_MS,
             responseType: "json",
-            // Axios hands back the RAW STRING when a body fails to parse unless this is off. WDQS serves an
-            // HTML error page under some failures, and returning that as `SPARQLResults` would surface as an
-            // `undefined` destructure far from the cause.
+            // Malformed or HTML error bodies throw parse errors instead of passing as JSON.
             transitional: { silentJSONParsing: false },
             ...options.axios,
         },
     });
 }
 /**
- * Write a JSON payload and return its manifest entry.
+ * Writes a response as JSON and returns its manifest entry.
  */
 async function writePayload(destDir, filename, query, results) {
-    const path = join(destDir, filename);
-    const body = JSON.stringify(results, null, 2) + "\n";
+    const path = destDir(filename);
+    const body = prettyJSON(results);
     await writeLocalFile(body, path);
     return {
         filename,
@@ -225,17 +134,13 @@ async function writePayload(destDir, filename, query, results) {
     };
 }
 /**
- * Run both queries and write their raw SPARQL JSON into `<outRoot>/wikidata-subvenue/`, with a `MANIFEST.json` carrying
- * the endpoint, the exact queries, the concept table, row counts and sha256s.
- *
- * The RAW envelope is written rather than a reshaped one on purpose: the lexicon build is a separate, pure step
- * (`sub-venue-lexicon.ts`) and keeping the fetch output byte-faithful to what WDQS served means a lexicon regeneration
- * never needs the network.
+ * Runs the label and terminal queries.
+ * It saves their raw JSON and writes a manifest.
  */
 export async function fetchWikidataSubVenue(options, report) {
-    const destDir = join(options.outRoot, SLUG);
+    const destDir = options.outRoot(SLUG);
     await makeDirectories(destDir);
-    await using client = createWikidataClient({ cacheDir: join(destDir, "http-cache") });
+    await using client = createWikidataClient({ cacheDir: destDir("http-cache") });
     const jobs = [
         { filename: "designator-labels.json", query: buildDesignatorLabelQuery() },
         { filename: "terminal-instance-labels.json", query: buildTerminalInstanceQuery() },
@@ -268,7 +173,7 @@ export async function fetchWikidataSubVenue(options, report) {
         concepts: SUBVENUE_CONCEPTS,
         files,
     };
-    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
+    await writeManifest(destDir("MANIFEST.json"), manifest);
     return { fetched, skipped: 0, failed, failedCodes };
 }
 //# sourceMappingURL=wikidata-subvenue.js.map

package/out/tools/fetch/wikidata-subvenue.js.map

@@ -1 +1 @@
-{"version":3,"file":"wikidata-subvenue.js","sourceRoot":"","sources":["../../../lib/tools/fetch/wikidata-subvenue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AAEH,OAAO,EAAE,SAAS,EAAkB,MAAM,qBAAqB,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAA;AACnE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC5E,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAErD,MAAM,IAAI,GAAG,mBAAmB,CAAA;AAEhC;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,mCAAmC,CAAA;AAEhE;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAC/B,4GAA4G,CAAA;AAE7G;;;;;;;;;;GAUG;AACH,MAAM,4BAA4B,GAAG,IAAI,CAAA;AAEzC;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAEtC;;GAEG;AACH,MAAM,iBAAiB,GAAG,CAAC,CAAA;AAE3B;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AA0BjD;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAA+B;IAC5D,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,uCAAuC,EAAE;IAC5F,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,yDAAyD,EAAE;IAC1G,EAAE,YAAY,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,gDAAgD,EAAE;IACtG,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,kEAAkE,EAAE;IACrH,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,4CAA4C,EAAE;IAChG,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,oDAAoD,EAAE;IACvG,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,4BAA4B,EAAE;IAC7E,EAAE,YAAY,EAAE,WAAW,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,gDAAgD,EAAE;CACxG,CAAA;AAED;;GAEG;AACH,MAAM,kBAAkB,GAAG,SAAS,CAAA;AAEpC;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CAAC,WAAuC,iBAAiB;IACjG,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAE3D,OAAO;mBACW,MAAM;;;;;EAKvB,CAAA;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,0BAA0B,CAAC,WAAmB,kBAAkB;IAC/E,OAAO;+BACuB,QAAQ;;;EAGrC,CAAA;AACF,CAAC;AAWD;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAc;IAC7C,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAAuB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;AAChH,CAAC;AAmBD;;GAEG;AACH,MAAM,OAAO,cAAe,SAAQ,SAAS;IAC5C;;OAEG;IACI,KAAK,CAAC,KAAK,CAAC,MAAc;QAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,CAAA;QAClC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAErC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAgB,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;QAEzE,OAAO,QAAQ,CAAC,IAAI,CAAA;IACrB,CAAC;CACD;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAoC;IACxE,OAAO,IAAI,cAAc,CAAC;QACzB,WAAW,EAAE,wBAAwB;QACrC,oBAAoB,EAAE,4BAA4B;QAClD,KAAK,EAAE,EAAE,WAAW,EAAE,iBAAiB,EAAE;QACzC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE;YACR,OAAO,EAAE,gBAAgB,CAAC;gBACzB,SAAS,EAAE,OAAO,CAAC,QAAQ;gBAC3B,gGAAgG;gBAChG,iGAAiG;gBACjG,+FAA+F;gBAC/F,qCAAqC;gBACrC,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC;aACtD,CAAC;YACF,GAAG,EAAE,iBAAiB;YACtB,4FAA4F;YAC5F,0FAA0F;YAC1F,eAAe,EAAE,KAAK;SACtB;QACD,KAAK,EAAE;YACN,OAAO,EAAE;gBACR,YAAY,EAAE,mBAAmB;gBACjC,MAAM,EAAE,iCAAiC;aACzC;YACD,OAAO,EAAE,uBAAuB;YAChC,YAAY,EAAE,MAAM;YACpB,gGAAgG;YAChG,iGAAiG;YACjG,8CAA8C;YAC9C,YAAY,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE;YAC1C,GAAG,OAAO,CAAC,KAAK;SAChB;KACD,CAAC,CAAA;AACH,CAAC;AAsBD;;GAEG;AACH,KAAK,UAAU,YAAY,CAC1B,OAAe,EACf,QAAgB,EAChB,KAAa,EACb,OAAsB;IAEtB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAA;IACpD,MAAM,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAEhC,OAAO;QACN,QAAQ;QACR,KAAK;QACL,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM;QACrC,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC;QAC9B,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;KAC9B,CAAA;AACF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAC1C,OAAqC,EACrC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,YAAY,MAAM,GAAG,oBAAoB,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC,CAAA;IAEpF,MAAM,IAAI,GAA+C;QACxD,EAAE,QAAQ,EAAE,wBAAwB,EAAE,KAAK,EAAE,yBAAyB,EAAE,EAAE;QAC1E,EAAE,QAAQ,EAAE,+BAA+B,EAAE,KAAK,EAAE,0BAA0B,EAAE,EAAE;KAClF,CAAA;IAED,MAAM,KAAK,GAAwB,EAAE,CAAA;IACrC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;QAEzC,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAC7C,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;YAC3E,MAAM,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAA;YACtD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEjB,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,KAAK,GAAG,CAAC,QAAQ,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACxF,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAE9B,MAAM,EAAE,CAAA;QACT,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAqB;QAClC,MAAM,EAAE,wBAAwB;QAChC,QAAQ,EAAE,aAAa;QACvB,OAAO,EAAE,KAAK;QACd,UAAU,EAAE,mBAAmB;QAC/B,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,iBAAiB;QAC3B,KAAK;KACL,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}
+{"version":3,"file":"wikidata-subvenue.js","sourceRoot":"","sources":["../../../lib/tools/fetch/wikidata-subvenue.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,SAAS,EAAkB,MAAM,qBAAqB,CAAA;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAA;AACnE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC5E,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAIjD,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAErD,MAAM,IAAI,GAAG,mBAAmB,CAAA;AAEhC;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,mCAAmC,CAAA;AAEhE;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAC/B,4GAA4G,CAAA;AAE7G,MAAM,4BAA4B,GAAG,IAAI,CAAA;AAEzC,wGAAwG;AACxG,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAEtC,MAAM,iBAAiB,GAAG,CAAC,CAAA;AAE3B,MAAM,iBAAiB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAcjD;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAA+B;IAC5D,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,uCAAuC,EAAE;IAC5F,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,yDAAyD,EAAE;IAC1G,EAAE,YAAY,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,gDAAgD,EAAE;IACtG,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,kEAAkE,EAAE;IACrH,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,4CAA4C,EAAE;IAChG,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,oDAAoD,EAAE;IACvG,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,4BAA4B,EAAE;IAC7E,EAAE,YAAY,EAAE,WAAW,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,gDAAgD,EAAE;CACxG,CAAA;AAED,MAAM,kBAAkB,GAAG,SAAS,CAAA;AAEpC;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAC,QAAQ,GAA+B,iBAAiB;IACjG,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAE3D,OAAO;mBACW,MAAM;;;;;EAKvB,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,0BAA0B,CAAC,QAAQ,GAAW,kBAAkB;IAC/E,OAAO;+BACuB,QAAQ;;;EAGrC,CAAA;AACF,CAAC;AAWD;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAc;IAC7C,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAAuB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;AAChH,CAAC;AAsBD;;GAEG;AACH,MAAM,OAAO,cAAe,SAAQ,SAAS;IAC5C;;OAEG;IACI,KAAK,CAAC,KAAK,CAAC,MAAc;QAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,CAAA;QAClC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAErC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAgB,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;QAEzE,OAAO,QAAQ,CAAC,IAAI,CAAA;IACrB,CAAC;CACD;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAoC;IACxE,OAAO,IAAI,cAAc,CAAC;QACzB,WAAW,EAAE,wBAAwB;QACrC,oBAAoB,EAAE,4BAA4B;QAClD,KAAK,EAAE,EAAE,WAAW,EAAE,iBAAiB,EAAE;QACzC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE;YACR,OAAO,EAAE,gBAAgB,CAAC;gBACzB,SAAS,EAAE,OAAO,CAAC,QAAQ;gBAC3B,qEAAqE;gBACrE,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC;aACtD,CAAC;YACF,GAAG,EAAE,iBAAiB;YACtB,sDAAsD;YACtD,eAAe,EAAE,KAAK;SACtB;QACD,KAAK,EAAE;YACN,OAAO,EAAE;gBACR,YAAY,EAAE,mBAAmB;gBACjC,MAAM,EAAE,iCAAiC;aACzC;YACD,OAAO,EAAE,uBAAuB;YAChC,YAAY,EAAE,MAAM;YACpB,gFAAgF;YAChF,YAAY,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE;YAC1C,GAAG,OAAO,CAAC,KAAK;SAChB;KACD,CAAC,CAAA;AACH,CAAC;AAyBD;;GAEG;AACH,KAAK,UAAU,YAAY,CAC1B,OAAoB,EACpB,QAAgB,EAChB,KAAa,EACb,OAAsB;IAEtB,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC9B,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,CAAA;IAChC,MAAM,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAEhC,OAAO;QACN,QAAQ;QACR,KAAK;QACL,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM;QACrC,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC;QAC9B,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;KAC9B,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAC1C,OAAqC,EACrC,MAA+B;IAE/B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACrC,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,YAAY,MAAM,GAAG,oBAAoB,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;IAE9E,MAAM,IAAI,GAA+C;QACxD,EAAE,QAAQ,EAAE,wBAAwB,EAAE,KAAK,EAAE,yBAAyB,EAAE,EAAE;QAC1E,EAAE,QAAQ,EAAE,+BAA+B,EAAE,KAAK,EAAE,0BAA0B,EAAE,EAAE;KAClF,CAAA;IAED,MAAM,KAAK,GAAwB,EAAE,CAAA;IACrC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;QAEzC,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAC7C,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;YAC3E,MAAM,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAA;YACtD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEjB,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,KAAK,GAAG,CAAC,QAAQ,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACxF,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAE9B,MAAM,EAAE,CAAA;QACT,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAqB;QAClC,MAAM,EAAE,wBAAwB;QAChC,QAAQ,EAAE,aAAa;QACvB,OAAO,EAAE,KAAK;QACd,UAAU,EAAE,mBAAmB;QAC/B,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,iBAAiB;QAC3B,KAAK;KACL,CAAA;IAED,MAAM,aAAa,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAEvD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/tools/fetch/zip-archive.d.ts

@@ -0,0 +1,54 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Downloads one zip archive through an {@linkcode APIClient} and answers what arrived.
+ *
+ *   An INSPIRE ATOM service that publishes one archive per area is harvested through many small
+ *   requests rather than through one large transfer, so the transfer runs on the client the harvest
+ *   already paces and a test can drive it over a stubbed transport. `downloadToFile` runs on global
+ *   `fetch`, which a unit test cannot intercept.
+ *
+ *   The body is checked against the zip local-file-header signature rather than against a length.
+ *   Two publishers answer a withdrawn or misspelled path with an html page under http 200, and the
+ *   advertised media type does not settle what arrived either: ČÚZK advertises
+ *   `application/gml+xml` on a body it serves as `application/zip`, and the Dirección General del
+ *   Catastro advertises `application/atom+xml` on its zipped GML. The signature is the one statement
+ *   the body itself makes.
+ */
+import type { APIClient } from "@mailwoman/core/api";
+import type { PathBuilderLike } from "path-ts";
+/**
+ * What one archive transfer delivered.
+ */
+export interface ZipArchiveDownload {
+    /**
+     * The byte count of the delivered body, counted rather than read from a header or a feed.
+     *
+     * An INSPIRE feed's `length` attribute is a claim: Denmark's understates its file by
+     * 23 times, and Navarra states the same 34,987 on all 272 of its partitions.
+     */
+    bytes: number;
+    sha256: string;
+    /**
+     * The `Last-Modified` the host served the archive under, or `null` where it served none.
+     */
+    lastModified: string | null;
+}
+export interface DownloadZipArchiveOptions {
+    url: string;
+    dest: PathBuilderLike;
+    /**
+     * Per-request timeout, in milliseconds.
+     */
+    timeout?: number;
+    signal?: AbortSignal;
+}
+/**
+ * Downloads one zip archive to `dest` and answers its byte count, digest and served modification time.
+ *
+ * @throws When the body does not begin with a zip signature, so an html error page
+ * is never written to disk under an archive's name.
+ */
+export declare function downloadZipArchive(client: Pick<APIClient, "fetch">, options: DownloadZipArchiveOptions): Promise<ZipArchiveDownload>;
+//# sourceMappingURL=zip-archive.d.ts.map

package/out/tools/fetch/zip-archive.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"zip-archive.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/zip-archive.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAGpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAW9C;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;;;;OAKG;IACH,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;CAC3B;AAED,MAAM,WAAW,yBAAyB;IACzC,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,eAAe,CAAA;IACrB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACvC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,yBAAyB,GAChC,OAAO,CAAC,kBAAkB,CAAC,CA6B7B"}

package/out/tools/fetch/zip-archive.js

@@ -0,0 +1,58 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Downloads one zip archive through an {@linkcode APIClient} and answers what arrived.
+ *
+ *   An INSPIRE ATOM service that publishes one archive per area is harvested through many small
+ *   requests rather than through one large transfer, so the transfer runs on the client the harvest
+ *   already paces and a test can drive it over a stubbed transport. `downloadToFile` runs on global
+ *   `fetch`, which a unit test cannot intercept.
+ *
+ *   The body is checked against the zip local-file-header signature rather than against a length.
+ *   Two publishers answer a withdrawn or misspelled path with an html page under http 200, and the
+ *   advertised media type does not settle what arrived either: ČÚZK advertises
+ *   `application/gml+xml` on a body it serves as `application/zip`, and the Dirección General del
+ *   Catastro advertises `application/atom+xml` on its zipped GML. The signature is the one statement
+ *   the body itself makes.
+ */
+import { writeLocalFile } from "@mailwoman/core/fs/writers";
+import { sha256Hex } from "@mailwoman/core/hash";
+/**
+ * The first two bytes of every zip archive, `PK`.
+ *
+ * The check is on the signature rather than on a size, because an error page and a small
+ * area's archive are told apart by what the body is rather than by how long it is.
+ * A size floor between the two would be a number fitted to the archives measured.
+ */
+const ZIP_SIGNATURE = "PK";
+/**
+ * Downloads one zip archive to `dest` and answers its byte count, digest and served modification time.
+ *
+ * @throws When the body does not begin with a zip signature, so an html error page
+ * is never written to disk under an archive's name.
+ */
+export async function downloadZipArchive(client, options) {
+    const response = await client.fetch({
+        method: "GET",
+        url: options.url,
+        responseType: "arraybuffer",
+        timeout: options.timeout,
+        signal: options.signal,
+    });
+    const bytes = Buffer.from(response.data);
+    if (bytes.subarray(0, ZIP_SIGNATURE.length).toString("latin1") !== ZIP_SIGNATURE) {
+        throw new Error(`${options.url} answered ${bytes.byteLength} bytes that do not begin with a zip signature, ` +
+            `so the body is not the archive`);
+    }
+    await writeLocalFile(bytes, options.dest);
+    // A stubbed transport may answer without headers at all, so the served modification
+    // time is read defensively rather than indexed.
+    const headers = response.headers;
+    return {
+        bytes: bytes.byteLength,
+        sha256: sha256Hex(bytes),
+        lastModified: headers?.["last-modified"] ?? null,
+    };
+}
+//# sourceMappingURL=zip-archive.js.map

package/out/tools/fetch/zip-archive.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"zip-archive.js","sourceRoot":"","sources":["../../../lib/tools/fetch/zip-archive.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAGhD;;;;;;GAMG;AACH,MAAM,aAAa,GAAG,IAAI,CAAA;AA8B1B;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,MAAgC,EAChC,OAAkC;IAElC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CAAc;QAChD,MAAM,EAAE,KAAK;QACb,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,YAAY,EAAE,aAAa;QAC3B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;KACtB,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IAExC,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,aAAa,EAAE,CAAC;QAClF,MAAM,IAAI,KAAK,CACd,GAAG,OAAO,CAAC,GAAG,aAAa,KAAK,CAAC,UAAU,iDAAiD;YAC3F,gCAAgC,CACjC,CAAA;IACF,CAAC;IAED,MAAM,cAAc,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;IAEzC,oFAAoF;IACpF,gDAAgD;IAChD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAA6C,CAAA;IAEtE,OAAO;QACN,KAAK,EAAE,KAAK,CAAC,UAAU;QACvB,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,YAAY,EAAE,OAAO,EAAE,CAAC,eAAe,CAAC,IAAI,IAAI;KAChD,CAAA;AACF,CAAC"}