@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/fetch/vlaanderen-ad.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vlaanderen-ad.js","sourceRoot":"","sources":["../../../lib/tools/fetch/vlaanderen-ad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AAEvC,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AACpG,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAE,qBAAqB,EAA0B,MAAM,iCAAiC,CAAA;AAE/F,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAA;AAEzE;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,sCAAsC,CAAA;AAEvE;;;;;;GAMG;AACH,MAAM,aAAa,GAAG,kCAAkC,CAAA;AAExD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,MAAM,CAAA;AAEnC;;;;;;;;;GASG;AACH,MAAM,eAAe,GAAG;IACvB,gBAAgB,EAAE,mBAAmB;IACrC,gBAAgB,EAAE,mBAAmB;IACrC,aAAa,EAAE,iBAAiB;CACvB,CAAA;AAyBV;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,MAAgC,EAChC,MAAwC;IAExC,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,EAAE,CAAC,CAAA;IAEjH,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAAA;AACnG,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,MAAgC,EAAE,QAAgB;IAClG,8FAA8F;IAC9F,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAA;IAE3G,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CACd,yCAAyC,QAAQ,4CAA4C;YAC5F,gDAAgD,CACjD,CAAA;IACF,CAAC;IAED,+FAA+F;IAC/F,4FAA4F;IAC5F,OAAO,IAAI,CAAC,aAAa,CAAA;AAC1B,CAAC;AA8BD;;;;;;;;GAQG;AACH;;;;;;GAMG;AACH,MAAM,sBAAsB,GAAG,IAAI,CAAA;AAUnC;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,WAAW,CAAC,IAAiB,EAAE,IAAY;IACzD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAA;IAC3C,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,IAAI,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;IACxG,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,IAAI,IAAI,qBAAqB,EAAE,GAAG,CAAC,CAAA;IAE9D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAElC,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;QAEpE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC;YAAE,SAAQ;QAE3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,GAAG,CAAC,CAAA;QAErD,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAA;YACnD,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,CAAC,CAAA;YAE7E,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE;gBAChG,GAAG,EAAE,IAAI;aACT,CAAC,CAAA;YAEF,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;gBAAE,SAAQ;YAEhE,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;QACxE,CAAC;gBAAS,CAAC;YACV,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACrB,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,MAAgC,EAChC,OAAiC;IAEjC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,aAAa,EAAE,aAAa,CAAC,CAAA;IAC3E,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,IAAI,aAAa,EAAE,aAAa,CAAC,CAAA;IAE7F,MAAM,eAAe,CAAC,IAAI,CAAC,CAAA;IAE3B,MAAM,cAAc,GAAoC;QACvD,gBAAgB,EAAE,EAAE;QACpB,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,EAAE;KACjB,CAAA;IAED,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAA8B,EAAE,CAAC;QAC7F,MAAM,KAAK,GAAG,MAAM,0BAA0B,CAAC,MAAM,EAAE,MAAM,QAAQ,EAAE,CAAC,CAAA;QACxE,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QAC9C,IAAI,IAAI,GAAG,CAAC,CAAA;QAEZ,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,KAAK,EAAE,UAAU,IAAI,iBAAiB,EAAE,CAAC;YAC9E,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YAErC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACxB,cAAc,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBACxC,IAAI,IAAI,IAAI,CAAC,cAAc,CAAA;gBAE3B,SAAQ;YACT,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE;gBAC7C,SAAS,EAAE,MAAM,QAAQ,EAAE;gBAC3B,KAAK,EAAE,MAAM,CAAC,iBAAiB,CAAC;gBAChC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;aAC9B,CAAC,CAAA;YAEF,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CACd,6BAA6B,QAAQ,uBAAuB,UAAU,aAAa;oBAClF,+DAA+D,CAChE,CAAA;YACF,CAAC;YAED,IAAI,IAAI,CAAC,cAAc,KAAK,CAAC;gBAAE,MAAK;YAEpC,MAAM,IAAI,GAAG,KAAK,IAAI,iBAAiB,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAA;YAE/G,MAAM,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;YAE/C,cAAc,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACnC,IAAI,IAAI,IAAI,CAAC,cAAc,CAAA;QAC5B,CAAC;QAED,oFAAoF;QACpF,sFAAsF;QACtF,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACd,yBAAyB,QAAQ,aAAa,IAAI,iBAAiB;gBAClE,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG;gBACrC,uCAAuC,KAAK,kCAAkC,CAC/E,CAAA;QACF,CAAC;IACF,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,0BAA0B,CAAC,MAAM,EAAE,YAAY,CAAC,CAAA;IAC3E,MAAM,YAAY,GAAwE,EAAE,CAAA;IAC5F,IAAI,WAAW,GAAG,EAAE,CAAA;IAEpB,iDAAiD;IACjD,sFAAsF;IACtF,2FAA2F;IAC3F,4FAA4F;IAC5F,+FAA+F;IAC/F,wFAAwF;IACxF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;IAEjD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,YAAY,EAAE,UAAU,IAAI,QAAQ,EAAE,CAAC;QAC5E,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAK;QAElC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,YAAY,CAAC,MAAM,IAAI,OAAO,CAAC,QAAQ;YAAE,MAAK;QAEpF,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QAEvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC5B,YAAY,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAA;YAE/F,SAAQ;QACT,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE;YAC7C,SAAS,EAAE,YAAY;YACvB,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC;YACvB,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;SAC9B,CAAC,CAAA;QAEF,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CACd,6CAA6C,UAAU,wCAAwC;gBAC9F,oCAAoC,CACrC,CAAA;QACF,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,KAAK,CAAC;YAAE,MAAK;QAEpC,MAAM,IAAI,GAAG,WAAW,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAA;QAElE,MAAM,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QAE/C,YAAY,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAA;QAE5E,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,IAAI,WAAW,CAAA;IAClF,CAAC;IAED,MAAM,OAAO,GAAsB;QAClC,OAAO,EAAE,iBAAiB;QAC1B,UAAU,EAAE,WAAW;QACvB,YAAY,EAAE,aAAa;QAC3B,WAAW;QACX,YAAY;QACZ,UAAU,EAAE,cAAc;QAC1B,YAAY;KACZ,CAAA;IAED,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,CAAA;IAEvD,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAoC;IACvE,OAAO,OAAO,CAAC,qBAAqB,CAAC,CAAA;AACtC,CAAC;AAYD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,OAA+B,EAC/B,MAA+B;IAE/B,YAAY,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,qBAAqB,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAA;IAExD,MAAM,EAAE,CAAC,OAAO,qBAAqB,gBAAgB,iBAAiB,EAAE,CAAC,CAAA;IAEzE,yFAAyF;IACzF,0DAA0D;IAC1D,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;QACjD,SAAS;QACT,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;KACtB,CAAC,CAAA;IAEF,MAAM,EAAE,CACP,OAAO,OAAO,CAAC,YAAY,mBAAmB,OAAO,CAAC,YAAY,CAAC,MAAM,UAAU;QAClF,cAAc,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC;aAC9C,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;aACjD,IAAI,CAAC,GAAG,CAAC,EAAE,CACd,CAAA;IAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/tools/fetch/wfs-harvest.d.ts

@@ -0,0 +1,410 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * The paging, manifest and resumption machinery a paged WFS harvest shares.
+ *
+ * `./inspire-addresses.ts` owns what an INSPIRE Addresses service exposes — its capabilities, its
+ * feature-type names and its GeoJSON pages. This module owns what a harvest of any paged WFS does
+ * with those pages: it appends each one to the single file an adapter's `inputPath` points at,
+ * records the page in a manifest beside it, and resumes from that manifest after an interrupted run.
+ *
+ * One file rather than one file per page, because the adapters read one. `#ee/adapters/ads/adapter`
+ * opens `opts.inputPath` as newline-delimited JSON and `#pl/adapters/emuia/adapter` opens it as one
+ * markup stream, so a directory of pages reaches neither. The manifest therefore records each page's
+ * byte range inside that file: `offset`, `bytes` and the sha256 of exactly those bytes. A range is
+ * what makes a page verifiable after it has been concatenated into its neighbors, and what lets an
+ * interrupted append be truncated back to a page boundary rather than discarding the harvest.
+ *
+ * Three properties of a WFS make this harder than reading pages until they run out, and each is
+ * handled here rather than by each country's module:
+ *
+ * 1. A service may ignore `startIndex` and answer every page with the first. `countWFSFeaturesByPaging`
+ *    in `@mailwoman/core/api` refuses such a service by comparing two pages' leading feature. A
+ *    harvest sees the same defect as two consecutive pages with equal bytes, which this refuses.
+ * 2. A service's own `numberMatched` may be a per-request cap rather than a count. The caller states
+ *    what its count is and where it came from, and {@linkcode PagedWFSHarvestOptions.featureCount}
+ *    carries both. No count is written into this module, and a harvest with no usable count ends on
+ *    the first page that returns no features.
+ * 3. A page may return fewer features than `count` asks for. The next `startIndex` therefore advances
+ *    by the features the service returned rather than by the page size, so a service that caps a page
+ *    below the requested size is read whole instead of being read in strides that skip features.
+ */
+import { APIClient, type CheckedWFSFeatureCount } from "@mailwoman/core/api";
+import { type PathBuilderLike } from "path-ts";
+import { type FetchSummary } from "#tools/fetch/download";
+/**
+ * The WFS version this repository's harvesters speak.
+ *
+ * 2.0.0 defines `startIndex` paging and the `hits` result type, and it is what
+ * every service measured for these modules advertises.
+ */
+export declare const WFS_VERSION = "2.0.0";
+/**
+ * The file name a harvest writes its manifest under, beside the data file.
+ */
+export declare const HARVEST_MANIFEST_FILE = "MANIFEST.json";
+/**
+ * The first element of a response, as the service wrote it.
+ */
+export interface MarkupRoot {
+    /**
+     * The opening tag verbatim, `<wfs:FeatureCollection …>`.
+     */
+    tag: string;
+    /**
+     * The element's qualified name, prefix included.
+     */
+    name: string;
+    /**
+     * The text between the name and the tag's terminator, which holds the attributes.
+     */
+    attributes: string;
+    /**
+     * The offset just past the opening tag.
+     */
+    contentStart: number;
+}
+/**
+ * The count an opening tag states, or `null` where it states none or declines to state one.
+ *
+ * WFS 2.0 permits `numberMatched="unknown"`, which declines to count rather than counting none.
+ */
+export declare function rootCount(root: MarkupRoot, attribute: string): number | null;
+/**
+ * The document's first element tag.
+ *
+ * @throws When the body holds no element tag, so a response that is not markup
+ * reads as unreadable rather than as a page of no features.
+ */
+export declare function readMarkupRoot(body: string, context: string): MarkupRoot;
+/**
+ * The content between a document's root tags, which for a `wfs:FeatureCollection` is its members.
+ *
+ * @throws When the root element is not closed, so a truncated transfer reads as
+ * unreadable rather than as a page of no features.
+ */
+export declare function rootContent(body: string, root: MarkupRoot, context: string): string;
+/**
+ * A page's root tag with its namespace declarations kept and its per-page attributes
+ * dropped, so it can open a file holding every page.
+ */
+export declare function collectionOpeningTag(root: MarkupRoot): string;
+/**
+ * One `GetFeature` response in a markup format, with what its root states about itself.
+ */
+export interface WFSMarkupPage {
+    body: string;
+    root: MarkupRoot;
+    /**
+     * The `numberReturned` the root states, or `null` where it states none.
+     */
+    numberReturned: number | null;
+    /**
+     * The `numberMatched` the root states, or `null` where it answers `unknown`.
+     */
+    numberMatched: number | null;
+    /**
+     * The `timeStamp` the root states, or `null` where it states none.
+     */
+    timeStamp: string | null;
+}
+/**
+ * Issues one `GetFeature` and reads the counts off the root element.
+ *
+ * @throws When the service answers an OGC exception report, so an HTTP 400 carrying
+ * `ows:ExceptionCode` raises rather than being written into the harvest as a page.
+ */
+export declare function readWFSMarkupPage(client: Pick<APIClient, "fetch">, options: {
+    wfsURL: string;
+    context: string;
+    params: Readonly<Record<string, string>>;
+}): Promise<WFSMarkupPage>;
+/**
+ * What a harvest asked the service for.
+ *
+ * Recorded in the manifest and compared against a resumed run's request: a harvest whose page
+ * size, ordering or format changed cannot be continued from pages taken under the old one.
+ */
+export interface WFSHarvestRequest {
+    wfsURL: string;
+    typeName: string;
+    outputFormat: string;
+    /**
+     * The property the service was asked to order by, or `null` where it grants no ordering.
+     *
+     * Without one, a resumed harvest rests on the service returning the same features
+     * in the same order as the earlier run, which no WFS guarantees.
+     */
+    sortBy: string | null;
+}
+/**
+ * A feature count and how it was obtained.
+ *
+ * `count` is `null` where the service states no usable count.
+ * The harvest then ends on the first page that returns no features, and
+ * `because` records what the service said instead.
+ */
+export interface FeatureCountStatement {
+    count: number | null;
+    because: string;
+}
+/**
+ * The count a service states, or no count and the reason there is none.
+ *
+ * `readCheckedWFSFeatureCount` proves a reported `numberMatched` against a page of the
+ * same type, which catches a service whose count is smaller than one of its own pages.
+ * It cannot catch a count that equals the service's page cap: no page can return more
+ * features than the cap, so the page it asks for agrees with the cap every time.
+ *
+ * Poland's service is that case.
+ * Measured 2026-10-02, `resultType=hits` at `startIndex=1` reports `numberMatched="1000"`, its
+ * capabilities document advertises `CountDefault` as 1000, and a page returns at most 1,000 features.
+ *
+ * The reported number is the cap, and a harvest that read it as a count would store 1,000
+ * of the 8.6 million features the service holds and record the type as read whole.
+ * A reported count equal to the cap is therefore carried as no count.
+ *
+ * @param pageCap The largest page the service will serve, from its advertised `CountDefault`.
+ */
+export declare function statedFeatureCount(checked: CheckedWFSFeatureCount, pageCap: number | null): FeatureCountStatement;
+/**
+ * One page's contribution to the harvest file.
+ */
+export interface HarvestedPayload {
+    /**
+     * The text appended to the harvest file for this page.
+     */
+    payload: string;
+    /**
+     * Text written once, before the first page's payload — a document prolog
+     * and root tag, where the format needs one.
+     *
+     * Read from the first page a run writes and recorded in the manifest.
+     */
+    header?: string;
+    /**
+     * Text written once the harvest reaches the end of the feature type, such as a closing root tag.
+     */
+    footer?: string;
+    /**
+     * The features the service returned, which is what the next `startIndex` advances by.
+     *
+     * `null` where the service stated none.
+     * The harvest refuses that rather than reading it as zero.
+     */
+    numberReturned: number | null;
+    numberMatched: number | null;
+    retrievedAt: string | null;
+}
+/**
+ * One page as the manifest records it.
+ *
+ * `offset` and `bytes` locate the page inside the harvest file and `sha256` is the digest of
+ * exactly those bytes, so a page stays verifiable after the pages around it were appended.
+ */
+export interface WFSHarvestPageRecord {
+    start_index: number;
+    /**
+     * The page size requested, which a service may answer with fewer features.
+     */
+    count: number;
+    offset: number;
+    bytes: number;
+    sha256: string;
+    number_returned: number;
+    /**
+     * The `numberMatched` this page stated, where the service states a real one.
+     */
+    number_matched: number | null;
+    retrieved_at: string | null;
+}
+/**
+ * The manifest a paged harvest writes beside its data file.
+ *
+ * Written after every page, so an interrupted run resumes from the last page that reached disk.
+ */
+export interface WFSHarvestManifest {
+    source: string;
+    source_url: string;
+    wfs_version: string;
+    type_name: string;
+    output_format: string;
+    sort_by: string | null;
+    license: string;
+    attribution: string;
+    downloaded_at: string;
+    /**
+     * The file an adapter's `inputPath` points at, beside this manifest.
+     */
+    filename: string;
+    page_size: number;
+    /**
+     * The service's own count where it states a usable one, and `null` where it does not.
+     */
+    feature_count: number | null;
+    feature_count_source: string;
+    features_written: number;
+    header_bytes: number;
+    /**
+     * The text that closes the data file, appended once the harvest reaches the end of the type.
+     */
+    footer: string;
+    /**
+     * The data file's byte length as this manifest describes it, footer included once complete.
+     */
+    bytes: number;
+    /**
+     * The whole data file's digest, once the harvest is complete.
+     *
+     * `null` on a partial harvest, where the file is still being appended to
+     * and a digest of it would describe a prefix rather than the harvest.
+     */
+    sha256: string | null;
+    complete: boolean;
+    pages: WFSHarvestPageRecord[];
+}
+export interface PagedWFSHarvestOptions {
+    /**
+     * Names the service in every refusal.
+     */
+    context: string;
+    /**
+     * The adapter slug this harvest feeds, stamped into the manifest.
+     */
+    source: string;
+    /**
+     * Where the harvest is written.
+     *
+     * The adapter reads {@linkcode PagedWFSHarvestOptions.filename} inside this directory.
+     */
+    outputDir: PathBuilderLike;
+    filename: string;
+    request: WFSHarvestRequest;
+    /**
+     * The license the address-source register elected for this publisher.
+     */
+    license: string;
+    /**
+     * The attribution the elected terms require, or an empty string where they require none.
+     */
+    attribution: string;
+    featureCount: FeatureCountStatement;
+    pageSize: number;
+    /**
+     * Stop once the manifest holds this many pages, for a probe rather than a full harvest.
+     *
+     * A cap on the manifest rather than on the run, so a second run under the same cap makes no request.
+     */
+    maxPages?: number;
+    signal?: AbortSignal;
+    /**
+     * Reads one page and states the bytes the harvest file receives for it.
+     */
+    readPage: (request: {
+        startIndex: number;
+        count: number;
+    }) => Promise<HarvestedPayload>;
+    report?: (line: string) => void;
+}
+/**
+ * What a caller knows about the request it is about to make before it has asked the service.
+ *
+ * The type name and the output format come from a capabilities document, so they are not here:
+ * the point of this check is to answer whether a harvest is already current without sending a request.
+ */
+export interface HarvestCurrencyExpectation {
+    wfsURL: string;
+    sortBy: string | null;
+    pageSize: number;
+}
+/**
+ * The manifest of a harvest that needs no further request, or `null` where one does.
+ *
+ * A caller reads this before it reads a capabilities document, so a current harvest costs no request at all.
+ * Two harvests need none:
+ *
+ * - A complete one whose data file still hashes to the digest the manifest records.
+ * - A partial one that already holds `maxPages` pages, which is what a bounded probe asks for.
+ *
+ * @throws When a page's recorded byte range no longer hashes to its recorded digest,
+ * so a file that was altered under its manifest is reported rather than used.
+ */
+export declare function readCurrentHarvest(options: {
+    outputDir: PathBuilderLike;
+    filename: string;
+    context: string;
+    expect: HarvestCurrencyExpectation;
+    maxPages?: number;
+}): Promise<WFSHarvestManifest | null>;
+/**
+ * Harvests one WFS feature type into a single file, recording each page in a manifest beside it.
+ *
+ * Re-runnable in three ways, each of which makes no request:
+ *
+ * - A complete harvest whose file still hashes to the manifest's `sha256`.
+ * - A partial harvest that already holds {@linkcode PagedWFSHarvestOptions.maxPages} pages.
+ * - A partial harvest resumes at the page after the last one on disk,
+ *   so the pages already taken are not taken again.
+ *
+ * @returns The manifest as written.
+ */
+export declare function harvestPagedWFS(options: PagedWFSHarvestOptions): Promise<WFSHarvestManifest>;
+/**
+ * What a registered `mailwoman corpus fetch` entry needs to run one service's harvest.
+ */
+export interface RunWFSHarvestOptions {
+    /**
+     * The slug the harvest is written under and reported by, which matches the adapter's own id.
+     */
+    slug: string;
+    /**
+     * Where the harvest directory sits, as the fetch registry hands it over.
+     */
+    outRoot: (segment: string) => PathBuilderLike;
+    /**
+     * The data file inside the harvest directory, which is also the adapter's `inputPath`.
+     */
+    filename: string;
+    /**
+     * What a stored manifest has to agree with before its harvest counts as current.
+     */
+    expect: HarvestCurrencyExpectation;
+    /**
+     * The name the client reports itself under.
+     */
+    displayName: string;
+    /**
+     * The floor between two requests to one government host.
+     */
+    minRequestIntervalMs: number;
+    /**
+     * Runs the harvest against a client the caller does not own.
+     */
+    harvest: (client: Pick<APIClient, "fetch">) => Promise<WFSHarvestManifest>;
+    /**
+     * A client to use rather than constructing one, which a test supplies.
+     */
+    client?: Pick<APIClient, "fetch">;
+    /**
+     * A page cap, which makes an incomplete harvest a success.
+     */
+    maxPages?: number;
+}
+/**
+ * Runs one service's harvest and reports it as a fetch registry entry does.
+ *
+ * Estonia and Poland wrote this sequence twice: ask whether the harvest on disk is
+ * already current, construct a paced client unless the caller supplied one, run,
+ * report, and decide whether an incomplete harvest is a failure.
+ * A third WFS source would have written it a third time.
+ *
+ * The currency check runs before any request, so a complete harvest or a bounded
+ * probe that already holds its pages makes no request at all.
+ *
+ * A bounded run stops short by instruction rather than by failure, so a page cap the caller
+ * asked for reads as a success while an incomplete harvest without one reads as a failure.
+ */
+export declare function runWFSHarvest(options: RunWFSHarvestOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=wfs-harvest.d.ts.map

package/out/tools/fetch/wfs-harvest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wfs-harvest.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/wfs-harvest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EAAE,SAAS,EAA+B,KAAK,sBAAsB,EAAE,MAAM,qBAAqB,CAAA;AAOzG,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAE3D,OAAO,EAA+B,KAAK,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEtF;;;;;GAKG;AACH,eAAO,MAAM,WAAW,UAAU,CAAA;AAElC;;GAEG;AACH,eAAO,MAAM,qBAAqB,kBAAkB,CAAA;AAEpD;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IACX;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;CACpB;AAsBD;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAI5E;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,UAAU,CAexE;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAWnF;AAWD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAE7D;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,UAAU,CAAA;IAChB;;OAEG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B;;OAEG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B;;OAEG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACtC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CAAE,GACpF,OAAO,CAAC,aAAa,CAAC,CAmBxB;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,YAAY,EAAE,MAAM,CAAA;IACpB;;;;;OAKG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,qBAAqB;IACrC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,sBAAsB,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,qBAAqB,CAgBjH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;;OAKG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACpC,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,eAAe,EAAE,MAAM,CAAA;IACvB;;OAEG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;CAC3B;AAED;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,oBAAoB,EAAE,MAAM,CAAA;IAC5B,gBAAgB,EAAE,MAAM,CAAA;IACxB,YAAY,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;;;;OAKG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,QAAQ,EAAE,OAAO,CAAA;IACjB,KAAK,EAAE,oBAAoB,EAAE,CAAA;CAC7B;AAED,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;;;OAIG;IACH,SAAS,EAAE,eAAe,CAAA;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,iBAAiB,CAAA;IAC1B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,qBAAqB,CAAA;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB;;OAEG;IACH,QAAQ,EAAE,CAAC,OAAO,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACvF,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AA+JD;;;;;GAKG;AACH,MAAM,WAAW,0BAA0B;IAC1C,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE;IACjD,SAAS,EAAE,eAAe,CAAA;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,0BAA0B,CAAA;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CA2CrC;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA6JlG;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IAEZ;;OAEG;IACH,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,eAAe,CAAA;IAE7C;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAEhB;;OAEG;IACH,MAAM,EAAE,0BAA0B,CAAA;IAElC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IAEnB;;OAEG;IACH,oBAAoB,EAAE,MAAM,CAAA;IAE5B;;OAEG;IACH,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,kBAAkB,CAAC,CAAA;IAE1E;;OAEG;IACH,MAAM,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAEjC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,aAAa,CAClC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA6CvB"}