@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/fetch/ourairports.js

@@ -3,62 +3,29 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Fetch the OurAirports CSV dumps — the VENUE side of the sub-venue corpus arc (#35).
+ * Fetch the OurAirports CSV dumps — the venue side of the sub-venue corpus arc.
  *
- *   Source : https://davidmegginson.github.io/ourairports-data/ (the project's own GitHub Pages
- *            mirror of the nightly export; `ourairports.com/data/` redirects here).
- *   License: PUBLIC DOMAIN. OurAirports places its data in the public domain and asks only for a
- *            courtesy credit — no attribution obligation rides on a derived slice, which makes this
- *            the one transport source in the arc with no licensing question at all. Tier A.
- *
- *   ## What it is good for, and what it is not
- *
- *   `airports.csv` is every airport on earth with ICAO/IATA codes, coordinates, `municipality`, and
- *   `iso_country` — 12.7 MB, ~83,000 rows as of 2026-08-04. That is the CONTAINING VENUE for a
- *   `<sub-venue>, <venue>, <street>, <locality>, <postcode>` corpus line, and it is better at that job
- *   than OSM: every row is named, the name is canonical, and `municipality` gives the locality without
- *   a spatial join.
- *
- *   It carries NO interior structure. There is no terminal, concourse, gate or pier table — the
- *   corpus task says as much ("Good for the venue side of each pair, weaker on interior structure")
- *   and a row-level read confirms it. Pair it with the OSM `aeroway` extractor
- *   (`@mailwoman/osm/sdk`'s `extractOSMSubVenues`), which is where the sub-venue half comes from.
- *
- *   ## Why `downloadToFile` and not `APIClient`
- *
- *   `AGENTS.md` routes HTTP through `APIClient`, and that rule is about API REQUESTS — small bodies,
- *   repeated calls, rate-limited hosts. This is four static file transfers against a GitHub Pages CDN
- *   with no rate limit and nothing to pace, run once per refresh. It uses the same `downloadToFile`
- *   every other module in this `fetch/` family uses, which is where the retry and timeout live.
- *   The Wikidata sibling (`wikidata-subvenue.ts`) IS an API client and is built on `APIClient`
- *   accordingly; the split between the two is the one `AGENTS.md` draws.
- *
- *   Invoke via `mailwoman corpus fetch ourairports --out-root <path>`.
+ * Source: https://davidmegginson.github.io/ourairports-data
+ * License: public domain, with a courtesy credit requested.
  */
 import { APIClient } from "@mailwoman/core/api";
 import { makeDirectories } from "@mailwoman/core/fs/writers";
 import { sha256File } from "@mailwoman/core/hash";
-import { join } from "path-ts";
 import { downloadToFile, writeManifest } from "#tools/fetch/download";
 const SLUG = "ourairports";
 /**
- * The GitHub Pages mirror the project itself publishes. `ourairports.com/data/*.csv` 302s here, so pointing at the
- * mirror directly saves a redirect and is the URL the project's own README gives.
+ * `ourairports.com/data/*.csv` redirects to the project's own GitHub Pages mirror.
  */
 const BASE_URL = "https://davidmegginson.github.io/ourairports-data";
 /**
- * The files worth having, and why each one.
+ * `airports.csv` is the payload.
  *
- * `airports.csv` is the payload. The other three are small joins that turn its codes into text: `countries.csv` and
- * `regions.csv` expand `iso_country`/`iso_region` into names (a corpus line needs "Germany", not "DE"), and
- * `runways.csv` is the only file carrying per-airport sub-structure of any kind — runway designators, which are NOT
- * sub-venue designators (nobody addresses mail to a runway) but are worth having on disk as the negative class if the
- * slice ever needs one.
+ * The other three join its codes to text or provide the negative class.
  */
 const FILES = ["airports.csv", "countries.csv", "regions.csv", "runways.csv"];
 /**
- * Read the upstream `Last-Modified` with a HEAD. Returns `null` on any failure — provenance metadata is nice to have
- * and must never fail a download that otherwise succeeded.
+ * @returns `null` on any failure, because provenance metadata must never fail
+ * a download that otherwise succeeded.
  */
 async function readLastModified(url) {
     try {
@@ -74,11 +41,11 @@ async function readLastModified(url) {
     }
 }
 /**
- * Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `MANIFEST.json` carrying each file's
- * origin URL, sha256, byte count and upstream `Last-Modified`.
+ * Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `manifest.json`
+ * carrying each file's origin URL, sha256, byte count and upstream `Last-Modified`.
  */
 export async function fetchOurAirports(options, report) {
-    const destDir = join(options.outRoot, SLUG);
+    const destDir = options.outRoot(SLUG);
     await makeDirectories(destDir);
     const entries = [];
     const failedCodes = [];
@@ -86,7 +53,7 @@ export async function fetchOurAirports(options, report) {
     let failed = 0;
     for (const filename of FILES) {
         const url = `${BASE_URL}/${filename}`;
-        const dest = join(destDir, filename);
+        const dest = destDir(filename);
         report?.(`=== ${SLUG} / ${filename}`);
         try {
             const [{ bytes }, lastModified] = await Promise.all([
@@ -122,7 +89,7 @@ export async function fetchOurAirports(options, report) {
         downloaded_at: new Date().toISOString(),
         files: entries,
     };
-    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
+    await writeManifest(destDir("MANIFEST.json"), manifest);
     return { fetched, skipped: 0, failed, failedCodes };
 }
 //# sourceMappingURL=ourairports.js.map

package/out/tools/fetch/ourairports.js.map

@@ -1 +1 @@
-{"version":3,"file":"ourairports.js","sourceRoot":"","sources":["../../../lib/tools/fetch/ourairports.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAErE,MAAM,IAAI,GAAG,aAAa,CAAA;AAE1B;;;GAGG;AACH,MAAM,QAAQ,GAAG,mDAAmD,CAAA;AAEpE;;;;;;;;GAQG;AACH,MAAM,KAAK,GAAG,CAAC,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,aAAa,CAAU,CAAA;AAyBtF;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,GAAW;IAC1C,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,MAAM,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;YAClF,GAAG;YACH,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,MAAM;SACf,CAAC,CAAA;QAEF,OAAQ,GAAG,CAAC,OAAO,CAAC,eAAe,CAAwB,IAAI,IAAI,CAAA;IACpE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,OAAgC,EAChC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAA;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,CAAC;YACJ,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACnD,cAAc,CAAC;oBACd,GAAG;oBACH,IAAI;oBACJ,SAAS,EAAE,OAAO;oBAClB,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;oBAC1C,MAAM;iBACN,CAAC;gBACF,gBAAgB,CAAC,GAAG,CAAC;aACrB,CAAC,CAAA;YAEF,OAAO,CAAC,IAAI,CAAC;gBACZ,QAAQ;gBACR,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC;gBAC9B,KAAK;gBACL,aAAa,EAAE,YAAY;aAC3B,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,KAAK,QAAQ,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACpF,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAE1B,MAAM,EAAE,CAAA;QACT,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAwB;QACrC,MAAM,EAAE,aAAa;QACrB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,2CAA2C;QACpD,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}
+{"version":3,"file":"ourairports.js","sourceRoot":"","sources":["../../../lib/tools/fetch/ourairports.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAGjD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAErE,MAAM,IAAI,GAAG,aAAa,CAAA;AAE1B;;GAEG;AACH,MAAM,QAAQ,GAAG,mDAAmD,CAAA;AAEpE;;;;GAIG;AACH,MAAM,KAAK,GAAG,CAAC,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,aAAa,CAAU,CAAA;AAyBtF;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,GAAW;IAC1C,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,MAAM,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;YAClF,GAAG;YACH,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,MAAM;SACf,CAAC,CAAA;QAEF,OAAQ,GAAG,CAAC,OAAO,CAAC,eAAe,CAAwB,IAAI,IAAI,CAAA;IACpE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,OAAgC,EAChC,MAA+B;IAE/B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACrC,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAA;QACrC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;QAE9B,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,CAAC;YACJ,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACnD,cAAc,CAAC;oBACd,GAAG;oBACH,IAAI;oBACJ,SAAS,EAAE,OAAO;oBAClB,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;oBAC1C,MAAM;iBACN,CAAC;gBACF,gBAAgB,CAAC,GAAG,CAAC;aACrB,CAAC,CAAA;YAEF,OAAO,CAAC,IAAI,CAAC;gBACZ,QAAQ;gBACR,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC;gBAC9B,KAAK;gBACL,aAAa,EAAE,YAAY;aAC3B,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,KAAK,QAAQ,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACpF,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAE1B,MAAM,EAAE,CAAA;QACT,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAwB;QACrC,MAAM,EAAE,aAAa;QACrB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,2CAA2C;QACpD,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAEvD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/tools/fetch/ppd.d.ts

@@ -6,9 +6,10 @@ export interface PPDExtractStats {
     skippedNoPostcode: number;
 }
 /**
- * Convert PPD rows (`id,price,date,postcode,type,new,tenure,PAON,SAON,street,locality,town,district,county,cat,status`
- * — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing. Accepts a plain array
- * of rows (tests) or a streamed async source (the real 31M-row extraction).
+ * Convert PPD rows (`id,price,date,postcode,type,new,tenure,paon,saon,street,locality,town,district,county,cat,status`
+ * — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing.
+ *
+ * Accepts a plain array of rows (tests) or a streamed async source (the real 31M-row extraction).
  */
 export declare function extractPPDTuples(input: AsyncIterable<string[]> | Iterable<string[]>, write: (line: string) => void): Promise<PPDExtractStats>;
 //# sourceMappingURL=ppd.d.ts.map

package/out/tools/fetch/ppd.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ppd.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/ppd.ts"],"names":[],"mappings":"AAWA,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;CACzB;AAID;;;;GAIG;AACH,wBAAsB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EACnD,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC3B,OAAO,CAAC,eAAe,CAAC,CAwD1B"}
+{"version":3,"file":"ppd.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/ppd.ts"],"names":[],"mappings":"AAWA,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;CACzB;AAID;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EACnD,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC3B,OAAO,CAAC,eAAe,CAAC,CAwD1B"}

package/out/tools/fetch/ppd.js

@@ -1,16 +1,18 @@
+import { titleCaseGB } from "@mailwoman/codex/gb";
 import { dataRootPath } from "@mailwoman/core/data-root";
 import { openReadStream, openWriteStream } from "@mailwoman/core/fs/streams";
+import { stringifyJSON } from "@mailwoman/core/json";
 import { runIfScript } from "@mailwoman/core/scripting";
 import { parseArguments } from "@mailwoman/core/scripting/arguments";
 import { resolvePath } from "path-ts";
 import { CSVSpliterator } from "spliterator";
-import { titleCaseGB } from "#tools/gb-title-case";
 const HOUSE_NUMBER_PATTERN = /^\d+[A-Za-z]?(\s*-\s*\d+[A-Za-z]?)?$/;
 const quote = (value) => (value ? `"${value.replaceAll('"', '""')}"` : "");
 /**
- * Convert PPD rows (`id,price,date,postcode,type,new,tenure,PAON,SAON,street,locality,town,district,county,cat,status`
- * — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing. Accepts a plain array
- * of rows (tests) or a streamed async source (the real 31M-row extraction).
+ * Convert PPD rows (`id,price,date,postcode,type,new,tenure,paon,saon,street,locality,town,district,county,cat,status`
+ * — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing.
+ *
+ * Accepts a plain array of rows (tests) or a streamed async source (the real 31M-row extraction).
  */
 export async function extractPPDTuples(input, write) {
     const stats = { kept: 0, skippedSAON: 0, skippedPAON: 0, skippedNoStreet: 0, skippedNoPostcode: 0 };
@@ -58,12 +60,11 @@ export async function extractPPDTuples(input, write) {
  * Stream `inputPath` (PPD CSV) → `outputPath` (OA-shaped tuples CSV), returning the row-count stats.
  */
 async function runPPDExtract(inputPath, outputPath) {
-    // No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself (see readTuples in
-    // recipes/locale.ts). `header: false` yields every row as data — PPD ships no header row.
+    // No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself
+    // (see readTuples in recipes/international/locale.ts).
+    // `header: false` yields every row as data — PPD ships no header row.
     const rows = CSVSpliterator.fromAsync(openReadStream(inputPath), {
-        mode: "array",
         header: false,
-        enableQuoteHandling: true,
     });
     const out = openWriteStream(outputPath, { encoding: "utf8" });
     const stats = await extractPPDTuples(rows, (line) => out.write(line + "\n"));
@@ -80,6 +81,6 @@ runIfScript(import.meta, async () => {
         },
     });
     const stats = await runPPDExtract(values.input, values.output);
-    console.log(`[ppd] ${JSON.stringify(stats)}`);
+    console.log(`[ppd] ${stringifyJSON(stats)}`);
 });
 //# sourceMappingURL=ppd.js.map

package/out/tools/fetch/ppd.js.map

@@ -1 +1 @@
-{"version":3,"file":"ppd.js","sourceRoot":"","sources":["../../../lib/tools/fetch/ppd.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAA;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAElD,MAAM,oBAAoB,GAAG,sCAAsC,CAAA;AAUnE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;AAE1F;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,KAAmD,EACnD,KAA6B;IAE7B,MAAM,KAAK,GAAoB,EAAE,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACpH,KAAK,CAAC,6CAA6C,CAAC,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QACjC,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACzB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QACtB,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAExB,IAAI,IAAI,EAAE,CAAC;YACV,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,KAAK,CAAC,eAAe,EAAE,CAAA;YAEvB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,KAAK,CAAC,iBAAiB,EAAE,CAAA;YAEzB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAEvE,KAAK,CACJ;YACC,MAAM;YACN,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC;YACX,KAAK,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ;SACR,CAAC,IAAI,CAAC,GAAG,CAAC,CACX,CAAA;QAED,KAAK,CAAC,IAAI,EAAE,CAAA;IACb,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAAC,SAAiB,EAAE,UAAkB;IACjE,gGAAgG;IAChG,0FAA0F;IAC1F,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAW,cAAc,CAAC,SAAS,CAAC,EAAE;QAC1E,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,KAAK;QACb,mBAAmB,EAAE,IAAI;KACzB,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,eAAe,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC7D,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAA;IAE5E,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACb,CAAC,CAAC,CAAA;IAEF,OAAO,KAAK,CAAA;AACb,CAAC;AAED,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,EAAE,GAAG,cAAc,CAAC;QACjC,OAAO,EAAE;YACR,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC,EAAE;YACrG,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC,EAAE;SACpG;KACD,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,KAAM,EAAE,MAAM,CAAC,MAAO,CAAC,CAAA;IAEhE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC,CAAC,CAAA"}
+{"version":3,"file":"ppd.js","sourceRoot":"","sources":["../../../lib/tools/fetch/ppd.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5E,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAA;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,MAAM,oBAAoB,GAAG,sCAAsC,CAAA;AAUnE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;AAE1F;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,KAAmD,EACnD,KAA6B;IAE7B,MAAM,KAAK,GAAoB,EAAE,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACpH,KAAK,CAAC,6CAA6C,CAAC,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QACjC,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACzB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QACtB,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAExB,IAAI,IAAI,EAAE,CAAC;YACV,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,KAAK,CAAC,eAAe,EAAE,CAAA;YAEvB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,KAAK,CAAC,iBAAiB,EAAE,CAAA;YAEzB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAEvE,KAAK,CACJ;YACC,MAAM;YACN,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC;YACX,KAAK,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ;SACR,CAAC,IAAI,CAAC,GAAG,CAAC,CACX,CAAA;QAED,KAAK,CAAC,IAAI,EAAE,CAAA;IACb,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAAC,SAAiB,EAAE,UAAkB;IACjE,6EAA6E;IAC7E,uDAAuD;IACvD,sEAAsE;IACtE,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAW,cAAc,CAAC,SAAS,CAAC,EAAE;QAC1E,MAAM,EAAE,KAAK;KACb,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,eAAe,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC7D,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAA;IAE5E,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACb,CAAC,CAAC,CAAA;IAEF,OAAO,KAAK,CAAA;AACb,CAAC;AAED,WAAW,CAAC,OAAO,IAAI,EAAE,KAAK,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,EAAE,GAAG,cAAc,CAAC;QACjC,OAAO,EAAE;YACR,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC,EAAE;YACrG,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC,EAAE;SACpG;KACD,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,KAAM,EAAE,MAAM,CAAC,MAAO,CAAC,CAAA;IAEhE,OAAO,CAAC,GAAG,CAAC,SAAS,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAC7C,CAAC,CAAC,CAAA"}

package/out/tools/fetch/vlaanderen-ad.d.ts

@@ -0,0 +1,131 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Harvests the INSPIRE Addresses (AD) theme of agentschap Digitaal Vlaanderen into the directory
+ * that `#be/adapters/vlaanderen/adapter` reads.
+ *
+ * `./inspire-addresses.ts` owns what every INSPIRE WFS shares, in GeoJSON. This module is Flanders'
+ * own GML paging, kept apart for three reasons the service forced:
+ *
+ * 1. The component types are small enough to take whole — 167,218 `ad:ThoroughfareName`, 1,190
+ *    `ad:PostalDescriptor`, 609 `ad:AdminUnitName` — so the harvest writes one file per type and the
+ *    adapter joins locally rather than resolving four references per address against 4,563,062 of
+ *    them.
+ * 2. `sortBy=gml:identifier` answers HTTP 504, so the service grants no ordering. The manifest
+ *    records the pages taken and their order, which is what makes a harvest reproducible even though
+ *    a second harvest need not match the first.
+ * 3. A bare `resultType=hits` answers `numberMatched="10000"` whatever `count` asks for. The count
+ *    has to be taken past index 0: `startIndex=1` answers `numberMatched="4563062"`, and the
+ *    bisected tail agrees — `startIndex=4563061` returns one feature and `startIndex=4563062`
+ *    returns none.
+ *
+ * Article 3 of the elected `Modellicentie voor gratis hergebruik Vlaanderen` grants reproduction and
+ * transmission, which is what paging performs, and article 6 states that access is free. The
+ * capabilities document's `ows:Fees` and `ows:AccessConstraints` describe the service rather than
+ * the data, so neither is read here.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { type PathBuilderLike } from "path-ts";
+import { type VlaanderenHarvest } from "#be/adapters/vlaanderen/adapter";
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+/**
+ * The service this module harvests.
+ */
+export declare const VLAANDEREN_AD_WFS = "https://geo.api.vlaanderen.be/ad/wfs";
+/**
+ * The largest page the service will serve.
+ *
+ * A `count=50000` request answers with 10,000 features, so asking for more costs
+ * a request and returns the same body.
+ */
+export declare const MAX_PAGE_SIZE = 10000;
+/**
+ * One `GetFeature` response, with what its root element states about itself.
+ */
+export interface FeaturePageBody {
+    body: string;
+    /**
+     * The `numberReturned` the root states, or `null` where it states none.
+     */
+    numberReturned: number | null;
+    /**
+     * The `numberMatched` the root states, or `null` where it answers `unknown`.
+     *
+     * Every results page of this service answers `unknown`, so only a `resultType=hits`
+     * request past index 0 yields a number.
+     */
+    numberMatched: number | null;
+}
+/**
+ * Issues one `GetFeature` against the service.
+ *
+ * `#tools/fetch/wfs-harvest` owns the request, the exception check and the counts,
+ * which every markup-serving WFS in this directory shares.
+ * This states the service and its context.
+ *
+ * @throws When the service answers an OGC exception report, so a 400 carrying
+ * `ows:ExceptionCode/NoApplicableCode` raises rather than being written into the harvest as a page.
+ */
+export declare function readVlaanderenPage(client: Pick<APIClient, "fetch">, params: Readonly<Record<string, string>>): Promise<FeaturePageBody>;
+/**
+ * The number of features a type holds, asked past index 0.
+ *
+ * @throws When the service states no count, so a missing count never reads as zero features.
+ */
+export declare function readVlaanderenFeatureCount(client: Pick<APIClient, "fetch">, typeName: string): Promise<number>;
+export interface HarvestVlaanderenOptions {
+    /**
+     * Where the harvest is written.
+     * The adapter reads this same directory.
+     */
+    outputDir: PathBuilderLike;
+    /**
+     * Stop after this many address pages, for a probe rather than a full harvest.
+     */
+    maxPages?: number;
+    /**
+     * Features per address page, capped at {@link MAX_PAGE_SIZE}.
+     *
+     * This sizes the address pages only.
+     * A probe lowers it to take a small page without also splitting `ad:ThoroughfareName`
+     * into 167,218 / `pageSize` requests.
+     */
+    pageSize?: number;
+    /**
+     * Features per component page, capped at {@link MAX_PAGE_SIZE} and defaulting to it.
+     *
+     * Only a test lowers this.
+     * A component type is always taken whole, so a smaller page buys requests alone.
+     */
+    componentPageSize?: number;
+    signal?: AbortSignal;
+}
+export declare function harvestVlaanderenAD(client: Pick<APIClient, "fetch">, options: HarvestVlaanderenOptions): Promise<VlaanderenHarvest>;
+/**
+ * The directory `#be/adapters/vlaanderen/adapter` reads, given the root a fetch wrote under.
+ *
+ * The adapter reads the directory rather than one file, because a harvest writes the addresses
+ * and each component type as separate documents beside `harvest.json`.
+ */
+export declare function vlaanderenInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike;
+/**
+ * Per-invocation options for the registry entry.
+ */
+export interface FetchVlaanderenOptions extends BaseFetchOptions {
+    maxPages?: number;
+    pageSize?: number;
+    componentPageSize?: number;
+    signal?: AbortSignal;
+}
+/**
+ * The registry entry.
+ *
+ * `harvestVlaanderenAD` takes a client so a test can drive it against stubbed bodies.
+ * This supplies the client the registry's callers expect, and reports the harvest as a
+ * {@linkcode FetchSummary}: one harvest is one fetched unit whatever the page count,
+ * because the adapter reads the directory rather than any single page.
+ */
+export declare function fetchVlaanderenAD(options: FetchVlaanderenOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=vlaanderen-ad.d.ts.map

package/out/tools/fetch/vlaanderen-ad.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vlaanderen-ad.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/vlaanderen-ad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAG/C,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAG3D,OAAO,EAAyB,KAAK,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AAC/F,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAG3E;;GAEG;AACH,eAAO,MAAM,iBAAiB,yCAAyC,CAAA;AAWvE;;;;;GAKG;AACH,eAAO,MAAM,aAAa,QAAS,CAAA;AAuBnC;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B;;;;;OAKG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CAC5B;AAED;;;;;;;;;GASG;AACH,wBAAsB,kBAAkB,CACvC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GACtC,OAAO,CAAC,eAAe,CAAC,CAI1B;AAED;;;;GAIG;AACH,wBAAsB,0BAA0B,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAcpH;AAED,MAAM,WAAW,wBAAwB;IACxC;;;OAGG;IACH,SAAS,EAAE,eAAe,CAAA;IAC1B;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAiFD,wBAAsB,mBAAmB,CACxC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,wBAAwB,GAC/B,OAAO,CAAC,iBAAiB,CAAC,CA4H5B;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC,GAAG,eAAe,CAEzF;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,gBAAgB;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAED;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACtC,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAyBvB"}

package/out/tools/fetch/vlaanderen-ad.js

@@ -0,0 +1,293 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Harvests the INSPIRE Addresses (AD) theme of agentschap Digitaal Vlaanderen into the directory
+ * that `#be/adapters/vlaanderen/adapter` reads.
+ *
+ * `./inspire-addresses.ts` owns what every INSPIRE WFS shares, in GeoJSON. This module is Flanders'
+ * own GML paging, kept apart for three reasons the service forced:
+ *
+ * 1. The component types are small enough to take whole — 167,218 `ad:ThoroughfareName`, 1,190
+ *    `ad:PostalDescriptor`, 609 `ad:AdminUnitName` — so the harvest writes one file per type and the
+ *    adapter joins locally rather than resolving four references per address against 4,563,062 of
+ *    them.
+ * 2. `sortBy=gml:identifier` answers HTTP 504, so the service grants no ordering. The manifest
+ *    records the pages taken and their order, which is what makes a harvest reproducible even though
+ *    a second harvest need not match the first.
+ * 3. A bare `resultType=hits` answers `numberMatched="10000"` whatever `count` asks for. The count
+ *    has to be taken past index 0: `startIndex=1` answers `numberMatched="4563062"`, and the
+ *    bisected tail agrees — `startIndex=4563061` returns one feature and `startIndex=4563062`
+ *    returns none.
+ *
+ * Article 3 of the elected `Modellicentie voor gratis hergebruik Vlaanderen` grants reproduction and
+ * transmission, which is what paging performs, and article 6 states that access is free. The
+ * capabilities document's `ows:Fees` and `ows:AccessConstraints` describe the service rather than
+ * the data, so neither is read here.
+ */
+import { open } from "node:fs/promises";
+import { APIClient } from "@mailwoman/core/api";
+import { makeDirectories, writeLocalJSONFile, writeLocalTextFile } from "@mailwoman/core/fs/writers";
+import { rootAttribute } from "@mailwoman/core/html/document";
+import { PathBuilder } from "path-ts";
+import { Globerator } from "spliterator/node/fs";
+import { VLAANDEREN_ADAPTER_ID } from "#be/adapters/vlaanderen/adapter";
+import { readWFSMarkupPage, WFS_VERSION } from "#tools/fetch/wfs-harvest";
+/**
+ * The service this module harvests.
+ */
+export const VLAANDEREN_AD_WFS = "https://geo.api.vlaanderen.be/ad/wfs";
+/**
+ * The `outputFormat` the harvest is written in.
+ *
+ * GML is the format whose `ad:component/@xlink:href` the adapter reads.
+ * `application/json` carries the same references under `@href` and serves the same lost characters,
+ * so the choice is a matter of which reader the adapter uses rather than of what the service knows.
+ */
+const OUTPUT_FORMAT = "application/gml+xml; version=3.2";
+/**
+ * The largest page the service will serve.
+ *
+ * A `count=50000` request answers with 10,000 features, so asking for more costs
+ * a request and returns the same body.
+ */
+export const MAX_PAGE_SIZE = 10_000;
+/**
+ * The component feature types a harvest takes, and the file-name stem each is written under.
+ *
+ * Every type is paged, `ad:PostalDescriptor` and `ad:AdminUnitName` included.
+ * Measured 2026-10-02, `ad:ThoroughfareName` holds 167,218 features against a 10,000 page cap,
+ * so it needs 17 pages, while the other two hold 1,190 and 609 and fit in one.
+ *
+ * Writing the small ones as a single file would leave the harvester correct only
+ * until one of them passed the cap.
+ */
+const COMPONENT_STEMS = {
+    ThoroughfareName: "thoroughfare-name",
+    PostalDescriptor: "postal-descriptor",
+    AdminUnitName: "admin-unit-name",
+};
+/**
+ * Issues one `GetFeature` against the service.
+ *
+ * `#tools/fetch/wfs-harvest` owns the request, the exception check and the counts,
+ * which every markup-serving WFS in this directory shares.
+ * This states the service and its context.
+ *
+ * @throws When the service answers an OGC exception report, so a 400 carrying
+ * `ows:ExceptionCode/NoApplicableCode` raises rather than being written into the harvest as a page.
+ */
+export async function readVlaanderenPage(client, params) {
+    const page = await readWFSMarkupPage(client, { wfsURL: VLAANDEREN_AD_WFS, context: "vlaanderen ad wfs", params });
+    return { body: page.body, numberReturned: page.numberReturned, numberMatched: page.numberMatched };
+}
+/**
+ * The number of features a type holds, asked past index 0.
+ *
+ * @throws When the service states no count, so a missing count never reads as zero features.
+ */
+export async function readVlaanderenFeatureCount(client, typeName) {
+    // `startIndex` is what makes the count a count: without it the service answers its 10000 cap.
+    const page = await readVlaanderenPage(client, { typeNames: typeName, resultType: "hits", startIndex: "1" });
+    if (page.numberMatched === null) {
+        throw new Error(`vlaanderen ad wfs: a hits request for ${typeName} at startIndex=1 stated no numberMatched, ` +
+            `so the type's size is unknown rather than zero`);
+    }
+    // `numberMatched` counts the whole query rather than the page, so the skipped first feature is
+    // included: `ad:Address` answers 4563062 here, and `startIndex=4563062` returns no feature.
+    return page.numberMatched;
+}
+/**
+ * Harvests the service into `options.outputDir` and writes the manifest the adapter reads.
+ *
+ * The component types are taken first and whole, so a harvest that stops early
+ * holds address pages whose references all resolve.
+ * Taking addresses first would leave every page unjoinable.
+ *
+ * @returns The manifest as written.
+ */
+/**
+ * How much of a page is read to find its root attributes.
+ *
+ * `numberReturned` sits on the `wfs:FeatureCollection` element, which opens the document.
+ * A page of this service is about 20 MB, and reading all of one to find an attribute in
+ * its first kilobyte would cost 1.6 GB of reads across a whole-region harvest.
+ */
+const PAGE_ROOT_PREFIX_BYTES = 8192;
+/**
+ * The pages a previous run left in the output directory, keyed by their `startIndex`.
+ *
+ * `stem` is the file-name prefix a page group writes under: `address` for the addresses,
+ * and each component type's own stem.
+ * A whole-region harvest writes 457 address pages and 17 thoroughfare-name pages,
+ * and re-requesting either group costs the same bytes again.
+ *
+ * The feature count is read from each file's own `numberReturned` rather than assumed
+ * from the page size, so a page the service answered short is recorded as short.
+ * A file whose opening bytes state no `numberReturned` is left out, which makes the
+ * harvest request that index again: a page of unknown extent is not a page this
+ * harvest can count, and a run killed mid-write leaves exactly that.
+ *
+ * A group small enough for one page is written under `<stem>.gml` with no index,
+ * so that name reads as `startIndex` 0.
+ */
+async function pagesOnDisk(root, stem) {
+    const pages = new Map();
+    const files = await Globerator.from(`${stem}*.gml`, { cwd: root.toString(), absolute: false }).toArray();
+    const indexed = new RegExp(`^${stem}(?:-(\\d+))?\\.gml$`, "u");
+    for (const file of files) {
+        const name = String(file);
+        const matched = indexed.exec(name);
+        if (!matched)
+            continue;
+        const startIndex = matched[1] === undefined ? 0 : Number(matched[1]);
+        if (!Number.isInteger(startIndex))
+            continue;
+        const handle = await open(root(name).toString(), "r");
+        try {
+            const buffer = Buffer.alloc(PAGE_ROOT_PREFIX_BYTES);
+            const { bytesRead } = await handle.read(buffer, 0, PAGE_ROOT_PREFIX_BYTES, 0);
+            const returned = rootAttribute(buffer.subarray(0, bytesRead).toString("utf8"), "numberReturned", {
+                xml: true,
+            });
+            if (returned === undefined || !/^\d+$/u.test(returned))
+                continue;
+            pages.set(startIndex, { file: name, numberReturned: Number(returned) });
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    return pages;
+}
+export async function harvestVlaanderenAD(client, options) {
+    const root = PathBuilder.from(options.outputDir);
+    const pageSize = Math.min(options.pageSize ?? MAX_PAGE_SIZE, MAX_PAGE_SIZE);
+    const componentPageSize = Math.min(options.componentPageSize ?? MAX_PAGE_SIZE, MAX_PAGE_SIZE);
+    await makeDirectories(root);
+    const componentFiles = {
+        ThoroughfareName: [],
+        PostalDescriptor: [],
+        AdminUnitName: [],
+    };
+    for (const [typeName, stem] of Object.entries(COMPONENT_STEMS)) {
+        const count = await readVlaanderenFeatureCount(client, `ad:${typeName}`);
+        const existing = await pagesOnDisk(root, stem);
+        let read = 0;
+        for (let startIndex = 0; startIndex < count; startIndex += componentPageSize) {
+            const held = existing.get(startIndex);
+            if (held !== undefined) {
+                componentFiles[typeName].push(held.file);
+                read += held.numberReturned;
+                continue;
+            }
+            const page = await readVlaanderenPage(client, {
+                typeNames: `ad:${typeName}`,
+                count: String(componentPageSize),
+                startIndex: String(startIndex),
+            });
+            if (page.numberReturned === null) {
+                throw new Error(`vlaanderen ad wfs: the ad:${typeName} page at startIndex=${startIndex} stated no ` +
+                    `numberReturned, so what it holds is unknown rather than empty`);
+            }
+            if (page.numberReturned === 0)
+                break;
+            const file = count <= componentPageSize ? `${stem}.gml` : `${stem}-${String(startIndex).padStart(10, "0")}.gml`;
+            await writeLocalTextFile(page.body, root(file));
+            componentFiles[typeName].push(file);
+            read += page.numberReturned;
+        }
+        // A component type read short leaves every address referencing its tail unjoinable,
+        // and the adapter would refuse those rows as if the publisher had not published them.
+        if (read !== count) {
+            throw new Error(`vlaanderen ad wfs: ad:${typeName} answered ${read} features over ` +
+                `${componentFiles[typeName].length} ` +
+                `pages where its hits request stated ${count}, so the type was not read whole`);
+        }
+    }
+    const addressCount = await readVlaanderenFeatureCount(client, "ad:Address");
+    const addressPages = [];
+    let retrievedAt = "";
+    // A page already on disk is not requested again.
+    // The service answered HTTP 400 on one page of a whole-region harvest after 105 pages
+    // and 1.8 GB, and that page answered 200 on a later attempt, so the refusal was transient.
+    // Without this the next run starts at index 0 and re-downloads every page it already holds.
+    // The page record is kept per page rather than written once at the end, because `harvest.json`
+    // is what states which pages exist and a throw before the end leaves no record of them.
+    const onDisk = await pagesOnDisk(root, "address");
+    for (let startIndex = 0; startIndex < addressCount; startIndex += pageSize) {
+        if (options.signal?.aborted)
+            break;
+        if (options.maxPages !== undefined && addressPages.length >= options.maxPages)
+            break;
+        const existing = onDisk.get(startIndex);
+        if (existing !== undefined) {
+            addressPages.push({ startIndex, file: existing.file, numberReturned: existing.numberReturned });
+            continue;
+        }
+        const page = await readVlaanderenPage(client, {
+            typeNames: "ad:Address",
+            count: String(pageSize),
+            startIndex: String(startIndex),
+        });
+        if (page.numberReturned === null) {
+            throw new Error(`vlaanderen ad wfs: the page at startIndex=${startIndex} stated no numberReturned, so what it ` +
+                `holds is unknown rather than empty`);
+        }
+        if (page.numberReturned === 0)
+            break;
+        const file = `address-${String(startIndex).padStart(10, "0")}.gml`;
+        await writeLocalTextFile(page.body, root(file));
+        addressPages.push({ startIndex, file, numberReturned: page.numberReturned });
+        retrievedAt = rootAttribute(page.body, "timeStamp", { xml: true }) ?? retrievedAt;
+    }
+    const harvest = {
+        service: VLAANDEREN_AD_WFS,
+        wfsVersion: WFS_VERSION,
+        outputFormat: OUTPUT_FORMAT,
+        retrievedAt,
+        addressCount,
+        components: componentFiles,
+        addressPages,
+    };
+    await writeLocalJSONFile(harvest, root("harvest.json"));
+    return harvest;
+}
+/**
+ * The directory `#be/adapters/vlaanderen/adapter` reads, given the root a fetch wrote under.
+ *
+ * The adapter reads the directory rather than one file, because a harvest writes the addresses
+ * and each component type as separate documents beside `harvest.json`.
+ */
+export function vlaanderenInputPath(outRoot) {
+    return outRoot(VLAANDEREN_ADAPTER_ID);
+}
+/**
+ * The registry entry.
+ *
+ * `harvestVlaanderenAD` takes a client so a test can drive it against stubbed bodies.
+ * This supplies the client the registry's callers expect, and reports the harvest as a
+ * {@linkcode FetchSummary}: one harvest is one fetched unit whatever the page count,
+ * because the adapter reads the directory rather than any single page.
+ */
+export async function fetchVlaanderenAD(options, report) {
+    await using client = new APIClient({ displayName: VLAANDEREN_ADAPTER_ID, retry: true });
+    const outputDir = options.outRoot(VLAANDEREN_ADAPTER_ID);
+    report?.(`=== ${VLAANDEREN_ADAPTER_ID}: harvesting ${VLAANDEREN_AD_WFS}`);
+    // Awaited rather than returned: `await using` disposes the client when this scope exits,
+    // and a disposed `APIClient` refuses every later request.
+    const harvest = await harvestVlaanderenAD(client, {
+        outputDir,
+        maxPages: options.maxPages,
+        pageSize: options.pageSize,
+        componentPageSize: options.componentPageSize,
+        signal: options.signal,
+    });
+    report?.(`  ✓ ${harvest.addressCount} addresses over ${harvest.addressPages.length} pages, ` +
+        `components ${Object.entries(harvest.components)
+            .map(([type, files]) => `${type}=${files.length}`)
+            .join(" ")}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=vlaanderen-ad.js.map