@mailwoman/corpus 9.4.0 → 10.1.0

package/out/inspire/archive.d.ts

@@ -0,0 +1,36 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The byte stream of an INSPIRE GML publication, whether it arrives zipped or unpacked.
+ *
+ * Several publishers of the Addresses theme ship one zipped GML for a whole region, and the member
+ * inside is far larger than the archive: Wallonia's `AD.Addresses.gml.zip` is 106,665,832 bytes over
+ * the wire and holds a single 5,163,917,131-byte member, a ratio of 48. A member that size exceeds
+ * what a process may hold, so this hands the caller a chunk sequence and never a buffer.
+ *
+ * A directory path is not accepted here. A publisher that ships one archive per municipality needs a
+ * listing step of its own, and that step belongs to the adapter rather than to this module.
+ */
+import { type ZipEntrySelector } from "@mailwoman/core/fs/zip";
+import { type PathBuilderLike } from "path-ts";
+/**
+ * The inflated bytes of a publication's GML, in order.
+ *
+ * `inputPath` is either the publisher's `.zip` or a GML file already unpacked from one.
+ * Both are read, because an operator who unpacked a multi-gigabyte member once should not
+ * have to keep re-inflating it, and because a committed fixture is reviewable only as text.
+ *
+ * `readZipEntry` seeks the archive's central directory through a file handle and inflates the one
+ * matching member, so memory is bounded by the inflate window rather than by the member's size.
+ * It also reads the ZIP64 extra field.
+ *
+ * A 5 GB member needs that field: its 32-bit size slot holds the `0xFFFFFFFF` sentinel
+ * and its real length lives in the extra field.
+ *
+ * @param member Which archive member holds the GML, as a name or a pattern.
+ * It is ignored for a path that is already a GML file.
+ */
+export declare function inspireGMLChunks(inputPath: PathBuilderLike, member: ZipEntrySelector): AsyncIterable<Uint8Array>;
+//# sourceMappingURL=archive.d.ts.map

package/out/inspire/archive.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"archive.d.ts","sourceRoot":"","sources":["../../lib/inspire/archive.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EAAgB,KAAK,gBAAgB,EAAE,MAAM,wBAAwB,CAAA;AAC5E,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAE3D;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,eAAe,EAAE,MAAM,EAAE,gBAAgB,GAAG,aAAa,CAAC,UAAU,CAAC,CAMhH"}

package/out/inspire/archive.js

@@ -0,0 +1,42 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The byte stream of an INSPIRE GML publication, whether it arrives zipped or unpacked.
+ *
+ * Several publishers of the Addresses theme ship one zipped GML for a whole region, and the member
+ * inside is far larger than the archive: Wallonia's `AD.Addresses.gml.zip` is 106,665,832 bytes over
+ * the wire and holds a single 5,163,917,131-byte member, a ratio of 48. A member that size exceeds
+ * what a process may hold, so this hands the caller a chunk sequence and never a buffer.
+ *
+ * A directory path is not accepted here. A publisher that ships one archive per municipality needs a
+ * listing step of its own, and that step belongs to the adapter rather than to this module.
+ */
+import { openReadStream } from "@mailwoman/core/fs/streams";
+import { readZipEntry } from "@mailwoman/core/fs/zip";
+import { PathBuilder } from "path-ts";
+/**
+ * The inflated bytes of a publication's GML, in order.
+ *
+ * `inputPath` is either the publisher's `.zip` or a GML file already unpacked from one.
+ * Both are read, because an operator who unpacked a multi-gigabyte member once should not
+ * have to keep re-inflating it, and because a committed fixture is reviewable only as text.
+ *
+ * `readZipEntry` seeks the archive's central directory through a file handle and inflates the one
+ * matching member, so memory is bounded by the inflate window rather than by the member's size.
+ * It also reads the ZIP64 extra field.
+ *
+ * A 5 GB member needs that field: its 32-bit size slot holds the `0xFFFFFFFF` sentinel
+ * and its real length lives in the extra field.
+ *
+ * @param member Which archive member holds the GML, as a name or a pattern.
+ * It is ignored for a path that is already a GML file.
+ */
+export function inspireGMLChunks(inputPath, member) {
+    const path = PathBuilder.from(inputPath);
+    if (path.basename().toLowerCase().endsWith(".zip"))
+        return readZipEntry(path, member);
+    return openReadStream(path);
+}
+//# sourceMappingURL=archive.js.map

package/out/inspire/archive.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"archive.js","sourceRoot":"","sources":["../../lib/inspire/archive.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,EAAE,YAAY,EAAyB,MAAM,wBAAwB,CAAA;AAC5E,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAA0B,EAAE,MAAwB;IACpF,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAExC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAErF,OAAO,cAAc,CAAC,IAAI,CAAC,CAAA;AAC5B,CAAC"}

package/out/inspire/errors.d.ts

@@ -0,0 +1,55 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The two conditions an INSPIRE Addresses reader must report rather than read as an absence.
+ *
+ * An `ad:Address` states its street, its postcode and its place names by reference, and states the
+ * parts of its number as typed designators. Both forms can hold a value the reader asked for and
+ * could not read: a reference whose target the service never served, and a designator the publisher
+ * marked `xsi:nil`. Turning either into an empty component would publish an address the publisher
+ * did not state, so each raises and names what it could not read.
+ *
+ * The errors live beside {@linkcode "#inspire/address"} because every publisher of this theme writes
+ * both forms, so an adapter for the next one reports the same two conditions in the same words.
+ */
+/**
+ * Raised when an `ad:component` reference resolves to no feature the reader indexed.
+ *
+ * The message states the address's own `gml:id` and the href as the publisher wrote it,
+ * so the failure identifies the record and the reference rather than the run.
+ *
+ * A reader whose index is complete raises on the first unresolved reference.
+ * One reading a document that interleaves features with addresses defers the address
+ * instead, and raises only once the document has ended, because until then an
+ * unresolved reference and an absent one are the same observation.
+ */
+export declare class UnresolvedComponentReferenceError extends Error {
+    constructor(adapterID: string, addressID: string | undefined, href: string);
+}
+/**
+ * Raised when a locator designator the reader needs is present and marked void.
+ *
+ * `designatorsByType` leaves a void designator out, so a reader keyed on that map by itself
+ * cannot tell a number the publisher marked unknown from one the publisher never wrote.
+ * `voidDesignatorTypes` separates them, and this error reports the first case.
+ */
+export declare class VoidDesignatorError extends Error {
+    constructor(adapterID: string, addressID: string | undefined, type: string);
+}
+/**
+ * Raised when a publication's archive cannot answer what the adapter asked it for.
+ *
+ * An archive holding no component feature is not a smaller publication.
+ * Every address references a fixed set of components, so an empty index turns
+ * each row into a row with no street rather than into fewer rows, which is the
+ * absence the repository's partial-read rule refuses.
+ *
+ * The adapter id carries which publisher failed, so one class serves every
+ * publisher that ships its theme as one archive.
+ */
+export declare class InspireArchiveError extends Error {
+    constructor(adapterID: string, message: string);
+}
+//# sourceMappingURL=errors.d.ts.map

package/out/inspire/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../lib/inspire/errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH;;;;;;;;;;GAUG;AACH,qBAAa,iCAAkC,SAAQ,KAAK;IAC3D,YAAY,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE,IAAI,EAAE,MAAM,EAMzE;CACD;AAED;;;;;;GAMG;AACH,qBAAa,mBAAoB,SAAQ,KAAK;IAC7C,YAAY,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE,IAAI,EAAE,MAAM,EAIzE;CACD;AAED;;;;;;;;;;GAUG;AACH,qBAAa,mBAAoB,SAAQ,KAAK;IAC7C,YAAY,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAI7C;CACD"}

package/out/inspire/errors.js

@@ -0,0 +1,64 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The two conditions an INSPIRE Addresses reader must report rather than read as an absence.
+ *
+ * An `ad:Address` states its street, its postcode and its place names by reference, and states the
+ * parts of its number as typed designators. Both forms can hold a value the reader asked for and
+ * could not read: a reference whose target the service never served, and a designator the publisher
+ * marked `xsi:nil`. Turning either into an empty component would publish an address the publisher
+ * did not state, so each raises and names what it could not read.
+ *
+ * The errors live beside {@linkcode "#inspire/address"} because every publisher of this theme writes
+ * both forms, so an adapter for the next one reports the same two conditions in the same words.
+ */
+/**
+ * Raised when an `ad:component` reference resolves to no feature the reader indexed.
+ *
+ * The message states the address's own `gml:id` and the href as the publisher wrote it,
+ * so the failure identifies the record and the reference rather than the run.
+ *
+ * A reader whose index is complete raises on the first unresolved reference.
+ * One reading a document that interleaves features with addresses defers the address
+ * instead, and raises only once the document has ended, because until then an
+ * unresolved reference and an absent one are the same observation.
+ */
+export class UnresolvedComponentReferenceError extends Error {
+    constructor(adapterID, addressID, href) {
+        super(`${adapterID} adapter: address ${addressID ?? "(no gml:id)"} carries ad:component xlink:href="${href}", which resolves to no feature`);
+        this.name = "UnresolvedComponentReferenceError";
+    }
+}
+/**
+ * Raised when a locator designator the reader needs is present and marked void.
+ *
+ * `designatorsByType` leaves a void designator out, so a reader keyed on that map by itself
+ * cannot tell a number the publisher marked unknown from one the publisher never wrote.
+ * `voidDesignatorTypes` separates them, and this error reports the first case.
+ */
+export class VoidDesignatorError extends Error {
+    constructor(adapterID, addressID, type) {
+        super(`${adapterID} adapter: address ${addressID ?? "(no gml:id)"} marks its ${type} designator void`);
+        this.name = "VoidDesignatorError";
+    }
+}
+/**
+ * Raised when a publication's archive cannot answer what the adapter asked it for.
+ *
+ * An archive holding no component feature is not a smaller publication.
+ * Every address references a fixed set of components, so an empty index turns
+ * each row into a row with no street rather than into fewer rows, which is the
+ * absence the repository's partial-read rule refuses.
+ *
+ * The adapter id carries which publisher failed, so one class serves every
+ * publisher that ships its theme as one archive.
+ */
+export class InspireArchiveError extends Error {
+    constructor(adapterID, message) {
+        super(`${adapterID} adapter: ${message}`);
+        this.name = "InspireArchiveError";
+    }
+}
+//# sourceMappingURL=errors.js.map

package/out/inspire/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../lib/inspire/errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH;;;;;;;;;;GAUG;AACH,MAAM,OAAO,iCAAkC,SAAQ,KAAK;IAC3D,YAAY,SAAiB,EAAE,SAA6B,EAAE,IAAY;QACzE,KAAK,CACJ,GAAG,SAAS,qBAAqB,SAAS,IAAI,aAAa,qCAAqC,IAAI,iCAAiC,CACrI,CAAA;QAED,IAAI,CAAC,IAAI,GAAG,mCAAmC,CAAA;IAChD,CAAC;CACD;AAED;;;;;;GAMG;AACH,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC7C,YAAY,SAAiB,EAAE,SAA6B,EAAE,IAAY;QACzE,KAAK,CAAC,GAAG,SAAS,qBAAqB,SAAS,IAAI,aAAa,cAAc,IAAI,kBAAkB,CAAC,CAAA;QAEtG,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAA;IAClC,CAAC;CACD;AAED;;;;;;;;;;GAUG;AACH,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC7C,YAAY,SAAiB,EAAE,OAAe;QAC7C,KAAK,CAAC,GAAG,SAAS,aAAa,OAAO,EAAE,CAAC,CAAA;QAEzC,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAA;IAClC,CAAC;CACD"}

package/out/inspire/stream.d.ts

@@ -0,0 +1,137 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Reads addresses out of one INSPIRE document that interleaves them with the features they
+ *   reference.
+ *
+ * A publisher shipping the Addresses theme as one document writes an `ad:Address` beside the
+ * `ad:ThoroughfareName`, `ad:PostalDescriptor` and place-name features it references, and the order
+ * is the publisher's own. Czechia interleaves them per municipality, and the Netherlands writes
+ * 300,318 reference features before its first address.
+ *
+ * Either way a reader has to index as it goes, because a document of 32 GB cannot be read twice from
+ * one stream and the references are local. An address naming a feature the pass has not reached yet
+ * is held rather than refused, since until the document ends an unresolved reference and an absent
+ * one are the same observation. That distinction is what the repository's partial-read rule turns on.
+ *
+ * A publisher that writes every reference before any address, as Wallonia, Brussels and Slovakia do,
+ * needs no holding and indexes in a pass of its own instead. Those adapters call
+ * `streamMarkupElements` directly.
+ *
+ * A publisher that writes every reference *after* its addresses needs the opposite of holding, and
+ * {@linkcode streamIndexedInspireRows} serves it. The Dirección General del Catastro and the
+ * Diputación Foral de Gipuzkoa both do: Catastro's Ceuta member writes its 511 reference features in
+ * the last 530,000 of 11,861,975 bytes, and Gipuzkoa writes its 4,516 in the last 6,722,000 of
+ * 313,162,472. Holding would mean holding every address. That is 68,744 subtrees for Gipuzkoa and
+ * the whole of Madrid for Catastro, so those two read the member twice and index first.
+ */
+import type { MarkupElement } from "@mailwoman/core/html/elements";
+import type { CanonicalRow } from "#types";
+/**
+ * What a {@linkcode ComposeInspireRow} callback answers for one address.
+ *
+ * A row is emitted.
+ * `undefined` refuses the address, which the caller counts.
+ *
+ * `deferred` states that the address carries a reference the pass has not indexed yet,
+ * so the driver holds it and asks again once the document has ended.
+ */
+export type InspireRowResult = CanonicalRow | undefined | "deferred";
+/**
+ * Builds one row from an address and the references indexed so far.
+ *
+ * @param address The `ad:Address` subtree, with its attributes and children.
+ * @param referenced Every referenced feature the pass has indexed, keyed as
+ * {@linkcode StreamInspireRowsOptions.key} spells it.
+ * @param final Whether the document has ended.
+ * A caller returns `deferred` only while this is false, because afterwards an unresolved
+ * reference is an absence the publisher is answerable for.
+ */
+export type ComposeInspireRow<Indexed> = (address: MarkupElement, referenced: ReadonlyMap<string, Indexed>, final: boolean) => InspireRowResult;
+/**
+ * How to read one publisher's document.
+ */
+export interface StreamInspireRowsOptions<Indexed> {
+    /**
+     * The document's bytes, in order.
+     */
+    chunks: AsyncIterable<string | Uint8Array>;
+    /**
+     * The element each address is written as, which every INSPIRE publisher spells `ad:Address`.
+     */
+    addressElement?: string;
+    /**
+     * The elements an address references, which are indexed as they arrive.
+     */
+    componentElements: readonly string[];
+    /**
+     * What to index a referenced feature under.
+     *
+     * The default is its `gml:id`, which is what a local `#` fragment
+     * and a stored-query `Id` parameter both name.
+     * A feature carrying no key is skipped, because an address cannot reference it.
+     */
+    key?: (feature: MarkupElement) => string | undefined;
+    /**
+     * What to keep per referenced feature.
+     *
+     * Czechia keeps the subtree, because it reads several values off one feature.
+     * The Netherlands keeps the one string it needs, which holds 300,318 strings
+     * rather than 300,318 subtrees.
+     */
+    index: (feature: MarkupElement) => Indexed;
+    /**
+     * Builds a row, or defers it.
+     */
+    compose: ComposeInspireRow<Indexed>;
+    /**
+     * A soft cap on emitted rows.
+     */
+    limit?: number;
+    /**
+     * Stops the read where a caller cancels.
+     */
+    signal?: AbortSignal;
+}
+/**
+ * The rows one document holds, in the order the publisher wrote its addresses.
+ *
+ * A held address is answered after the document ends, so those rows arrive last.
+ * That reorders a publisher's output and changes no row, because each row carries its own `source_id`.
+ */
+export declare function streamInspireRows<Indexed>(options: StreamInspireRowsOptions<Indexed>): AsyncIterable<CanonicalRow>;
+/**
+ * How to read one publisher's document twice, indexing its references before reading its addresses.
+ */
+export interface StreamIndexedInspireRowsOptions<Indexed> extends Omit<StreamInspireRowsOptions<Indexed>, "chunks" | "compose"> {
+    /**
+     * Opens the document's bytes.
+     *
+     * Called once per pass, so it must deliver the same bytes each time.
+     *
+     * A factory rather than an iterable, because a byte stream is consumed by the first pass.
+     * `inspireGMLChunks` inflates the archive member again, which is what makes the
+     * second pass cheap relative to holding every address.
+     */
+    chunks: () => AsyncIterable<string | Uint8Array>;
+    /**
+     * Builds a row from an address and the complete reference index.
+     *
+     * There is no `final` argument and no deferral: the first pass has read the whole document,
+     * so a reference resolving to no feature is a value the reader asked for and could not read.
+     * The caller raises on one rather than writing the address without it.
+     */
+    compose: (address: MarkupElement, referenced: ReadonlyMap<string, Indexed>) => CanonicalRow | undefined;
+}
+/**
+ * The rows one document holds, read with its reference index already complete.
+ *
+ * The first pass reads only the component elements and the second only the addresses,
+ * so the resident set is the index rather than the addresses.
+ * The rows arrive in the publisher's own order, which {@linkcode streamInspireRows}
+ * cannot promise because it answers a held address last.
+ */
+export declare function streamIndexedInspireRows<Indexed>(options: StreamIndexedInspireRowsOptions<Indexed>): AsyncIterable<CanonicalRow>;
+//# sourceMappingURL=stream.d.ts.map

package/out/inspire/stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../lib/inspire/stream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAGlE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AAE1C;;;;;;;;GAQG;AACH,MAAM,MAAM,gBAAgB,GAAG,YAAY,GAAG,SAAS,GAAG,UAAU,CAAA;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,CAAC,OAAO,IAAI,CACxC,OAAO,EAAE,aAAa,EACtB,UAAU,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,EACxC,KAAK,EAAE,OAAO,KACV,gBAAgB,CAAA;AAErB;;GAEG;AACH,MAAM,WAAW,wBAAwB,CAAC,OAAO;IAChD;;OAEG;IACH,MAAM,EAAE,aAAa,CAAC,MAAM,GAAG,UAAU,CAAC,CAAA;IAE1C;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;IAEvB;;OAEG;IACH,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAA;IAEpC;;;;;;OAMG;IACH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,MAAM,GAAG,SAAS,CAAA;IAEpD;;;;;;OAMG;IACH,KAAK,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,OAAO,CAAA;IAE1C;;OAEG;IACH,OAAO,EAAE,iBAAiB,CAAC,OAAO,CAAC,CAAA;IAEnC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IAEd;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AASD;;;;;GAKG;AACH,wBAAuB,iBAAiB,CAAC,OAAO,EAC/C,OAAO,EAAE,wBAAwB,CAAC,OAAO,CAAC,GACxC,aAAa,CAAC,YAAY,CAAC,CA6D7B;AAED;;GAEG;AACH,MAAM,WAAW,+BAA+B,CAAC,OAAO,CAAE,SAAQ,IAAI,CACrE,wBAAwB,CAAC,OAAO,CAAC,EACjC,QAAQ,GAAG,SAAS,CACpB;IACA;;;;;;;;OAQG;IACH,MAAM,EAAE,MAAM,aAAa,CAAC,MAAM,GAAG,UAAU,CAAC,CAAA;IAEhD;;;;;;OAMG;IACH,OAAO,EAAE,CAAC,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,YAAY,GAAG,SAAS,CAAA;CACvG;AAED;;;;;;;GAOG;AACH,wBAAuB,wBAAwB,CAAC,OAAO,EACtD,OAAO,EAAE,+BAA+B,CAAC,OAAO,CAAC,GAC/C,aAAa,CAAC,YAAY,CAAC,CA8B7B"}

package/out/inspire/stream.js

@@ -0,0 +1,125 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Reads addresses out of one INSPIRE document that interleaves them with the features they
+ *   reference.
+ *
+ * A publisher shipping the Addresses theme as one document writes an `ad:Address` beside the
+ * `ad:ThoroughfareName`, `ad:PostalDescriptor` and place-name features it references, and the order
+ * is the publisher's own. Czechia interleaves them per municipality, and the Netherlands writes
+ * 300,318 reference features before its first address.
+ *
+ * Either way a reader has to index as it goes, because a document of 32 GB cannot be read twice from
+ * one stream and the references are local. An address naming a feature the pass has not reached yet
+ * is held rather than refused, since until the document ends an unresolved reference and an absent
+ * one are the same observation. That distinction is what the repository's partial-read rule turns on.
+ *
+ * A publisher that writes every reference before any address, as Wallonia, Brussels and Slovakia do,
+ * needs no holding and indexes in a pass of its own instead. Those adapters call
+ * `streamMarkupElements` directly.
+ *
+ * A publisher that writes every reference *after* its addresses needs the opposite of holding, and
+ * {@linkcode streamIndexedInspireRows} serves it. The Dirección General del Catastro and the
+ * Diputación Foral de Gipuzkoa both do: Catastro's Ceuta member writes its 511 reference features in
+ * the last 530,000 of 11,861,975 bytes, and Gipuzkoa writes its 4,516 in the last 6,722,000 of
+ * 313,162,472. Holding would mean holding every address. That is 68,744 subtrees for Gipuzkoa and
+ * the whole of Madrid for Catastro, so those two read the member twice and index first.
+ */
+import { streamMarkupElements } from "@mailwoman/core/html/elements";
+/**
+ * The default key a referenced feature is indexed under.
+ */
+function gmlID(feature) {
+    return feature.attributes["gml:id"];
+}
+/**
+ * The rows one document holds, in the order the publisher wrote its addresses.
+ *
+ * A held address is answered after the document ends, so those rows arrive last.
+ * That reorders a publisher's output and changes no row, because each row carries its own `source_id`.
+ */
+export async function* streamInspireRows(options) {
+    const addressElement = options.addressElement ?? "ad:Address";
+    const key = options.key ?? gmlID;
+    /**
+     * Every referenced feature the pass has indexed.
+     */
+    const referenced = new Map();
+    /**
+     * Addresses naming a reference the pass had not indexed when they arrived.
+     */
+    const held = [];
+    let emitted = 0;
+    const wanted = [addressElement, ...options.componentElements];
+    for await (const element of streamMarkupElements(options.chunks, wanted, { xml: true })) {
+        if (options.signal?.aborted)
+            return;
+        if (element.name !== addressElement) {
+            const id = key(element);
+            if (id) {
+                referenced.set(id, options.index(element));
+            }
+            continue;
+        }
+        if (options.limit !== undefined && emitted >= options.limit)
+            break;
+        const row = options.compose(element, referenced, false);
+        if (row === "deferred") {
+            held.push(element);
+            continue;
+        }
+        if (!row)
+            continue;
+        yield row;
+        emitted++;
+    }
+    for (const element of held) {
+        if (options.signal?.aborted)
+            return;
+        if (options.limit !== undefined && emitted >= options.limit)
+            break;
+        const row = options.compose(element, referenced, true);
+        // `deferred` after the document ended is a caller that ignored `final`,
+        // and the row is refused rather than held again, which would never terminate.
+        if (!row || row === "deferred")
+            continue;
+        yield row;
+        emitted++;
+    }
+}
+/**
+ * The rows one document holds, read with its reference index already complete.
+ *
+ * The first pass reads only the component elements and the second only the addresses,
+ * so the resident set is the index rather than the addresses.
+ * The rows arrive in the publisher's own order, which {@linkcode streamInspireRows}
+ * cannot promise because it answers a held address last.
+ */
+export async function* streamIndexedInspireRows(options) {
+    const addressElement = options.addressElement ?? "ad:Address";
+    const key = options.key ?? gmlID;
+    const referenced = new Map();
+    for await (const feature of streamMarkupElements(options.chunks(), options.componentElements, { xml: true })) {
+        if (options.signal?.aborted)
+            return;
+        const id = key(feature);
+        if (id) {
+            referenced.set(id, options.index(feature));
+        }
+    }
+    let emitted = 0;
+    for await (const address of streamMarkupElements(options.chunks(), addressElement, { xml: true })) {
+        if (options.signal?.aborted)
+            return;
+        if (options.limit !== undefined && emitted >= options.limit)
+            return;
+        const row = options.compose(address, referenced);
+        if (!row)
+            continue;
+        yield row;
+        emitted++;
+    }
+}
+//# sourceMappingURL=stream.js.map

package/out/inspire/stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream.js","sourceRoot":"","sources":["../../lib/inspire/stream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAGH,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAA;AAoFpE;;GAEG;AACH,SAAS,KAAK,CAAC,OAAsB;IACpC,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAA;AACpC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,iBAAiB,CACvC,OAA0C;IAE1C,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,YAAY,CAAA;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,KAAK,CAAA;IAEhC;;OAEG;IACH,MAAM,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAA;IAC7C;;OAEG;IACH,MAAM,IAAI,GAAoB,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,MAAM,MAAM,GAAG,CAAC,cAAc,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAA;IAE7D,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,oBAAoB,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACzF,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAEnC,IAAI,OAAO,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACrC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,CAAA;YAEvB,IAAI,EAAE,EAAE,CAAC;gBACR,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAA;YAC3C,CAAC;YAED,SAAQ;QACT,CAAC;QAED,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK;YAAE,MAAK;QAElE,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,CAAA;QAEvD,IAAI,GAAG,KAAK,UAAU,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAElB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG;YAAE,SAAQ;QAElB,MAAM,GAAG,CAAA;QAET,OAAO,EAAE,CAAA;IACV,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;QAC5B,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAEnC,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK;YAAE,MAAK;QAElE,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;QAEtD,wEAAwE;QACxE,8EAA8E;QAC9E,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,UAAU;YAAE,SAAQ;QAExC,MAAM,GAAG,CAAA;QAET,OAAO,EAAE,CAAA;IACV,CAAC;AACF,CAAC;AA8BD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,wBAAwB,CAC9C,OAAiD;IAEjD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,YAAY,CAAA;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,KAAK,CAAA;IAChC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAA;IAE7C,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,oBAAoB,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,OAAO,CAAC,iBAAiB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QAC9G,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAEnC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,CAAA;QAEvB,IAAI,EAAE,EAAE,CAAC;YACR,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAA;QAC3C,CAAC;IACF,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,oBAAoB,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACnG,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAEnC,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK;YAAE,OAAM;QAEnE,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAA;QAEhD,IAAI,CAAC,GAAG;YAAE,SAAQ;QAElB,MAAM,GAAG,CAAA;QAET,OAAO,EAAE,CAAA;IACV,CAAC;AACF,CAAC"}

package/out/international/recipes/locale.d.ts

@@ -0,0 +1,90 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Samples real address tuples for one country and renders them in native or international order.
+ *   It emits aligned rows. The recipe samples and renders with separate seeded generators.
+ */
+import type { PathBuilderLike } from "path-ts";
+import type { CorpusRecipe } from "#recipes/scaffold";
+import { SourceRegister } from "#registers";
+import { type LocaleBaseTuple, type RenderedLocaleRow } from "#surfaces/locale";
+/**
+ * One source file: either a CSV member of a ZIP (`zip` and `csv`) or an extracted CSV (`path`).
+ */
+export interface LocalePart {
+    zip?: PathBuilderLike;
+    csv?: string;
+    path?: PathBuilderLike;
+    region?: string;
+    /**
+     * Maps the district column to locality and the city column to dependent locality.
+     * When the district is empty, the city becomes the locality.
+     */
+    districtAsLocality?: boolean;
+    /**
+     * Reads Spain's raw CNIG schema.
+     *
+     * `poblacion` is the settlement.
+     * `municipio` is its parent.
+     * The street combines `tipo_vial` with `nombre_via`.
+     */
+    cnigRaw?: boolean;
+}
+/**
+ * The source files and row metadata for one country.
+ */
+export interface LocaleCountrySource {
+    source: string;
+    parts: LocalePart[];
+    /**
+     * The publication register written to each row.
+     */
+    register: SourceRegister;
+    /**
+     * The corpus version written to each row.
+     */
+    corpusVersion: string;
+    /**
+     * Replacement parts used when `--district-as-locality` is passed as true.
+     */
+    pedaniaParts?: LocalePart[];
+}
+/**
+ * Cleans a city value.
+ *
+ * Returns `null` for a value with a comma or four digits.
+ * Strips a trailing parenthesized code of one to three letters.
+ */
+export declare function cleanCityNoise(city: string): string | null;
+/**
+ * Streams tuples from one part and reservoir-samples them with Algorithm R,
+ * using `rng` for reproducibility.
+ *
+ * A read failure logs a warning and returns an empty list.
+ */
+export declare function readTuples(part: LocalePart, rng: () => number): Promise<LocaleBaseTuple[]>;
+/**
+ * Appends a country name to the row's text and components with probability `countryFraction`.
+ *
+ * A zero fraction draws no random number, so seeded output stays the same as a run without the option.
+ *
+ * @throws When the codex has no surface forms for the country.
+ */
+export declare function applyCountryAppend(synth: RenderedLocaleRow, country: string, countryFraction: number, random: () => number): void;
+/**
+ * Applies the per-run `districtAsLocality` override to a part.
+ * An unset override returns the part unchanged.
+ */
+export declare function applyDistrictAsLocalityOverride(part: LocalePart, override: boolean | undefined): LocalePart;
+/**
+ * Returns the country's `pedaniaParts` when the override is true and that property exists.
+ * Returns the default parts in all other cases.
+ */
+export declare function resolveLocaleParts(countrySource: LocaleCountrySource, override: boolean | undefined): LocalePart[];
+/**
+ * The locale recipe registered with the corpus builder.
+ */
+export declare const localeRecipe: CorpusRecipe;
+//# sourceMappingURL=locale.d.ts.map

package/out/international/recipes/locale.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../lib/international/recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAUH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAI9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,iBAAiB,EAAmB,MAAM,kBAAkB,CAAA;AAIhG;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,CAAC,EAAE,eAAe,CAAA;IACrB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,eAAe,CAAA;IACtB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB;;OAEG;IACH,QAAQ,EAAE,cAAc,CAAA;IACxB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;OAEG;IACH,YAAY,CAAC,EAAE,UAAU,EAAE,CAAA;CAC3B;AA+GD;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM1D;AAgBD;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA0HhG;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CACjC,KAAK,EAAE,iBAAiB,EACxB,OAAO,EAAE,MAAM,EACf,eAAe,EAAE,MAAM,EACvB,MAAM,EAAE,MAAM,MAAM,GAClB,IAAI,CAcN;AAED;;;GAGG;AACH,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAE3G;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,aAAa,EAAE,mBAAmB,EAAE,QAAQ,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,EAAE,CAElH;AAED;;GAEG;AACH,eAAO,MAAM,YAAY,EAAE,YA0K1B,CAAA"}