@mailwoman/corpus 10.6.0 → 10.7.0

package/out/tools/variety-sample.d.ts

@@ -0,0 +1,196 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Draws a country's training rows from a complete adapter output so the sample spans the country's address forms
+ *   rather than its densest streets, and measures the variety of both.
+ *
+ *   An address source lists one house after another: a city street contributes every number along it, and a city
+ *   contributes thousands of such streets. A model trained on that order sees the same street, locality and form
+ *   repeated. The sample here keeps at most a few rows per street and at most a fixed number per place (a locality
+ *   with its dependent locality), and fills each place round-robin across its forms (which components are present,
+ *   the street type, the script) and its postcodes. Places are then filled in rounds, so every place is represented
+ *   once before any is represented twice, until the country's ceiling. The ceiling keeps one country from dominating
+ *   a mixture.
+ *
+ *   The adapter output stays complete beside the sample, because the same rows feed retrieval.
+ */
+import { type PathBuilderLike } from "path-ts";
+import type { CanonicalRow } from "#types";
+/**
+ * The script most of `text`'s letters are written in, or `Other` when none
+ * of {@link SCRIPTS} holds a letter.
+ */
+export declare function dominantScript(text: string): string;
+/**
+ * The street-type forms a country's addresses use, each mapped to its canonical
+ * spelling, and which lexicon supplied them.
+ */
+export interface StreetTypeLexicon {
+    readonly forms: ReadonlyMap<string, string>;
+    /**
+     * The libpostal languages read: the country's official languages that have a `street_types.txt`,
+     * or `all` when none does and every language's list was read instead.
+     */
+    readonly languages: readonly string[] | "all";
+}
+/**
+ * The libpostal street-type lexicon for `country`'s official languages.
+ *
+ * A country whose official languages have no libpostal list (Armenian, Uzbek, Kyrgyz) reads
+ * every language's list, so a street written in a neighbor's language still counts its type.
+ */
+export declare function loadStreetTypeLexicon(country: string): Promise<StreetTypeLexicon>;
+/**
+ * The canonical street type a street name carries, read from its first or last one or two words, or null.
+ */
+export declare function streetTypeOf(street: string, lexicon: StreetTypeLexicon): string | null;
+/**
+ * What one row contributes to a variety census.
+ */
+export interface RowVariety {
+    readonly locality: string;
+    readonly dependentLocality: string | null;
+    /**
+     * The locality and the dependent locality together: the settlement a row is stratified by.
+     */
+    readonly place: string;
+    readonly postcode: string | null;
+    readonly street: string | null;
+    readonly streetType: string | null;
+    readonly hasUnit: boolean;
+    readonly hasDependentLocality: boolean;
+    readonly hasVenue: boolean;
+    readonly streetless: boolean;
+    readonly script: string;
+    /**
+     * The components present, the street type and the script, joined: the row's form.
+     */
+    readonly form: string;
+}
+/**
+ * Reads one canonical row's variety keys.
+ *
+ * The locality is the row's `locality`, else its `dependent_locality`, else its `region`,
+ * so a streetless premise in a named place is stratified by that place.
+ * The place a row is stratified by joins the dependent locality to it, so an Estonian village is
+ * its own place rather than one of its municipality's, and a city's districts are each their own.
+ */
+export declare function rowVariety(row: Pick<CanonicalRow, "components" | "raw">, lexicon: StreetTypeLexicon): RowVariety;
+/**
+ * The variety measures of a set of rows.
+ */
+export interface VarietyMeasures {
+    rows: number;
+    localities: number;
+    dependentLocalities: number;
+    /**
+     * Distinct locality and dependent-locality pairs.
+     */
+    places: number;
+    postcodes: number;
+    streets: number;
+    streetTypes: number;
+    forms: number;
+    unitShare: number;
+    dependentLocalityShare: number;
+    venueShare: number;
+    streetlessShare: number;
+    postcodeShare: number;
+    scripts: Record<string, number>;
+    /**
+     * The share of rows in the largest locality, a measure of how concentrated the rows are.
+     */
+    largestLocalityShare: number;
+}
+/**
+ * Accumulates {@link VarietyMeasures} one row at a time.
+ */
+export declare class VarietyCensus {
+    #private;
+    add(variety: RowVariety): void;
+    measures(): VarietyMeasures;
+}
+/**
+ * The limits a variety sample is drawn under.
+ */
+export interface VarietySampleLimits {
+    /**
+     * The most rows kept from one street in one locality.
+     */
+    perStreet: number;
+    /**
+     * The most rows kept from one place, a locality and dependent-locality pair.
+     */
+    perPlace: number;
+    /**
+     * The most rows kept for the country.
+     */
+    ceiling: number;
+    seed: number;
+}
+/**
+ * The selection key of one row: interned ids, so a country's rows fit in memory as integers.
+ */
+export interface SelectionKey {
+    readonly place: number;
+    readonly street: number;
+    readonly form: number;
+    readonly postcode: number;
+}
+/**
+ * Chooses the row indices of a variety sample.
+ *
+ * Rows are visited in a seeded random order.
+ * A street keeps its first {@link VarietySampleLimits.perStreet} rows.
+ *
+ * Within a place the kept rows are interleaved across forms, and within a form across postcodes,
+ * and the first {@link VarietySampleLimits.perPlace} are the place's candidates.
+ * Candidates are then taken by their rank within their place, lowest first, so every place supplies
+ * its first row before any supplies its second, until {@link VarietySampleLimits.ceiling}.
+ * The returned indices are ascending.
+ */
+export declare function selectVarietySample(keys: readonly SelectionKey[], limits: VarietySampleLimits): number[];
+/**
+ * What a variety-sample run read, wrote and measured.
+ */
+export interface VarietySampleReport {
+    input: string;
+    inputSHA256: string;
+    output: string;
+    outputSHA256: string;
+    country: string;
+    limits: VarietySampleLimits;
+    streetTypeLanguages: readonly string[] | "all";
+    /**
+     * Lines that did not parse as a canonical row of the requested country.
+     */
+    skipped: Record<string, number>;
+    full: VarietyMeasures;
+    sample: VarietyMeasures;
+    /**
+     * The share of the full output's distinct values the sample keeps.
+     */
+    coverage: {
+        localities: number;
+        places: number;
+        postcodes: number;
+        streetTypes: number;
+        forms: number;
+    };
+}
+/**
+ * Reads a complete adapter output, writes its variety sample to `output` and the
+ * report beside it as `<output>.variety.json`, and returns the report.
+ *
+ * The input is read twice, once to key every row and once to copy the chosen lines,
+ * so memory holds integers per row rather than the rows themselves.
+ */
+export declare function writeVarietySample(options: {
+    input: PathBuilderLike;
+    output: PathBuilderLike;
+    country: string;
+    limits: VarietySampleLimits;
+}): Promise<VarietySampleReport>;
+//# sourceMappingURL=variety-sample.d.ts.map

package/out/tools/variety-sample.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"variety-sample.d.ts","sourceRoot":"","sources":["../../lib/tools/variety-sample.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AASH,OAAO,EAAE,KAAK,eAAe,EAAe,MAAM,SAAS,CAAA;AAK3D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AAqB1C;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAsBnD;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC3C;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAAA;CAC7C;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CA+BvF;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,MAAM,GAAG,IAAI,CAyBtF;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAA;IACzC;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IAClC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,oBAAoB,EAAE,OAAO,CAAA;IACtC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAA;IAC1B,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACrB;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,GAAG,KAAK,CAAC,EAAE,OAAO,EAAE,iBAAiB,GAAG,UAAU,CA2BhH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,mBAAmB,EAAE,MAAM,CAAA;IAC3B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,sBAAsB,EAAE,MAAM,CAAA;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,eAAe,EAAE,MAAM,CAAA;IACvB,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC/B;;OAEG;IACH,oBAAoB,EAAE,MAAM,CAAA;CAC5B;AAED;;GAEG;AACH,qBAAa,aAAa;;IAgBzB,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI,CAwC7B;IAED,QAAQ,IAAI,eAAe,CAyB1B;CACD;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CACzB;AA6CD;;;;;;;;;;;GAWG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,SAAS,YAAY,EAAE,EAAE,MAAM,EAAE,mBAAmB,GAAG,MAAM,EAAE,CAwDxG;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,mBAAmB,CAAA;IAC3B,mBAAmB,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAAA;IAC9C;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC/B,IAAI,EAAE,eAAe,CAAA;IACrB,MAAM,EAAE,eAAe,CAAA;IACvB;;OAEG;IACH,QAAQ,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CACvG;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE;IACjD,KAAK,EAAE,eAAe,CAAA;IACtB,MAAM,EAAE,eAAe,CAAA;IACvB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,mBAAmB,CAAA;CAC3B,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAoH/B"}

package/out/tools/variety-sample.js

@@ -0,0 +1,417 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Draws a country's training rows from a complete adapter output so the sample spans the country's address forms
+ *   rather than its densest streets, and measures the variety of both.
+ *
+ *   An address source lists one house after another: a city street contributes every number along it, and a city
+ *   contributes thousands of such streets. A model trained on that order sees the same street, locality and form
+ *   repeated. The sample here keeps at most a few rows per street and at most a fixed number per place (a locality
+ *   with its dependent locality), and fills each place round-robin across its forms (which components are present,
+ *   the street type, the script) and its postcodes. Places are then filled in rounds, so every place is represented
+ *   once before any is represented twice, until the country's ceiling. The ceiling keeps one country from dominating
+ *   a mixture.
+ *
+ *   The adapter output stays complete beside the sample, because the same rows feed retrieval.
+ */
+import { OFFICIAL_LANGUAGES } from "@mailwoman/codex/country/official-languages";
+import { pathExists } from "@mailwoman/core/fs/readers/stat";
+import { writeLocalJSONFile, writeLocalTextFile } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { tryParsingJSON } from "@mailwoman/core/json";
+import { resourceDictionaryPath } from "@mailwoman/core/paths";
+import { mulberry32, shuffleWith } from "@mailwoman/core/random";
+import { resolvePath } from "path-ts";
+import { TextSpliterator } from "spliterator";
+import { Globerator } from "spliterator/node/fs";
+import { loadLibpostalCanonicalForms } from "#adapters/libpostal-dictionary";
+/**
+ * The scripts a row's letters are classified into, in the order a tie is broken.
+ */
+const SCRIPTS = [
+    "Latin",
+    "Cyrillic",
+    "Arabic",
+    "Armenian",
+    "Georgian",
+    "Greek",
+    "Hebrew",
+    "Han",
+    "Hangul",
+    "Thai",
+    "Devanagari",
+];
+const SCRIPT_PATTERNS = SCRIPTS.map((script) => [script, new RegExp(`\\p{Script=${script}}`, "u")]);
+/**
+ * The script most of `text`'s letters are written in, or `Other` when none
+ * of {@link SCRIPTS} holds a letter.
+ */
+export function dominantScript(text) {
+    const counts = new Map();
+    for (const character of text) {
+        if (!/\p{L}/u.test(character))
+            continue;
+        const script = SCRIPT_PATTERNS.find(([, pattern]) => pattern.test(character))?.[0] ?? "Other";
+        counts.set(script, (counts.get(script) ?? 0) + 1);
+    }
+    let best = "Other";
+    let most = 0;
+    for (const [script, count] of counts) {
+        if (count > most) {
+            best = script;
+            most = count;
+        }
+    }
+    return best;
+}
+/**
+ * The libpostal street-type lexicon for `country`'s official languages.
+ *
+ * A country whose official languages have no libpostal list (Armenian, Uzbek, Kyrgyz) reads
+ * every language's list, so a street written in a neighbor's language still counts its type.
+ */
+export async function loadStreetTypeLexicon(country) {
+    const official = (OFFICIAL_LANGUAGES[country.toUpperCase()]?.official ?? []).filter((code) => code.length === 2);
+    const available = [];
+    for (const language of official) {
+        if (await pathExists(resourceDictionaryPath("libpostal", language, "street_types.txt"))) {
+            available.push(language);
+        }
+    }
+    const every = async () => {
+        const files = await Globerator.from("*/street_types.txt", {
+            cwd: resourceDictionaryPath("libpostal").toString(),
+            absolute: false,
+        }).toSorted();
+        return files.map((file) => file.split("/")[0]);
+    };
+    const languages = available.length ? available : await every();
+    const forms = new Map();
+    for (const language of languages) {
+        for (const [form, canonical] of await loadLibpostalCanonicalForms(language, "street_types.txt")) {
+            if (form && !forms.has(form)) {
+                forms.set(form, `${language}:${canonical}`);
+            }
+        }
+    }
+    return { forms, languages: available.length ? available : "all" };
+}
+/**
+ * The canonical street type a street name carries, read from its first or last one or two words, or null.
+ */
+export function streetTypeOf(street, lexicon) {
+    const words = street
+        .toLowerCase()
+        .split(/\s+/u)
+        .map((word) => word.replaceAll(/^[\p{P}\p{S}]+|[,;:]+$/gu, ""))
+        .filter((word) => word.length > 0);
+    if (!words.length)
+        return null;
+    const candidates = [
+        words.slice(0, 2).join(" "),
+        words.slice(-2).join(" "),
+        words[0],
+        words.at(-1),
+        words[0].replace(/\.$/u, ""),
+        words.at(-1).replace(/\.$/u, ""),
+    ];
+    for (const candidate of candidates) {
+        const canonical = lexicon.forms.get(candidate);
+        if (canonical)
+            return canonical;
+    }
+    return null;
+}
+/**
+ * Reads one canonical row's variety keys.
+ *
+ * The locality is the row's `locality`, else its `dependent_locality`, else its `region`,
+ * so a streetless premise in a named place is stratified by that place.
+ * The place a row is stratified by joins the dependent locality to it, so an Estonian village is
+ * its own place rather than one of its municipality's, and a city's districts are each their own.
+ */
+export function rowVariety(row, lexicon) {
+    const { components } = row;
+    const street = components.street?.trim() || null;
+    const streetType = street ? streetTypeOf(street, lexicon) : null;
+    const script = dominantScript(row.raw);
+    const tags = Object.keys(components)
+        .filter((tag) => components[tag])
+        .toSorted();
+    const locality = components.locality ?? components.dependent_locality ?? components.region ?? "";
+    const dependentLocality = components.dependent_locality ?? null;
+    return {
+        locality,
+        dependentLocality,
+        place: `${locality}\u0000${dependentLocality ?? ""}`,
+        postcode: components.postcode ?? null,
+        street,
+        streetType,
+        hasUnit: Boolean(components.unit),
+        hasDependentLocality: Boolean(components.dependent_locality),
+        hasVenue: Boolean(components.venue),
+        streetless: !street,
+        script,
+        form: `${tags.join("+")}|${streetType ?? "-"}|${script}`,
+    };
+}
+/**
+ * Accumulates {@link VarietyMeasures} one row at a time.
+ */
+export class VarietyCensus {
+    #rows = 0;
+    #units = 0;
+    #dependent = 0;
+    #venues = 0;
+    #streetless = 0;
+    #withPostcode = 0;
+    #localities = new Map();
+    #dependentLocalities = new Set();
+    #places = new Set();
+    #postcodes = new Set();
+    #streets = new Set();
+    #streetTypes = new Set();
+    #forms = new Set();
+    #scripts = new Map();
+    add(variety) {
+        this.#rows++;
+        this.#localities.set(variety.locality, (this.#localities.get(variety.locality) ?? 0) + 1);
+        this.#places.add(variety.place);
+        if (variety.dependentLocality) {
+            this.#dependentLocalities.add(`${variety.locality}\u0000${variety.dependentLocality}`);
+        }
+        if (variety.postcode) {
+            this.#withPostcode++;
+            this.#postcodes.add(variety.postcode);
+        }
+        if (variety.street) {
+            this.#streets.add(`${variety.locality}\u0000${variety.street}`);
+        }
+        if (variety.streetType) {
+            this.#streetTypes.add(variety.streetType);
+        }
+        this.#forms.add(variety.form);
+        this.#scripts.set(variety.script, (this.#scripts.get(variety.script) ?? 0) + 1);
+        if (variety.hasUnit) {
+            this.#units++;
+        }
+        if (variety.hasDependentLocality) {
+            this.#dependent++;
+        }
+        if (variety.hasVenue) {
+            this.#venues++;
+        }
+        if (variety.streetless) {
+            this.#streetless++;
+        }
+    }
+    measures() {
+        const share = (count) => (this.#rows ? Number((count / this.#rows).toFixed(4)) : 0);
+        let largestLocality = 0;
+        for (const count of this.#localities.values()) {
+            largestLocality = Math.max(largestLocality, count);
+        }
+        return {
+            rows: this.#rows,
+            localities: this.#localities.size,
+            dependentLocalities: this.#dependentLocalities.size,
+            places: this.#places.size,
+            postcodes: this.#postcodes.size,
+            streets: this.#streets.size,
+            streetTypes: this.#streetTypes.size,
+            forms: this.#forms.size,
+            unitShare: share(this.#units),
+            dependentLocalityShare: share(this.#dependent),
+            venueShare: share(this.#venues),
+            streetlessShare: share(this.#streetless),
+            postcodeShare: share(this.#withPostcode),
+            scripts: Object.fromEntries([...this.#scripts].toSorted((a, b) => b[1] - a[1])),
+            largestLocalityShare: share(largestLocality),
+        };
+    }
+}
+/**
+ * Groups `items` by the key `groupOf` assigns, in first-seen order.
+ */
+function groupBy(items, groupOf) {
+    const groups = new Map();
+    for (const item of items) {
+        const key = groupOf(item);
+        let group = groups.get(key);
+        if (!group) {
+            groups.set(key, (group = []));
+        }
+        group.push(item);
+    }
+    return [...groups.values()];
+}
+/**
+ * Interleaves `lists`, the first item of each and then the second of each,
+ * until every list is spent or `limit` items are taken.
+ */
+function interleave(lists, limit) {
+    const ordered = [];
+    const total = Math.min(limit, lists.reduce((sum, list) => sum + list.length, 0));
+    for (let round = 0; ordered.length < total; round++) {
+        for (const list of lists) {
+            if (round < list.length && ordered.length < total) {
+                ordered.push(list[round]);
+            }
+        }
+    }
+    return ordered;
+}
+/**
+ * Chooses the row indices of a variety sample.
+ *
+ * Rows are visited in a seeded random order.
+ * A street keeps its first {@link VarietySampleLimits.perStreet} rows.
+ *
+ * Within a place the kept rows are interleaved across forms, and within a form across postcodes,
+ * and the first {@link VarietySampleLimits.perPlace} are the place's candidates.
+ * Candidates are then taken by their rank within their place, lowest first, so every place supplies
+ * its first row before any supplies its second, until {@link VarietySampleLimits.ceiling}.
+ * The returned indices are ascending.
+ */
+export function selectVarietySample(keys, limits) {
+    const order = keys.map((_, index) => index);
+    shuffleWith(order, mulberry32(limits.seed));
+    const perStreet = new Map();
+    const byPlace = new Map();
+    for (const index of order) {
+        const key = keys[index];
+        const kept = perStreet.get(key.street) ?? 0;
+        if (kept >= limits.perStreet)
+            continue;
+        perStreet.set(key.street, kept + 1);
+        let rows = byPlace.get(key.place);
+        if (!rows) {
+            byPlace.set(key.place, (rows = []));
+        }
+        rows.push(index);
+    }
+    // Candidates bucketed by rank within their place.
+    // The places' order is the shuffled first-seen order, so the partial last
+    // round is a seeded draw across places.
+    const byRank = [];
+    for (const rows of byPlace.values()) {
+        // Forms take turns, and within a form its postcodes take turns, so a rare form such as a unit row
+        // is drawn as early as the common one and a form's rows spread across the place's postcodes.
+        const forms = groupBy(rows, (index) => keys[index].form).map((formRows) => interleave(groupBy(formRows, (index) => keys[index].postcode), limits.perPlace));
+        const ranked = interleave(forms, limits.perPlace);
+        for (const [rank, index] of ranked.entries()) {
+            ;
+            (byRank[rank] ??= []).push(index);
+        }
+    }
+    const chosen = [];
+    for (const bucket of byRank) {
+        if (chosen.length >= limits.ceiling)
+            break;
+        chosen.push(...bucket.slice(0, limits.ceiling - chosen.length));
+    }
+    return chosen.toSorted((a, b) => a - b);
+}
+/**
+ * Reads a complete adapter output, writes its variety sample to `output` and the
+ * report beside it as `<output>.variety.json`, and returns the report.
+ *
+ * The input is read twice, once to key every row and once to copy the chosen lines,
+ * so memory holds integers per row rather than the rows themselves.
+ */
+export async function writeVarietySample(options) {
+    const { input, output, limits } = options;
+    const country = options.country.toUpperCase();
+    const lexicon = await loadStreetTypeLexicon(country);
+    const intern = () => {
+        const ids = new Map();
+        return (value) => {
+            let id = ids.get(value);
+            if (typeof id !== "number") {
+                ids.set(value, (id = ids.size));
+            }
+            return id;
+        };
+    };
+    const placeID = intern();
+    const streetID = intern();
+    const formID = intern();
+    const postcodeID = intern();
+    const keys = [];
+    const lineOfKey = [];
+    const full = new VarietyCensus();
+    const varieties = [];
+    const skipped = {};
+    let line = -1;
+    for await (const text of TextSpliterator.fromAsync(input)) {
+        line++;
+        if (!text.trim())
+            continue;
+        const row = tryParsingJSON(text);
+        if (!row || typeof row !== "object" || !row.components || typeof row.raw !== "string") {
+            skipped["unparseable"] = (skipped["unparseable"] ?? 0) + 1;
+            continue;
+        }
+        if (row.country !== country) {
+            skipped[`country:${row.country}`] = (skipped[`country:${row.country}`] ?? 0) + 1;
+            continue;
+        }
+        const variety = rowVariety(row, lexicon);
+        full.add(variety);
+        varieties.push(variety);
+        lineOfKey.push(line);
+        keys.push({
+            place: placeID(variety.place),
+            // A streetless premise numbered within a place repeats that place as a street's
+            // houses repeat the street, so the place is its street.
+            // A named site (an Estonian farm) is its own name, as a street is.
+            street: streetID(`${variety.place}\u0001${variety.street ?? row.components.venue ?? ""}`),
+            form: formID(variety.form),
+            postcode: postcodeID(variety.postcode ?? ""),
+        });
+    }
+    const chosen = selectVarietySample(keys, limits);
+    const sample = new VarietyCensus();
+    const chosenLines = new Set();
+    for (const index of chosen) {
+        sample.add(varieties[index]);
+        chosenLines.add(lineOfKey[index]);
+    }
+    async function* selectedLines() {
+        let current = -1;
+        for await (const text of TextSpliterator.fromAsync(input)) {
+            current++;
+            if (chosenLines.has(current)) {
+                yield text;
+            }
+        }
+    }
+    await writeLocalTextFile(selectedLines(), output);
+    const fullMeasures = full.measures();
+    const sampleMeasures = sample.measures();
+    const ratio = (kept, total) => (total ? Number((kept / total).toFixed(4)) : 0);
+    const report = {
+        input: resolvePath(input),
+        inputSHA256: await sha256File(input),
+        output: resolvePath(output),
+        outputSHA256: await sha256File(output),
+        country,
+        limits,
+        streetTypeLanguages: lexicon.languages,
+        skipped,
+        full: fullMeasures,
+        sample: sampleMeasures,
+        coverage: {
+            localities: ratio(sampleMeasures.localities, fullMeasures.localities),
+            places: ratio(sampleMeasures.places, fullMeasures.places),
+            postcodes: ratio(sampleMeasures.postcodes, fullMeasures.postcodes),
+            streetTypes: ratio(sampleMeasures.streetTypes, fullMeasures.streetTypes),
+            forms: ratio(sampleMeasures.forms, fullMeasures.forms),
+        },
+    };
+    await writeLocalJSONFile(report, `${resolvePath(output)}.variety.json`);
+    return report;
+}
+//# sourceMappingURL=variety-sample.js.map

package/out/tools/variety-sample.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"variety-sample.js","sourceRoot":"","sources":["../../lib/tools/variety-sample.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,6CAA6C,CAAA;AAChF,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AAC9D,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AAChE,OAAO,EAAwB,WAAW,EAAE,MAAM,SAAS,CAAA;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAE,2BAA2B,EAAE,MAAM,gCAAgC,CAAA;AAG5E;;GAEG;AACH,MAAM,OAAO,GAAG;IACf,OAAO;IACP,UAAU;IACV,QAAQ;IACR,UAAU;IACV,UAAU;IACV,OAAO;IACP,QAAQ;IACR,KAAK;IACL,QAAQ;IACR,MAAM;IACN,YAAY;CACH,CAAA;AAEV,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,IAAI,MAAM,CAAC,cAAc,MAAM,GAAG,EAAE,GAAG,CAAC,CAAU,CAAC,CAAA;AAE5G;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,SAAQ;QAEvC,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,CAAA;QAE7F,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAClD,CAAC;IAED,IAAI,IAAI,GAAG,OAAO,CAAA;IAClB,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACtC,IAAI,KAAK,GAAG,IAAI,EAAE,CAAC;YAClB,IAAI,GAAG,MAAM,CAAA;YACb,IAAI,GAAG,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAeD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAAe;IAC1D,MAAM,QAAQ,GAAG,CAAC,kBAAkB,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAA;IAChH,MAAM,SAAS,GAAa,EAAE,CAAA;IAE9B,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACjC,IAAI,MAAM,UAAU,CAAC,sBAAsB,CAAC,WAAW,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC,EAAE,CAAC;YACzF,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,IAAuB,EAAE;QAC3C,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,oBAAoB,EAAE;YACzD,GAAG,EAAE,sBAAsB,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE;YACnD,QAAQ,EAAE,KAAK;SACf,CAAC,CAAC,QAAQ,EAAE,CAAA;QAEb,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;IAChD,CAAC,CAAA;IAED,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CAAA;IAC9D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEvC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,2BAA2B,CAAC,QAAQ,EAAE,kBAAkB,CAAC,EAAE,CAAC;YACjG,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9B,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,QAAQ,IAAI,SAAS,EAAE,CAAC,CAAA;YAC5C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE,CAAA;AAClE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,MAAc,EAAE,OAA0B;IACtE,MAAM,KAAK,GAAG,MAAM;SAClB,WAAW,EAAE;SACb,KAAK,CAAC,MAAM,CAAC;SACb,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;SAC9D,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAEnC,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IAE9B,MAAM,UAAU,GAAG;QAClB,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAC3B,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QACzB,KAAK,CAAC,CAAC,CAAE;QACT,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE;QACb,KAAK,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAC7B,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;KACjC,CAAA;IAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,SAAS;YAAE,OAAO,SAAS,CAAA;IAChC,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AA0BD;;;;;;;GAOG;AACH,MAAM,UAAU,UAAU,CAAC,GAA6C,EAAE,OAA0B;IACnG,MAAM,EAAE,UAAU,EAAE,GAAG,GAAG,CAAA;IAC1B,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,IAAI,CAAA;IAChD,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAChE,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAEtC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;SAClC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAA8B,CAAC,CAAC;SAC3D,QAAQ,EAAE,CAAA;IAEZ,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,IAAI,UAAU,CAAC,kBAAkB,IAAI,UAAU,CAAC,MAAM,IAAI,EAAE,CAAA;IAChG,MAAM,iBAAiB,GAAG,UAAU,CAAC,kBAAkB,IAAI,IAAI,CAAA;IAE/D,OAAO;QACN,QAAQ;QACR,iBAAiB;QACjB,KAAK,EAAE,GAAG,QAAQ,SAAS,iBAAiB,IAAI,EAAE,EAAE;QACpD,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,IAAI;QACrC,MAAM;QACN,UAAU;QACV,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;QACjC,oBAAoB,EAAE,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC;QAC5D,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;QACnC,UAAU,EAAE,CAAC,MAAM;QACnB,MAAM;QACN,IAAI,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,UAAU,IAAI,GAAG,IAAI,MAAM,EAAE;KACxD,CAAA;AACF,CAAC;AA6BD;;GAEG;AACH,MAAM,OAAO,aAAa;IACzB,KAAK,GAAG,CAAC,CAAA;IACT,MAAM,GAAG,CAAC,CAAA;IACV,UAAU,GAAG,CAAC,CAAA;IACd,OAAO,GAAG,CAAC,CAAA;IACX,WAAW,GAAG,CAAC,CAAA;IACf,aAAa,GAAG,CAAC,CAAA;IACR,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;IACvC,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAA;IACxC,OAAO,GAAG,IAAI,GAAG,EAAU,CAAA;IAC3B,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAC5B,YAAY,GAAG,IAAI,GAAG,EAAU,CAAA;IAChC,MAAM,GAAG,IAAI,GAAG,EAAU,CAAA;IAC1B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7C,GAAG,CAAC,OAAmB;QACtB,IAAI,CAAC,KAAK,EAAE,CAAA;QACZ,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACzF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAE/B,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,SAAS,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAA;QACvF,CAAC;QAED,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,CAAC,aAAa,EAAE,CAAA;YACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACtC,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,SAAS,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QAChE,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACxB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;QAC1C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAC7B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAE/E,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,EAAE,CAAA;QACd,CAAC;QAED,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,UAAU,EAAE,CAAA;QAClB,CAAC;QAED,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,CAAC,OAAO,EAAE,CAAA;QACf,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACxB,IAAI,CAAC,WAAW,EAAE,CAAA;QACnB,CAAC;IACF,CAAC;IAED,QAAQ;QACP,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC3F,IAAI,eAAe,GAAG,CAAC,CAAA;QAEvB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YAC/C,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,KAAK,CAAC,CAAA;QACnD,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;YACjC,mBAAmB,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI;YACnD,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YACzB,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;YAC/B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;YAC3B,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI;YACnC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;YACvB,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;YAC7B,sBAAsB,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;YAC9C,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;YAC/B,eAAe,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC;YACxC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC;YACxC,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/E,oBAAoB,EAAE,KAAK,CAAC,eAAe,CAAC;SAC5C,CAAA;IACF,CAAC;CACD;AA+BD;;GAEG;AACH,SAAS,OAAO,CAAI,KAAmB,EAAE,OAA4B;IACpE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAe,CAAA;IAErC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAE3B,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAA;QAC9B,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjB,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;AAC5B,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAI,KAAgC,EAAE,KAAa;IACrE,MAAM,OAAO,GAAQ,EAAE,CAAA;IAEvB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACrB,KAAK,EACL,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CACjD,CAAA;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;gBACnD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC,CAAA;YAC3B,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAA6B,EAAE,MAA2B;IAC7F,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAA;IAE3C,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;IAE3C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE3C,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAE,CAAA;QACxB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAE3C,IAAI,IAAI,IAAI,MAAM,CAAC,SAAS;YAAE,SAAQ;QAEtC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,CAAA;QAEnC,IAAI,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAEjC,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAA;QACpC,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACjB,CAAC;IAED,kDAAkD;IAClD,0EAA0E;IAC1E,wCAAwC;IACxC,MAAM,MAAM,GAAe,EAAE,CAAA;IAE7B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACrC,kGAAkG;QAClG,6FAA6F;QAC7F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC1E,UAAU,CACT,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC,QAAQ,CAAC,EACnD,MAAM,CAAC,QAAQ,CACf,CACD,CAAA;QAED,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;QAEjD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,CAAC;YAAA,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAA;IAE3B,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO;YAAE,MAAK;QAE1C,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;IAChE,CAAC;IAED,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AACxC,CAAC;AAyBD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAKxC;IACA,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACzC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,CAAA;IAEpD,MAAM,MAAM,GAAG,GAAG,EAAE;QACnB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAA;QAErC,OAAO,CAAC,KAAa,EAAE,EAAE;YACxB,IAAI,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAEvB,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;gBAC5B,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;YAChC,CAAC;YAED,OAAO,EAAE,CAAA;QACV,CAAC,CAAA;IACF,CAAC,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAA;IACxB,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAA;IACzB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAA;IACvB,MAAM,UAAU,GAAG,MAAM,EAAE,CAAA;IAC3B,MAAM,IAAI,GAAmB,EAAE,CAAA;IAC/B,MAAM,SAAS,GAAa,EAAE,CAAA;IAC9B,MAAM,IAAI,GAAG,IAAI,aAAa,EAAE,CAAA;IAChC,MAAM,SAAS,GAAiB,EAAE,CAAA;IAClC,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,IAAI,IAAI,GAAG,CAAC,CAAC,CAAA;IAEb,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,IAAI,EAAE,CAAA;QAEN,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAQ;QAE1B,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAwB,CAAA;QAEvD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,UAAU,IAAI,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YACvF,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;YAE1D,SAAQ;QACT,CAAC;QAED,IAAI,GAAG,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YAC7B,OAAO,CAAC,WAAW,GAAG,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;YAEhF,SAAQ;QACT,CAAC;QAED,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QAExC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACjB,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACvB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEpB,IAAI,CAAC,IAAI,CAAC;YACT,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC;YAC7B,gFAAgF;YAChF,wDAAwD;YACxD,mEAAmE;YACnE,MAAM,EAAE,QAAQ,CAAC,GAAG,OAAO,CAAC,KAAK,SAAS,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACzF,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;YAC1B,QAAQ,EAAE,UAAU,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;SAC5C,CAAC,CAAA;IACH,CAAC;IAED,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAChD,MAAM,MAAM,GAAG,IAAI,aAAa,EAAE,CAAA;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IAErC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAE,CAAC,CAAA;QAC7B,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAE,CAAC,CAAA;IACnC,CAAC;IAED,KAAK,SAAS,CAAC,CAAC,aAAa;QAC5B,IAAI,OAAO,GAAG,CAAC,CAAC,CAAA;QAEhB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3D,OAAO,EAAE,CAAA;YAET,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,CAAA;YACX,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,kBAAkB,CAAC,aAAa,EAAE,EAAE,MAAM,CAAC,CAAA;IAEjD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAA;IACpC,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IACxC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAE9F,MAAM,MAAM,GAAwB;QACnC,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC;QACzB,WAAW,EAAE,MAAM,UAAU,CAAC,KAAK,CAAC;QACpC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;QAC3B,YAAY,EAAE,MAAM,UAAU,CAAC,MAAM,CAAC;QACtC,OAAO;QACP,MAAM;QACN,mBAAmB,EAAE,OAAO,CAAC,SAAS;QACtC,OAAO;QACP,IAAI,EAAE,YAAY;QAClB,MAAM,EAAE,cAAc;QACtB,QAAQ,EAAE;YACT,UAAU,EAAE,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,YAAY,CAAC,UAAU,CAAC;YACrE,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC;YACzD,SAAS,EAAE,KAAK,CAAC,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC;YAClE,WAAW,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW,EAAE,YAAY,CAAC,WAAW,CAAC;YACxE,KAAK,EAAE,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;SACtD;KACD,CAAA;IAED,MAAM,kBAAkB,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,eAAe,CAAC,CAAA;IAEvE,OAAO,MAAM,CAAA;AACd,CAAC"}