@mailwoman/corpus 9.4.0 → 10.1.0

package/out/fi/tools/fetch/ryhti.js

@@ -0,0 +1,327 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download Suomen ympäristökeskus' Ryhti built-environment address CSV for the `ryhti` adapter.
+ *
+ * SYKE publishes the whole country, Åland included, as one gzipped CSV at
+ * `https://paikkatiedot.ymparisto.fi/geoserver/www/open_address.csv.gz`. There is no per-region
+ * distribution and no API, so the harvest is one transfer rather than a paging loop.
+ *
+ * The acquisition belongs under a `tools/` root. `@mailwoman/corpus` is one of the
+ * `TOOLING_PACKAGES` in `dependency-cruiser.config.mjs`, which keep their tooling under `lib/`, so
+ * this module sits beside `#fr/tools/fetch/ban` rather than in an `sdk/` root the workspace does
+ * not declare.
+ *
+ * SYKE's metadata record `{DBD610F4-3392-44CD-B601-BAE8FA547A57}` grants CC BY 4.0, which the
+ * address-source register elects, and states the attribution as `Lähde: Syke Ryhti`. The manifest
+ * records both, and `#fi/adapters/ryhti/adapter` records the license on every row.
+ *
+ * Four measured properties decide what this module does:
+ *
+ * 1. **The file is republished daily and its length changes with it.** `HEAD` answered
+ *    `content-length: 351290196` and `last-modified: Fri, 2 Oct 2026 05:20:14 GMT` on 2026-10-02,
+ *    against `351288399` and `Thu, 1 Oct 2026 05:20:03 GMT` the day before. So the re-run check is
+ *    a `HEAD` compared against the manifest, which costs one small request instead of 351 MB, and
+ *    neither number is written into this module as a constant.
+ * 2. **The host ignores `Range`.** `bytes=0-4095` answers HTTP 200 with no `content-range` and the
+ *    whole `content-length`, and `accept-ranges` is absent, so a resumed transfer is not available
+ *    and an interrupted one restarts. {@linkcode resumableDownload} would refuse the response.
+ * 3. **This module moves bytes and decodes none of them.** The body is streamed to disk compressed, then
+ *    decompressed stream to stream, so no chunk boundary is ever interpreted as a character
+ *    boundary. A `chunk.toString("utf8")` per gzip chunk produces a U+FFFD at every boundary that
+ *    splits a multi-byte character, and 60,014,592 decompressed bytes of this file were scanned for
+ *    `EF BF BD` with 0 found: the replacement characters such a reader reports are its own.
+ * 4. **The decompressed CSV is what the adapter can read.** `#fi/adapters/ryhti/adapter` hands
+ *    `opts.inputPath` to `CSVSpliterator.fromAsync`, which reads the bytes as they are on disk, so
+ *    a path to the `.gz` would be parsed as CSV over gzip bytes. The `.gz` is therefore removed once the CSV is
+ *    written: 793 MB of readable input instead of 351 MB plus a reader the adapter does not have.
+ *    A caller that wants the compressed copy kept passes
+ *    {@linkcode FetchRyhtiOptions.keepCompressed}, which costs 1.14 GB for both.
+ */
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
+import { APIClient } from "@mailwoman/core/api";
+import { gunzipChunks } from "@mailwoman/core/fs/compression";
+import { ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { tryStat } from "@mailwoman/core/fs/readers";
+import { openReadStream, openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { PathBuilder } from "path-ts";
+import { RYHTI_ADAPTER_ID } from "#fi/adapters/ryhti/adapter";
+import { readManifest, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download";
+import { assertHeaderColumns, readDelimitedHeader } from "#tools/fetch/header";
+/**
+ * The one file SYKE publishes for this theme.
+ */
+export const FI_RYHTI_CSV_URL = "https://paikkatiedot.ymparisto.fi/geoserver/www/open_address.csv.gz";
+/**
+ * The directory the download is written under, which is the adapter's `inputPath`'s parent.
+ */
+const SLUG = RYHTI_ADAPTER_ID;
+/**
+ * The decompressed file the adapter reads.
+ */
+export const FI_RYHTI_CSV_FILENAME = "open_address.csv";
+/**
+ * The compressed file as it arrives, kept only while it is being decompressed.
+ */
+const COMPRESSED_FILENAME = "open_address.csv.gz";
+/**
+ * The license the register elects for this file.
+ */
+export const FI_RYHTI_LICENSE = "CC-BY-4.0";
+/**
+ * The attribution SYKE states for its open data, which the model card must carry.
+ */
+export const FI_RYHTI_ATTRIBUTION = "Lähde: Syke Ryhti";
+/**
+ * The columns `#fi/adapters/ryhti/adapter` indexes by name.
+ *
+ * The header is checked against these rather than against a column count, because the adapter reads a
+ * subset by name and a renamed column would otherwise reach it as an empty string on every row.
+ * Which columns the publisher quotes is a property of the edition — the 2026-10-02
+ * edition quotes `address_number`, the two number-part columns, `municipality_number`,
+ * `postal_code` and `location_srid` and leaves the header and every name column bare —
+ * so the check reads a quote-aware split rather than assuming either form.
+ */
+export const FI_RYHTI_REQUIRED_COLUMNS = [
+    "address_key",
+    "address_name_fin",
+    "address_name_swe",
+    "postal_office_fin",
+    "postal_office_swe",
+    "municipality_number",
+    "postal_code",
+    "number_part_of_address_number",
+    "number_part_of_address_number2",
+    "subdivision_letter_of_address_number",
+    "subdivision_letter_of_address_number2",
+];
+/**
+ * Read what the host states about the file.
+ *
+ * Through `APIClient`, which is the interface for a repeated request with a small response.
+ * The body itself is not taken this way: `APIClient` buffers a response, and this one is 351 MB.
+ *
+ * A header the host does not send is reported as `null` rather than raised on,
+ * and the caller then downloads rather than treating the absence as a match.
+ */
+export async function readRyhtiHead(client, url = FI_RYHTI_CSV_URL) {
+    const response = await client.fetch({ method: "head", url });
+    const headers = response.headers;
+    const length = headers?.["content-length"];
+    return {
+        lastModified: headers?.["last-modified"] ?? null,
+        contentLength: length !== undefined && /^\d+$/u.test(length) ? Number(length) : null,
+        contentType: headers?.["content-type"] ?? null,
+    };
+}
+/**
+ * The header line's column names, read from the decompressed file.
+ *
+ * The file is comma-separated, so {@linkcode readDelimitedHeader} is called without a delimiter.
+ *
+ * @throws When the file holds no row at all, so an empty or truncated file reports itself
+ * rather than reading as a file with no columns.
+ */
+export function readRyhtiColumns(path) {
+    return readDelimitedHeader(path);
+}
+/**
+ * Refuse a header that does not name every column the adapter indexes.
+ *
+ * @throws Naming the columns that are absent, so a renamed column is a reported failure
+ * rather than an empty string on every row the adapter emits.
+ */
+export function assertRyhtiColumns(columns, context) {
+    assertHeaderColumns(columns, FI_RYHTI_REQUIRED_COLUMNS, context);
+}
+/**
+ * The directory `#fi/adapters/ryhti/adapter` reads under a fetch root.
+ */
+export function ryhtiInputPath(outRoot) {
+    return outRoot(SLUG)(FI_RYHTI_CSV_FILENAME);
+}
+/**
+ * Whether the CSV on disk is still the one the manifest describes and the host still serves.
+ *
+ * The `HEAD` decides first, because the publisher rewrites the file daily
+ * and states a new length and modification time when it does.
+ * The file's own length is then checked, so a truncated local copy is re-fetched even
+ * where the host has not changed.
+ */
+async function isCurrent(recorded, head, path, verifyDigest) {
+    if (!recorded)
+        return false;
+    // A host that states neither cannot be compared against, so the file is downloaded.
+    if (head.lastModified === null && head.contentLength === null)
+        return false;
+    if (head.lastModified !== null && head.lastModified !== recorded.last_modified)
+        return false;
+    if (head.contentLength !== null && head.contentLength !== recorded.compressed_bytes)
+        return false;
+    const stat = await tryStat(path);
+    if (!stat || stat.size !== recorded.bytes)
+        return false;
+    if (!verifyDigest)
+        return true;
+    return (await sha256File(path)) === recorded.sha256;
+}
+/**
+ * Download the Ryhti address CSV into `options.outputDir` and leave it beside its manifest.
+ *
+ * The client is injected so the re-run check is testable without the network, and so a caller
+ * decides the retry policy. {@linkcode fetchRyhti} is the registry entry point and supplies one.
+ *
+ * Re-runnable: a run whose `HEAD` matches the manifest's recorded modification time and compressed
+ * length, and whose CSV is still on disk at the recorded length, makes no request for the body.
+ *
+ * A full fetch is two requests, the `HEAD` and the body, and 351,290,196 compressed bytes as
+ * measured on 2026-10-02, decompressing to a CSV this module measures rather than predicts.
+ */
+export async function downloadRyhti(client, options) {
+    const { report } = options;
+    const destDir = PathBuilder.from(options.outputDir);
+    await makeDirectories(destDir);
+    const csvPath = destDir(FI_RYHTI_CSV_FILENAME);
+    const compressedPath = destDir(COMPRESSED_FILENAME);
+    const manifestPath = destDir("MANIFEST.json");
+    report?.(`=== ${SLUG}`);
+    // MARK: Ask what the host holds
+    //
+    // One small request.
+    // It is what holds a re-run to this one request when the publisher has not republished,
+    // and the publisher republishes daily.
+    const head = await readRyhtiHead(client, FI_RYHTI_CSV_URL);
+    report?.(`  HEAD: ${head.contentLength === null ? "no content-length" : `${ByteFormatter.formatIEC(head.contentLength)} compressed`}` +
+        `, last-modified ${head.lastModified ?? "absent"}`);
+    const recorded = await readManifest(manifestPath);
+    if (!options.force && (await isCurrent(recorded, head, csvPath, options.verifyDigest ?? false))) {
+        report?.("  ✓ Already current (the host states the modification time and length in MANIFEST) — no download.");
+        return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
+    }
+    // MARK: Transfer the compressed body
+    //
+    // Raw streaming rather than `APIClient`: the body is 351 MB, and `APIClient`
+    // buffers a response it does not stream.
+    // The host ignores `Range`, so an interrupted transfer restarts rather than resumes,
+    // and `withRetries` is what bounds that.
+    let compressedBytes;
+    try {
+        compressedBytes = await withRetries(async () => {
+            const response = await fetch(FI_RYHTI_CSV_URL, {
+                redirect: "follow",
+                signal: options.signal,
+                headers: { accept: "application/gzip, application/x-gzip, */*" },
+            });
+            if (!response.ok)
+                throw new Error(`HTTP ${response.status} ${response.statusText} — ${FI_RYHTI_CSV_URL}`);
+            const contentType = response.headers.get("content-type") ?? "";
+            // The host serves `application/x-gzip`.
+            // An `text/html` body under http 200 is a portal page, which would otherwise
+            // be written to disk and fail later as a corrupt archive.
+            if (contentType.includes("text/html")) {
+                throw new Error(`${FI_RYHTI_CSV_URL} answered ${contentType}, which is a page rather than the file`);
+            }
+            return streamBodyToFile(response, compressedPath);
+        }, { report, label: COMPRESSED_FILENAME, retries: options.retries, retryDelayMs: options.retryDelayMs });
+    }
+    catch (error) {
+        report?.(`  ✗ download failed: ${error instanceof Error ? error.message : String(error)}`);
+        await removePathIfPresent(compressedPath);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    report?.(`  Downloaded ${ByteFormatter.formatIEC(compressedBytes)} (${compressedBytes} bytes) compressed`);
+    // The delivered count rather than the stated one, and a disagreement is reported
+    // rather than trusted in either direction: the publisher rewrites the file daily,
+    // so a `HEAD` taken before the transfer can describe a different edition than the body.
+    if (head.contentLength !== null && head.contentLength !== compressedBytes) {
+        report?.(`  ! the HEAD stated ${head.contentLength} compressed bytes and ${compressedBytes} arrived, ` +
+            `so the file was republished between the two requests`);
+    }
+    // MARK: Decompress, stream to stream
+    //
+    // No `TextDecoder` and no `toString` anywhere on this path.
+    // The bytes reach the CSV as bytes, and the adapter decodes them when it parses.
+    try {
+        await decompressToCSV(compressedPath, csvPath);
+    }
+    catch (error) {
+        report?.(`  ✗ decompress failed: ${error instanceof Error ? error.message : String(error)}`);
+        await removePathIfPresent(compressedPath);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    const csvStat = await tryStat(csvPath);
+    if (!csvStat) {
+        report?.(`  ✗ the decompressed CSV is not at ${csvPath.toString()}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    // MARK: Check the header the adapter indexes by name
+    let columns;
+    try {
+        columns = await readRyhtiColumns(csvPath);
+        assertRyhtiColumns(columns, `${SLUG}: ${FI_RYHTI_CSV_FILENAME}`);
+    }
+    catch (error) {
+        report?.(`  ✗ ${error instanceof Error ? error.message : String(error)}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    report?.(`  Decompressed to ${ByteFormatter.formatIEC(csvStat.size)} over ${columns.length} columns`);
+    if (!options.keepCompressed) {
+        await removePathIfPresent(compressedPath);
+        report?.("  Removed the archive (the CSV is kept)");
+    }
+    // MARK: Write the manifest
+    const manifest = {
+        source_url: FI_RYHTI_CSV_URL,
+        downloaded_at: new Date().toISOString(),
+        filename: FI_RYHTI_CSV_FILENAME,
+        sha256: await sha256File(csvPath),
+        bytes: csvStat.size,
+        license: FI_RYHTI_LICENSE,
+        attribution: FI_RYHTI_ATTRIBUTION,
+        last_modified: head.lastModified,
+        compressed_bytes: compressedBytes,
+        columns,
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${ByteFormatter.formatIEC(manifest.bytes)}  sha256=${manifest.sha256}`);
+    report?.(`  MANIFEST written to ${manifestPath.toString()}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+/**
+ * Download the Ryhti address CSV into `<outRoot>/ryhti/`.
+ *
+ * The registry entry point.
+ * `#fi/adapters/ryhti/adapter` is pointed at {@linkcode ryhtiInputPath},
+ * the `open_address.csv` inside that directory, rather than at the directory itself.
+ */
+export async function fetchRyhti(options, report) {
+    await using client = new APIClient({ displayName: SLUG, retry: true });
+    // Awaited rather than returned: `await using` disposes the client when this scope exits,
+    // and a disposed `APIClient` refuses every later request.
+    return await downloadRyhti(client, {
+        outputDir: options.outRoot(SLUG),
+        keepCompressed: options.keepCompressed,
+        verifyDigest: options.verifyDigest,
+        force: options.force,
+        retries: options.retries,
+        retryDelayMs: options.retryDelayMs,
+        signal: options.signal,
+        report,
+    });
+}
+/**
+ * Decompress one gzip file to another path, stream to stream.
+ *
+ * Exported because it is the step the U+FFFD report came out of, and a test exercises it
+ * directly: a multi-byte character split across two gzip chunks must arrive whole.
+ */
+export async function decompressToCSV(source, destination) {
+    const decompressed = gunzipChunks(openReadStream(source));
+    await pipeline(Readable.fromWeb(decompressed), openWriteStream(destination));
+}
+//# sourceMappingURL=ryhti.js.map

package/out/fi/tools/fetch/ryhti.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ryhti.js","sourceRoot":"","sources":["../../../../lib/fi/tools/fetch/ryhti.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAEH;4GAC4G;AAE5G,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAA;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAChG,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAE7D,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAClG,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AAE9E;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,qEAAqE,CAAA;AAErG;;GAEG;AACH,MAAM,IAAI,GAAG,gBAAgB,CAAA;AAE7B;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,kBAAkB,CAAA;AAEvD;;GAEG;AACH,MAAM,mBAAmB,GAAG,qBAAqB,CAAA;AAEjD;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,WAAW,CAAA;AAE3C;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,mBAAmB,CAAA;AAEvD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAsB;IAC3D,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,mBAAmB;IACnB,mBAAmB;IACnB,qBAAqB;IACrB,aAAa;IACb,+BAA+B;IAC/B,gCAAgC;IAChC,sCAAsC;IACtC,uCAAuC;CACvC,CAAA;AAuCD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAgC,EAAE,GAAG,GAAG,gBAAgB;IAC3F,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CAAU,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;IACrE,MAAM,OAAO,GAAG,QAAQ,CAAC,OAA6C,CAAA;IACtE,MAAM,MAAM,GAAG,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAA;IAE1C,OAAO;QACN,YAAY,EAAE,OAAO,EAAE,CAAC,eAAe,CAAC,IAAI,IAAI;QAChD,aAAa,EAAE,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;QACpF,WAAW,EAAE,OAAO,EAAE,CAAC,cAAc,CAAC,IAAI,IAAI;KAC9C,CAAA;AACF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAqB;IACrD,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAA;AACjC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA0B,EAAE,OAAe;IAC7E,mBAAmB,CAAC,OAAO,EAAE,yBAAyB,EAAE,OAAO,CAAC,CAAA;AACjE,CAAC;AA4CD;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAoC;IAClE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,qBAAqB,CAAC,CAAA;AAC5C,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,SAAS,CACvB,QAA8B,EAC9B,IAAe,EACf,IAAqB,EACrB,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAA;IAE3B,oFAAoF;IACpF,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI;QAAE,OAAO,KAAK,CAAA;IAE3E,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ,CAAC,aAAa;QAAE,OAAO,KAAK,CAAA;IAE5F,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,aAAa,KAAK,QAAQ,CAAC,gBAAgB;QAAE,OAAO,KAAK,CAAA;IAEjG,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK;QAAE,OAAO,KAAK,CAAA;IAEvD,IAAI,CAAC,YAAY;QAAE,OAAO,IAAI,CAAA;IAE9B,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAA;AACpD,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC,EAChC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAC1B,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IAEnD,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,OAAO,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAA;IAC9C,MAAM,cAAc,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAA;IACnD,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IAE7C,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,gCAAgC;IAChC,EAAE;IACF,qBAAqB;IACrB,wFAAwF;IACxF,uCAAuC;IAEvC,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAA;IAE1D,MAAM,EAAE,CACP,WAAW,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE;QAC3H,mBAAmB,IAAI,CAAC,YAAY,IAAI,QAAQ,EAAE,CACnD,CAAA;IAED,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAgB,YAAY,CAAC,CAAA;IAEhE,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC;QACjG,MAAM,EAAE,CAAC,mGAAmG,CAAC,CAAA;QAE7G,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IAC9D,CAAC;IAED,qCAAqC;IACrC,EAAE;IACF,6EAA6E;IAC7E,yCAAyC;IACzC,qFAAqF;IACrF,yCAAyC;IAEzC,IAAI,eAAuB,CAAA;IAE3B,IAAI,CAAC;QACJ,eAAe,GAAG,MAAM,WAAW,CAClC,KAAK,IAAI,EAAE;YACV,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,gBAAgB,EAAE;gBAC9C,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,OAAO,EAAE,EAAE,MAAM,EAAE,2CAA2C,EAAE;aAChE,CAAC,CAAA;YAEF,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,MAAM,gBAAgB,EAAE,CAAC,CAAA;YAEzG,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAA;YAE9D,wCAAwC;YACxC,6EAA6E;YAC7E,0DAA0D;YAC1D,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CAAC,GAAG,gBAAgB,aAAa,WAAW,wCAAwC,CAAC,CAAA;YACrG,CAAC;YAED,OAAO,gBAAgB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAA;QAClD,CAAC,EACD,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CACpG,CAAA;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,wBAAwB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1F,MAAM,mBAAmB,CAAC,cAAc,CAAC,CAAA;QAEzC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,gBAAgB,aAAa,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,eAAe,oBAAoB,CAAC,CAAA;IAE1G,iFAAiF;IACjF,kFAAkF;IAClF,wFAAwF;IACxF,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;QAC3E,MAAM,EAAE,CACP,uBAAuB,IAAI,CAAC,aAAa,yBAAyB,eAAe,YAAY;YAC5F,sDAAsD,CACvD,CAAA;IACF,CAAC;IAED,qCAAqC;IACrC,EAAE;IACF,4DAA4D;IAC5D,iFAAiF;IAEjF,IAAI,CAAC;QACJ,MAAM,eAAe,CAAC,cAAc,EAAE,OAAO,CAAC,CAAA;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC5F,MAAM,mBAAmB,CAAC,cAAc,CAAC,CAAA;QAEzC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;IAEtC,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,sCAAsC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;QAEpE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,qDAAqD;IAErD,IAAI,OAA0B,CAAA;IAE9B,IAAI,CAAC;QACJ,OAAO,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAA;QACzC,kBAAkB,CAAC,OAAO,EAAE,GAAG,IAAI,KAAK,qBAAqB,EAAE,CAAC,CAAA;IACjE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAEzE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,qBAAqB,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,MAAM,UAAU,CAAC,CAAA;IAErG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC7B,MAAM,mBAAmB,CAAC,cAAc,CAAC,CAAA;QAEzC,MAAM,EAAE,CAAC,yCAAyC,CAAC,CAAA;IACpD,CAAC;IAED,2BAA2B;IAE3B,MAAM,QAAQ,GAAkB;QAC/B,UAAU,EAAE,gBAAgB;QAC5B,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,qBAAqB;QAC/B,MAAM,EAAE,MAAM,UAAU,CAAC,OAAO,CAAC;QACjC,KAAK,EAAE,OAAO,CAAC,IAAI;QACnB,OAAO,EAAE,gBAAgB;QACzB,WAAW,EAAE,oBAAoB;QACjC,aAAa,EAAE,IAAI,CAAC,YAAY;QAChC,gBAAgB,EAAE,eAAe;QACjC,OAAO;KACP,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,aAAa,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;IACrF,MAAM,EAAE,CAAC,yBAAyB,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAE5D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B,EAAE,MAA+B;IAC3F,YAAY,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtE,yFAAyF;IACzF,0DAA0D;IAC1D,OAAO,MAAM,aAAa,CAAC,MAAM,EAAE;QAClC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM;KACN,CAAC,CAAA;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAuB,EAAE,WAA4B;IAC1F,MAAM,YAAY,GAAG,YAAY,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAA;IAEzD,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAsD,CAAC,EAAE,eAAe,CAAC,WAAW,CAAC,CAAC,CAAA;AACvH,CAAC"}

package/out/fingerprints.d.ts

@@ -0,0 +1,68 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file A fixed-capacity set of 128-bit fingerprints in one flat `Uint32Array`.
+ *
+ *   The adapter runner deduplicates by holding every key it has seen. A V8 `Set` refuses its 16,777,217th
+ *   entry outright, so the runner capped itself at 10,000,000 keys. Three adapters passed that cap in
+ *   `v0.7.0-de-holdout`. The duplicate census over that build measured what the cap costs: of the
+ *   2,491,399 duplicate rows the three adapters wrote, 1,490,992 duplicate a key first seen after the cap.
+ *   A membership test on a capped set cannot refuse those rows.
+ *
+ *   This holds the keys as fingerprints instead. Each slot uses four 32-bit words and open addressing with linear
+ *   probing. The allocation happens once at construction. The entry count is bounded by the array
+ *   the caller sized rather than by a V8 limit. A 2^27-slot table holds 57,570,829 keys at load 0.43 in
+ *   2.0 GiB outside the V8 heap.
+ *
+ *   **A fingerprint collision drops a legitimate row.** Two distinct keys sharing all 128 bits are read as
+ *   one and the second row is dropped. Over 57,847,619 keys the expected number of colliding pairs is
+ *   about 4.9e-24, so the table trades that for the 1,490,992 duplicates a capped set admits. A caller
+ *   that cannot accept any false drop keeps the `Set`.
+ */
+/**
+ * The load factor at which linear probing's expected probe count stays low enough to ignore.
+ *
+ * Above this the table refuses further keys rather than degrading, because a build whose
+ * dedup slows by an order of magnitude partway through reports no cause.
+ */
+export declare const FINGERPRINT_MAX_LOAD = 0.7;
+/**
+ * The largest slot count this accepts, as a base-2 logarithm.
+ *
+ * 2^30 slots hold 751,619,276 keys at the load limit and occupy 16 GiB.
+ * Past that the backing `Uint32Array` exceeds what one allocation can hold on a 30.5 GB host,
+ * A run that asks for it aborts on the allocation rather than on the key that overflowed.
+ */
+export declare const MAX_SLOTS_LOG2 = 30;
+/**
+ * A fingerprint set sized once, holding 128-bit values outside the V8 heap.
+ */
+export declare class FingerprintSet {
+    #private;
+    /**
+     * @param slotsLog2 Base-2 logarithm of the slot count, so the mask is exact.
+     * 27 gives 134,217,728 slots in 2.0 GiB.
+     */
+    constructor(slotsLog2: number);
+    get size(): number;
+    /**
+     * How many keys this table accepts before {@linkcode add} throws.
+     */
+    get limit(): number;
+    get slots(): number;
+    /**
+     * Bytes the table occupies outside the V8 heap.
+     * They are absent from `--max-old-space-size`.
+     */
+    get bytes(): number;
+    /**
+     * Add a key's fingerprint.
+     *
+     * @returns `true` when the key is new, `false` when this table already holds its fingerprint.
+     * @throws When the table is at {@linkcode FINGERPRINT_MAX_LOAD}.
+     * A silent degradation would make a build slow down partway through with no cause to read.
+     */
+    add(key: string): boolean;
+}
+//# sourceMappingURL=fingerprints.d.ts.map

package/out/fingerprints.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fingerprints.d.ts","sourceRoot":"","sources":["../lib/fingerprints.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AASH;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,MAAM,CAAA;AAEvC;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,KAAK,CAAA;AAEhC;;GAEG;AACH,qBAAa,cAAc;;IAO1B;;;OAGG;IACH,YAAY,SAAS,EAAE,MAAM,EAW5B;IAED,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;OAEG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;;OAGG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;;;;;OAMG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAgDxB;CACD"}

package/out/fingerprints.js

@@ -0,0 +1,130 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file A fixed-capacity set of 128-bit fingerprints in one flat `Uint32Array`.
+ *
+ *   The adapter runner deduplicates by holding every key it has seen. A V8 `Set` refuses its 16,777,217th
+ *   entry outright, so the runner capped itself at 10,000,000 keys. Three adapters passed that cap in
+ *   `v0.7.0-de-holdout`. The duplicate census over that build measured what the cap costs: of the
+ *   2,491,399 duplicate rows the three adapters wrote, 1,490,992 duplicate a key first seen after the cap.
+ *   A membership test on a capped set cannot refuse those rows.
+ *
+ *   This holds the keys as fingerprints instead. Each slot uses four 32-bit words and open addressing with linear
+ *   probing. The allocation happens once at construction. The entry count is bounded by the array
+ *   the caller sized rather than by a V8 limit. A 2^27-slot table holds 57,570,829 keys at load 0.43 in
+ *   2.0 GiB outside the V8 heap.
+ *
+ *   **A fingerprint collision drops a legitimate row.** Two distinct keys sharing all 128 bits are read as
+ *   one and the second row is dropped. Over 57,847,619 keys the expected number of colliding pairs is
+ *   about 4.9e-24, so the table trades that for the 1,490,992 duplicates a capped set admits. A caller
+ *   that cannot accept any false drop keeps the `Set`.
+ */
+import { md5Bytes } from "@mailwoman/core/hash";
+/**
+ * Words per slot: 128 bits of fingerprint.
+ */
+const WORDS_PER_SLOT = 4;
+/**
+ * The load factor at which linear probing's expected probe count stays low enough to ignore.
+ *
+ * Above this the table refuses further keys rather than degrading, because a build whose
+ * dedup slows by an order of magnitude partway through reports no cause.
+ */
+export const FINGERPRINT_MAX_LOAD = 0.7;
+/**
+ * The largest slot count this accepts, as a base-2 logarithm.
+ *
+ * 2^30 slots hold 751,619,276 keys at the load limit and occupy 16 GiB.
+ * Past that the backing `Uint32Array` exceeds what one allocation can hold on a 30.5 GB host,
+ * A run that asks for it aborts on the allocation rather than on the key that overflowed.
+ */
+export const MAX_SLOTS_LOG2 = 30;
+/**
+ * A fingerprint set sized once, holding 128-bit values outside the V8 heap.
+ */
+export class FingerprintSet {
+    #words;
+    #slots;
+    #mask;
+    #limit;
+    #size = 0;
+    /**
+     * @param slotsLog2 Base-2 logarithm of the slot count, so the mask is exact.
+     * 27 gives 134,217,728 slots in 2.0 GiB.
+     */
+    constructor(slotsLog2) {
+        if (!Number.isInteger(slotsLog2) || slotsLog2 < 1 || slotsLog2 > MAX_SLOTS_LOG2) {
+            throw new RangeError(`FingerprintSet: slotsLog2 must be an integer in 1..${MAX_SLOTS_LOG2}, received ${slotsLog2}`);
+        }
+        this.#slots = 2 ** slotsLog2;
+        this.#mask = this.#slots - 1;
+        this.#limit = Math.floor(this.#slots * FINGERPRINT_MAX_LOAD);
+        this.#words = new Uint32Array(this.#slots * WORDS_PER_SLOT);
+    }
+    get size() {
+        return this.#size;
+    }
+    /**
+     * How many keys this table accepts before {@linkcode add} throws.
+     */
+    get limit() {
+        return this.#limit;
+    }
+    get slots() {
+        return this.#slots;
+    }
+    /**
+     * Bytes the table occupies outside the V8 heap.
+     * They are absent from `--max-old-space-size`.
+     */
+    get bytes() {
+        return this.#words.byteLength;
+    }
+    /**
+     * Add a key's fingerprint.
+     *
+     * @returns `true` when the key is new, `false` when this table already holds its fingerprint.
+     * @throws When the table is at {@linkcode FINGERPRINT_MAX_LOAD}.
+     * A silent degradation would make a build slow down partway through with no cause to read.
+     */
+    add(key) {
+        const digest = md5Bytes(key);
+        // A slot holding four zero words is empty, so an all-zero fingerprint would read as one.
+        // The low bit of the first word is forced.
+        // This merges values whose fingerprints differ only there.
+        // The forced bit costs one bit of the 128.
+        //
+        // `>>> 0` is required because `|` evaluates as int32.
+        // A word above 2^31 comes back negative.
+        // A negative value never equals the unsigned value stored by `Uint32Array`.
+        // Without it every key reads as new.
+        const w0 = (digest.readUInt32LE(0) | 1) >>> 0;
+        const w1 = digest.readUInt32LE(4);
+        const w2 = digest.readUInt32LE(8);
+        const w3 = digest.readUInt32LE(12);
+        let slot = w0 & this.#mask;
+        for (;;) {
+            const at = slot * WORDS_PER_SLOT;
+            const found = this.#words[at];
+            if (found === 0) {
+                if (this.#size >= this.#limit) {
+                    throw new Error(`FingerprintSet: ${this.#size.toLocaleString()} keys reached the load limit of ` +
+                        `${this.#limit.toLocaleString()} over ${this.#slots.toLocaleString()} slots. Construct it with a ` +
+                        `larger slotsLog2: probing past this load costs an order of magnitude per lookup and reports no cause.`);
+                }
+                this.#words[at] = w0;
+                this.#words[at + 1] = w1;
+                this.#words[at + 2] = w2;
+                this.#words[at + 3] = w3;
+                this.#size++;
+                return true;
+            }
+            if (found === w0 && this.#words[at + 1] === w1 && this.#words[at + 2] === w2 && this.#words[at + 3] === w3) {
+                return false;
+            }
+            slot = (slot + 1) & this.#mask;
+        }
+    }
+}
+//# sourceMappingURL=fingerprints.js.map

package/out/fingerprints.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fingerprints.js","sourceRoot":"","sources":["../lib/fingerprints.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAE/C;;GAEG;AACH,MAAM,cAAc,GAAG,CAAC,CAAA;AAExB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAEvC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,EAAE,CAAA;AAEhC;;GAEG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,CAAa;IACnB,MAAM,CAAQ;IACd,KAAK,CAAQ;IACb,MAAM,CAAQ;IACvB,KAAK,GAAG,CAAC,CAAA;IAET;;;OAGG;IACH,YAAY,SAAiB;QAC5B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,cAAc,EAAE,CAAC;YACjF,MAAM,IAAI,UAAU,CACnB,sDAAsD,cAAc,cAAc,SAAS,EAAE,CAC7F,CAAA;QACF,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAA;QAC5B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,oBAAoB,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,CAAA;IAC5D,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAA;IAClB,CAAC;IAED;;OAEG;IACH,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,MAAM,CAAA;IACnB,CAAC;IAED,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,MAAM,CAAA;IACnB,CAAC;IAED;;;OAGG;IACH,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAA;IAC9B,CAAC;IAED;;;;;;OAMG;IACH,GAAG,CAAC,GAAW;QACd,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAE5B,yFAAyF;QACzF,2CAA2C;QAC3C,2DAA2D;QAC3D,2CAA2C;QAC3C,EAAE;QACF,sDAAsD;QACtD,yCAAyC;QACzC,4EAA4E;QAC5E,qCAAqC;QACrC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;QAC7C,MAAM,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACjC,MAAM,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACjC,MAAM,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;QAElC,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAA;QAE1B,SAAS,CAAC;YACT,MAAM,EAAE,GAAG,IAAI,GAAG,cAAc,CAAA;YAChC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAE,CAAA;YAE9B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CACd,mBAAmB,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,kCAAkC;wBAC/E,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,SAAS,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,8BAA8B;wBAClG,uGAAuG,CACxG,CAAA;gBACF,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,CAAA;gBACpB,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAA;gBACxB,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAA;gBACxB,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAA;gBAExB,IAAI,CAAC,KAAK,EAAE,CAAA;gBAEZ,OAAO,IAAI,CAAA;YACZ,CAAC;YAED,IAAI,KAAK,KAAK,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;gBAC5G,OAAO,KAAK,CAAA;YACb,CAAC;YAED,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAA;QAC/B,CAAC;IACF,CAAC;CACD"}