@mailwoman/corpus 10.0.0 → 10.2.0

package/out/parquet/writers.js

@@ -0,0 +1,252 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Writes Parquet files and the manifest that describes their contents.
+ *
+ *   This module exposes two writers. {@linkcode writeParquetFile} writes one file from rows already in memory.
+ *   {@linkcode writeParquetSplits} streams per-split row iterables into as many files as the row cap requires. It
+ *   returns the manifest. Both writers create their parent directory, as `@mailwoman/core/fs` writers do
+ *   encode rather than repeat at each call site.
+ *
+ *   Layout under `<outputDir>`:
+ *
+ *   ```
+ *   corpus-v<version>/
+ *     manifest.json
+ *     train/
+ *       part-0000.parquet
+ *       part-0001.parquet
+ *     val/
+ *       part-0000.parquet
+ *     test/
+ *       part-0000.parquet
+ *   ```
+ *
+ *   Each file caps at `rowsPerFile` (default 1,000,000). Within a file DuckDB writes a row group every
+ *   {@linkcode ROW_GROUP_SIZE} rows. The manifest captures every file's path, row count, byte size and SHA-256, the
+ *   digest computed by re-reading the file once after close — cheap against the cost of writing it.
+ *
+ *   A part is closed by its row count and never by a source boundary, so one part may hold the tail of one source and
+ *   the head of the next. A reader must therefore take a file's sources from all of its rows.
+ */
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers";
+import { pathExists, tryStat } from "@mailwoman/core/fs/readers/stat";
+import { openWriteStream } from "@mailwoman/core/fs/streams";
+import { temporaryDirectory } from "@mailwoman/core/fs/temporary";
+import { makeDirectories, writeLocalBuffer, writeLocalJSONFile } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { stringifyJSON } from "@mailwoman/core/json";
+import { once } from "@mailwoman/core/utils/events";
+import { Field, Int32, List, Table as ArrowTable, tableToIPC, Utf8, vectorFromArray } from "apache-arrow";
+import { Compression, Table as WasmTable, WriterPropertiesBuilder, writeParquet } from "parquet-wasm";
+import { resolvePathBuilder } from "path-ts";
+import { corpusDirectoryName } from "#directory";
+import { escapeSQLIdentifier, escapeSQLString, openDuckDB } from "#parquet/duckdb";
+import { PARQUET_COLUMN_TYPES, PARQUET_COLUMNS, PARQUET_COMPRESSION, ROW_GROUP_SIZE, rowToParquet, ROWS_PER_FILE, } from "#parquet/schema";
+const stringListType = new List(new Field("item", new Utf8(), true));
+const int32ListType = new List(new Field("item", new Int32(), true));
+function parquetTable(rows) {
+    return new ArrowTable({
+        raw: vectorFromArray(rows.map((row) => row.raw), new Utf8()),
+        tokens: vectorFromArray(rows.map((row) => row.tokens), stringListType),
+        labels: vectorFromArray(rows.map((row) => row.labels), stringListType),
+        span_starts: vectorFromArray(rows.map((row) => row.span_starts), int32ListType),
+        span_ends: vectorFromArray(rows.map((row) => row.span_ends), int32ListType),
+        span_tags: vectorFromArray(rows.map((row) => row.span_tags), stringListType),
+        country: vectorFromArray(rows.map((row) => row.country), new Utf8()),
+        locale: vectorFromArray(rows.map((row) => row.locale ?? null), new Utf8()),
+        source: vectorFromArray(rows.map((row) => row.source), new Utf8()),
+        source_id: vectorFromArray(rows.map((row) => row.source_id), new Utf8()),
+        corpus_version: vectorFromArray(rows.map((row) => row.corpus_version), new Utf8()),
+        license: vectorFromArray(rows.map((row) => row.license), new Utf8()),
+        register: vectorFromArray(rows.map((row) => row.register ?? null), new Utf8()),
+        surface: vectorFromArray(rows.map((row) => row.surface), new Utf8()),
+        recipe: vectorFromArray(rows.map((row) => row.recipe ?? null), new Utf8()),
+        base_source_id: vectorFromArray(rows.map((row) => row.base_source_id ?? null), new Utf8()),
+    });
+}
+/**
+ * Write one parquet file from rows held in memory.
+ */
+export async function writeParquetFile(rows, path) {
+    const arrow = parquetTable(rows);
+    const wasmTable = WasmTable.fromIPCStream(tableToIPC(arrow, "stream"));
+    const properties = new WriterPropertiesBuilder()
+        .setCompression(Compression.SNAPPY)
+        .setCreatedBy("mailwoman")
+        .setMaxRowGroupSize(ROW_GROUP_SIZE)
+        .build();
+    // parquet-wasm serializes key-value metadata through a hash map.
+    // Its order can change between writes.
+    // File identity and provenance live in manifest.json, so omitting file
+    // metadata preserves deterministic bytes.
+    await writeLocalBuffer(writeParquet(wasmTable, properties), path);
+}
+async function writeStagedParquet(stagePath, outputPath) {
+    using db = await openDuckDB();
+    const columns = [...PARQUET_COLUMNS];
+    const columnsLiteral = "{" + columns.map((column) => `'${column}': '${PARQUET_COLUMN_TYPES[column]}'`).join(", ") + "}";
+    const selectList = columns.map(escapeSQLIdentifier).join(", ");
+    await db.run("SET preserve_insertion_order=true");
+    await db.run(`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath)}', ` +
+        `columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
+        `TO '${escapeSQLString(outputPath)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${ROW_GROUP_SIZE})`);
+}
+async function writeStagedRow(stage, row) {
+    if (!stage.write(stringifyJSON(row) + "\n")) {
+        await once(stage, "drain");
+    }
+}
+/**
+ * Streams labeled rows into `.parquet` files, with one set per split.
+ * It writes the manifest that describes them.
+ *
+ * Splits are processed sequentially so only one file is open at a time.
+ * Rows are staged to newline-delimited JSON with backpressure, then DuckDB
+ * writes the Parquet file from disk.
+ */
+export async function writeParquetSplits(perSplit, opts) {
+    const rowsPerFile = opts.rowsPerFile ?? ROWS_PER_FILE;
+    const corpusDir = resolvePathBuilder(opts.outputDir, corpusDirectoryName(opts.corpusVersion));
+    await makeDirectories(corpusDir);
+    const manifestPath = corpusDir("MANIFEST.json");
+    const files = opts.resume ? await readFinishedParquetFiles(manifestPath) : [];
+    /**
+     * The manifest as the finished files describe it.
+     *
+     * `counts` and `total_rows` are summed from the descriptors rather than from a running row counter,
+     * so a manifest written between two files never claims the rows of the file being written.
+     */
+    const manifestNow = () => {
+        const counts = { train: 0, val: 0, test: 0 };
+        let totalRows = 0;
+        for (const file of files) {
+            counts[file.split] += file.rows;
+            totalRows += file.rows;
+        }
+        return {
+            corpus_version: opts.corpusVersion,
+            schema: PARQUET_COLUMNS,
+            rows_per_slice: rowsPerFile,
+            row_group_size: ROW_GROUP_SIZE,
+            slices: files.toSorted((a, b) => a.split === b.split ? a.path.localeCompare(b.path) : a.split.localeCompare(b.split)),
+            counts,
+            total_rows: totalRows,
+        };
+    };
+    for (const split of ["train", "val", "test"]) {
+        const rows = perSplit[split];
+        if (!rows)
+            continue;
+        await using staging = await temporaryDirectory(`mailwoman-parquet-${split}-`);
+        const finished = files.filter((file) => file.split === split);
+        // The rows of this split already in `.parquet` files.
+        // A resumed run reads the same iterable from its start, so these rows arrive again
+        // and are discarded rather than written twice.
+        let skipRows = finished.reduce((total, file) => total + file.rows, 0);
+        let fileIndex = finished.length;
+        let path = "";
+        let stagePath = "";
+        let fileRows = 0;
+        let stage = null;
+        let firstSourceID = "";
+        let lastSourceID = "";
+        const openFile = async () => {
+            const splitDir = corpusDir(split);
+            await makeDirectories(splitDir);
+            // A string, because the manifest records it.
+            path = splitDir(`part-${String(fileIndex).padStart(4, "0")}.parquet`).toString();
+            // A string, because DuckDB reads it inside SQL text.
+            stagePath = staging.path(`part-${String(fileIndex).padStart(4, "0")}.ndjson`).toString();
+            stage = staging.use(openWriteStream(stagePath));
+            fileRows = 0;
+            firstSourceID = "";
+            lastSourceID = "";
+        };
+        const closeFile = async () => {
+            const activeStage = stage;
+            if (!activeStage)
+                return;
+            await new Promise((resolve, reject) => {
+                activeStage.end((error) => (error ? reject(error) : resolve()));
+            });
+            if (fileRows) {
+                await writeStagedParquet(stagePath, path);
+                const fileStat = await tryStat(path);
+                const sha256 = await sha256File(path);
+                files.push({
+                    split,
+                    path,
+                    format: "parquet",
+                    compression: PARQUET_COMPRESSION,
+                    rows: fileRows,
+                    bytes: fileStat?.size ?? 0,
+                    sha256,
+                    first_source_id: firstSourceID,
+                    last_source_id: lastSourceID,
+                });
+                // Written per file rather than once at the end, so an interrupted build resumes
+                // at the next file index instead of rewriting every split.
+                // About 700 small writes over a full build.
+                await writeLocalJSONFile(manifestNow(), manifestPath);
+            }
+            stage = null;
+            fileRows = 0;
+        };
+        for await (const row of rows) {
+            if (skipRows > 0) {
+                skipRows--;
+                continue;
+            }
+            if (!path) {
+                await openFile();
+            }
+            const pq = rowToParquet(row);
+            if (!stage)
+                throw new Error("Parquet file writer was not opened");
+            await writeStagedRow(stage, pq);
+            fileRows++;
+            if (fileRows === 1) {
+                firstSourceID = row.source_id;
+            }
+            lastSourceID = row.source_id;
+            if (fileRows >= rowsPerFile) {
+                await closeFile();
+                fileIndex++;
+                path = "";
+            }
+        }
+        await closeFile();
+    }
+    const manifest = manifestNow();
+    await writeLocalJSONFile(manifest, manifestPath);
+    return manifest;
+}
+/**
+ * The finished `.parquet` files an earlier run recorded, each verified against its file on disk.
+ *
+ * @throws When a recorded file is missing or its size differs from the descriptor.
+ * The writer creates a descriptor only after closing the file.
+ * Either case therefore means the file changed after its run.
+ * The writer would produce a corpus whose manifest describes bytes the files
+ * no longer contain if it continued.
+ */
+async function readFinishedParquetFiles(manifestPath) {
+    if (!(await pathExists(manifestPath)))
+        return [];
+    const manifest = await readLocalJSONFile(manifestPath);
+    for (const file of manifest.slices) {
+        const stat = await tryStat(file.path);
+        if (!stat) {
+            throw new Error(`resume refused: ${manifestPath.toString()} records ${file.path} and no file sits at that path. ` +
+                `Delete the manifest to write every parquet file again.`);
+        }
+        if (stat.size !== file.bytes) {
+            throw new Error(`resume refused: ${file.path} is ${stat.size.toLocaleString()} bytes and ` +
+                `${manifestPath.toString()} records ${file.bytes.toLocaleString()}.`);
+        }
+    }
+    return [...manifest.slices];
+}
+//# sourceMappingURL=writers.js.map

package/out/parquet/writers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"writers.js","sourceRoot":"","sources":["../../lib/parquet/writers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAA;AACrE,OAAO,EAAE,eAAe,EAAoB,MAAM,4BAA4B,CAAA;AAC9E,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAClG,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,8BAA8B,CAAA;AACnD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AACzG,OAAO,EAAE,WAAW,EAAE,KAAK,IAAI,SAAS,EAAE,uBAAuB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AACrG,OAAO,EAAwB,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAElE,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAChD,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAClF,OAAO,EACN,oBAAoB,EACpB,eAAe,EACf,mBAAmB,EAEnB,cAAc,EACd,YAAY,EACZ,aAAa,GACb,MAAM,iBAAiB,CAAA;AAIxB,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC,CAAA;AACpE,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC,CAAA;AAEpE,SAAS,YAAY,CAAC,IAA2B;IAChD,OAAO,IAAI,UAAU,CAAC;QACrB,GAAG,EAAE,eAAe,CACnB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAC1B,IAAI,IAAI,EAAE,CACV;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAC7B,cAAc,CACd;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAC7B,cAAc,CACd;QACD,WAAW,EAAE,eAAe,CAC3B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAClC,aAAa,CACb;QACD,SAAS,EAAE,eAAe,CACzB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,aAAa,CACb;QACD,SAAS,EAAE,eAAe,CACzB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,cAAc,CACd;QACD,OAAO,EAAE,eAAe,CACvB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAC9B,IAAI,IAAI,EAAE,CACV;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,EACrC,IAAI,IAAI,EAAE,CACV;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAC7B,IAAI,IAAI,EAAE,CACV;QACD,SAAS,EAAE,eAAe,CACzB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,IAAI,IAAI,EAAE,CACV;QACD,cAAc,EAAE,eAAe,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,EACrC,IAAI,IAAI,EAAE,CACV;QACD,OAAO,EAAE,eAAe,CACvB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAC9B,IAAI,IAAI,EAAE,CACV;QACD,QAAQ,EAAE,eAAe,CACxB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC,EACvC,IAAI,IAAI,EAAE,CACV;QACD,OAAO,EAAE,eAAe,CACvB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAC9B,IAAI,IAAI,EAAE,CACV;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,EACrC,IAAI,IAAI,EAAE,CACV;QACD,cAAc,EAAE,eAAe,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI,CAAC,EAC7C,IAAI,IAAI,EAAE,CACV;KACD,CAAC,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAA2B,EAAE,IAAqB;IACxF,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAChC,MAAM,SAAS,GAAG,SAAS,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAA;IAEtE,MAAM,UAAU,GAAG,IAAI,uBAAuB,EAAE;SAC9C,cAAc,CAAC,WAAW,CAAC,MAAM,CAAC;SAClC,YAAY,CAAC,WAAW,CAAC;SACzB,kBAAkB,CAAC,cAAc,CAAC;SAClC,KAAK,EAAE,CAAA;IAET,iEAAiE;IACjE,uCAAuC;IACvC,uEAAuE;IACvE,0CAA0C;IAC1C,MAAM,gBAAgB,CAAC,YAAY,CAAC,SAAS,EAAE,UAAU,CAAC,EAAE,IAAI,CAAC,CAAA;AAClE,CAAC;AAmFD,KAAK,UAAU,kBAAkB,CAAC,SAAiB,EAAE,UAAkB;IACtE,MAAM,EAAE,GAAG,MAAM,UAAU,EAAE,CAAA;IAC7B,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,CAAA;IAEpC,MAAM,cAAc,GACnB,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,MAAM,OAAO,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;IAEjG,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE9D,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;IAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,eAAe,CAAC,SAAS,CAAC,KAAK;QAC5E,aAAa,cAAc,mCAAmC;QAC9D,OAAO,eAAe,CAAC,UAAU,CAAC,yDAAyD,cAAc,GAAG,CAC7G,CAAA;AACF,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,KAAkB,EAAE,GAAe;IAChE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAC3B,CAAC;AACF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,QAAsB,EACtB,IAA+B;IAE/B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,aAAa,CAAA;IACrD,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAA;IAC7F,MAAM,eAAe,CAAC,SAAS,CAAC,CAAA;IAEhC,MAAM,YAAY,GAAG,SAAS,CAAC,eAAe,CAAC,CAAA;IAC/C,MAAM,KAAK,GAA4B,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,wBAAwB,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAEtG;;;;;OAKG;IACH,MAAM,WAAW,GAAG,GAAoB,EAAE;QACzC,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;QACvE,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAA;YAC/B,SAAS,IAAI,IAAI,CAAC,IAAI,CAAA;QACvB,CAAC;QAED,OAAO;YACN,cAAc,EAAE,IAAI,CAAC,aAAa;YAClC,MAAM,EAAE,eAAe;YACvB,cAAc,EAAE,WAAW;YAC3B,cAAc,EAAE,cAAc;YAC9B,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC/B,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CACnF;YACD,MAAM;YACN,UAAU,EAAE,SAAS;SACrB,CAAA;IACF,CAAC,CAAA;IAED,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,CAAC,IAAI;YAAE,SAAQ;QAEnB,YAAY,OAAO,GAAG,MAAM,kBAAkB,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAA;QAE7E,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,CAAA;QAE7D,sDAAsD;QACtD,mFAAmF;QACnF,+CAA+C;QAC/C,IAAI,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;QAErE,IAAI,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAA;QAC/B,IAAI,IAAI,GAAG,EAAE,CAAA;QACb,IAAI,SAAS,GAAG,EAAE,CAAA;QAClB,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,IAAI,KAAK,GAAuB,IAAI,CAAA;QACpC,IAAI,aAAa,GAAG,EAAE,CAAA;QACtB,IAAI,YAAY,GAAG,EAAE,CAAA;QAErB,MAAM,QAAQ,GAAG,KAAK,IAAmB,EAAE;YAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAA;YACjC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;YAC/B,6CAA6C;YAC7C,IAAI,GAAG,QAAQ,CAAC,QAAQ,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAA;YAEhF,qDAAqD;YACrD,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAA;YACxF,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAA;YAC/C,QAAQ,GAAG,CAAC,CAAA;YACZ,aAAa,GAAG,EAAE,CAAA;YAClB,YAAY,GAAG,EAAE,CAAA;QAClB,CAAC,CAAA;QAED,MAAM,SAAS,GAAG,KAAK,IAAmB,EAAE;YAC3C,MAAM,WAAW,GAAG,KAAK,CAAA;YAEzB,IAAI,CAAC,WAAW;gBAAE,OAAM;YAExB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC3C,WAAW,CAAC,GAAG,CAAC,CAAC,KAAoB,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;YAC/E,CAAC,CAAC,CAAA;YAEF,IAAI,QAAQ,EAAE,CAAC;gBACd,MAAM,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;gBAEzC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;gBACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;gBAErC,KAAK,CAAC,IAAI,CAAC;oBACV,KAAK;oBACL,IAAI;oBACJ,MAAM,EAAE,SAAS;oBACjB,WAAW,EAAE,mBAAmB;oBAChC,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC;oBAC1B,MAAM;oBACN,eAAe,EAAE,aAAa;oBAC9B,cAAc,EAAE,YAAY;iBAC5B,CAAC,CAAA;gBAEF,gFAAgF;gBAChF,2DAA2D;gBAC3D,4CAA4C;gBAC5C,MAAM,kBAAkB,CAAC,WAAW,EAAE,EAAE,YAAY,CAAC,CAAA;YACtD,CAAC;YAED,KAAK,GAAG,IAAI,CAAA;YACZ,QAAQ,GAAG,CAAC,CAAA;QACb,CAAC,CAAA;QAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBAClB,QAAQ,EAAE,CAAA;gBAEV,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,MAAM,QAAQ,EAAE,CAAA;YACjB,CAAC;YAED,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAE5B,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;YACjE,MAAM,cAAc,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;YAE/B,QAAQ,EAAE,CAAA;YAEV,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACpB,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;YAC9B,CAAC;YAED,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;YAE5B,IAAI,QAAQ,IAAI,WAAW,EAAE,CAAC;gBAC7B,MAAM,SAAS,EAAE,CAAA;gBAEjB,SAAS,EAAE,CAAA;gBACX,IAAI,GAAG,EAAE,CAAA;YACV,CAAC;QACF,CAAC;QAED,MAAM,SAAS,EAAE,CAAA;IAClB,CAAC;IAED,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAA;IAE9B,MAAM,kBAAkB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;IAEhD,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,wBAAwB,CAAC,YAA6B;IACpE,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IAEhD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAkB,YAAY,CAAC,CAAA;IAEvE,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAErC,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACd,mBAAmB,YAAY,CAAC,QAAQ,EAAE,YAAY,IAAI,CAAC,IAAI,kCAAkC;gBAChG,wDAAwD,CACzD,CAAA;QACF,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACd,mBAAmB,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa;gBACzE,GAAG,YAAY,CAAC,QAAQ,EAAE,YAAY,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CACrE,CAAA;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAA;AAC5B,CAAC"}

package/out/pl/adapters/emuia/adapter.d.ts

@@ -0,0 +1,81 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `emuia`: Poland's INSPIRE Addresses (AD) theme, published by Główny Urząd Geodezji i Kartografii out
+ * of the Państwowy Rejestr Granic.
+ *
+ * Input is GML, because the service serves four GML flavors and no JSON. Its capabilities document
+ * advertises no `outputFormat` values at all and a `GetFeature` returns
+ * `application/gml+xml; version=3.2`. A page is a `wfs:FeatureCollection` of `ms:AD.Address` members,
+ * and the adapter reads it with `streamMarkupElements` from `@mailwoman/core/html/elements`, which
+ * yields one member subtree at a time, so neither a page nor a whole extract sits in memory.
+ *
+ * The payload is the Polish register's own schema rather than the harmonized INSPIRE model: a member's
+ * element names are the register's columns and every address value is inline, so there is no
+ * `ad:ThoroughfareName` to join to and no `xlink:href` to resolve. Read with that same function over
+ * 500 members across ten windows spread through the service, a member holds 21 children and never a
+ * twenty-second. The children are `gml:boundedBy`, `ms:msGeometry` and these nineteen columns: `ms:id`,
+ * `ms:gml_id`, `ms:lokalnyid`, `ms:jednostkaadmnistracyjna`, `ms:miejscowosc`,
+ * `ms:miejscowosc_lower`, `ms:czescmiejscowosci`, `ms:ulica`, `ms:ulica_pol`, `ms:ulica_skr`,
+ * `ms:numer`, `ms:kod`, `ms:teryt`, `ms:simc`, `ms:ulic`, `ms:status`, `ms:data_od`, `ms:wazny_od`
+ * and `ms:adres`. The adapter reads `ulica` as the street, `numer` as the house number, `kod` as the
+ * postcode and `miejscowosc` as the locality, with `czescmiejscowosci` as the part of the locality
+ * where the publisher populates it. Every column is written on every member, and three hold a value
+ * on every one of the 500: `numer`, `kod` and `miejscowosc`. Two are written and populated on none:
+ * `status` and `czescmiejscowosci`.
+ *
+ * A row with no street is the ordinary rural case rather than a defect. Over those 500 members 299
+ * hold a street and 201 leave `ms:ulica` written and empty, and the publisher's own `ms:adres` writes
+ * the streetless form as locality and number: `Księżyki 5`, `Jawor 19`. The adapter emits those rows
+ * with a locality and a number and no street. Poland delivers to that form.
+ *
+ * `ms:jednostkaadmnistracyjna` holds the administrative path as a PostgreSQL array literal,
+ * `{Polska,łódzkie,"Piotrków Trybunalski","Piotrków Trybunalski"}`, whose second element is the
+ * voivodeship. It is left unread: a Polish postal address does not include the voivodeship, the codex
+ * layout for `PL` leaves a `region` component unplaced, and parsing a quoted array literal to reach a
+ * value the layout leaves unplaced would cost a parse and reach no rendered component.
+ *
+ * The service states no count of its own. `resultType=hits` answers `numberMatched="1000"`, which is
+ * MapServer's default feature cap rather than a count — it stays 1000 when asked with `count=2000` and
+ * when asked at `startIndex=1` — and a feature page answers `numberMatched="unknown"`. The size is a
+ * property of the service on the day it is asked, so this file states no count. The acquisition
+ * step reads it by paging, with `countWFSFeaturesByPaging` from `@mailwoman/core/api`.
+ *
+ * GUGiK's dataset record states one condition of use, in `otherConstraints`: `Brak warunków dostępu i
+ * użytkowania`, no conditions for access and use. The address-source register elected that and names
+ * no license instrument, so no SPDX identifier exists and the publisher's own words are the label the
+ * adapter stamps on every row. GUGiK credit remains good practice and the statement requires it
+ * nowhere.
+ *
+ * The adapter honors `opts.limit` and `opts.signal`. `opts.country` is optional and accepts only `PL`.
+ */
+import { type CorpusAdapter } from "#types";
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ * The name is the municipal register the feature ids point at: a `ms:gml_id`
+ * reads `PL.ZIPIN.3742.EMUiA_9945001100000172050`.
+ */
+export declare const EMUIA_ADAPTER_ID = "emuia";
+/**
+ * The one jurisdiction this adapter emits.
+ */
+export declare const EMUIA_COUNTRIES: readonly string[];
+/**
+ * The terms the address-source register elected for this publisher.
+ *
+ * `unchecked-access-free-pl-g-wny-urz-d-geodezji-i-kartografii` is elected with no `spdx`
+ * and no `termsVersion`, so `electedTerms` is the whole of what the publisher stated
+ * and `electedLicenseLabel` returns exactly this string.
+ * Czechia's row is recorded the same way.
+ */
+export declare const EMUIA_DEFAULT_LICENSE = "Brak warunk\u00F3w dost\u0119pu i u\u017Cytkowania";
+export declare function createEMUiAAdapter(): CorpusAdapter;
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export declare const emuiaAdapter: CorpusAdapter;
+//# sourceMappingURL=adapter.d.ts.map

package/out/pl/adapters/emuia/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../lib/pl/adapters/emuia/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AASH,OAAO,EAAuD,KAAK,aAAa,EAAiB,MAAM,QAAQ,CAAA;AAE/G;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,UAAU,CAAA;AAEvC;;GAEG;AACH,eAAO,MAAM,eAAe,EAAE,SAAS,MAAM,EAAW,CAAA;AAExD;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,uDAAwC,CAAA;AAuB1E,wBAAgB,kBAAkB,IAAI,aAAa,CAwFlD;AAED;;GAEG;AACH,eAAO,MAAM,YAAY,eAAuB,CAAA"}

package/out/pl/adapters/emuia/adapter.js

@@ -0,0 +1,178 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `emuia`: Poland's INSPIRE Addresses (AD) theme, published by Główny Urząd Geodezji i Kartografii out
+ * of the Państwowy Rejestr Granic.
+ *
+ * Input is GML, because the service serves four GML flavors and no JSON. Its capabilities document
+ * advertises no `outputFormat` values at all and a `GetFeature` returns
+ * `application/gml+xml; version=3.2`. A page is a `wfs:FeatureCollection` of `ms:AD.Address` members,
+ * and the adapter reads it with `streamMarkupElements` from `@mailwoman/core/html/elements`, which
+ * yields one member subtree at a time, so neither a page nor a whole extract sits in memory.
+ *
+ * The payload is the Polish register's own schema rather than the harmonized INSPIRE model: a member's
+ * element names are the register's columns and every address value is inline, so there is no
+ * `ad:ThoroughfareName` to join to and no `xlink:href` to resolve. Read with that same function over
+ * 500 members across ten windows spread through the service, a member holds 21 children and never a
+ * twenty-second. The children are `gml:boundedBy`, `ms:msGeometry` and these nineteen columns: `ms:id`,
+ * `ms:gml_id`, `ms:lokalnyid`, `ms:jednostkaadmnistracyjna`, `ms:miejscowosc`,
+ * `ms:miejscowosc_lower`, `ms:czescmiejscowosci`, `ms:ulica`, `ms:ulica_pol`, `ms:ulica_skr`,
+ * `ms:numer`, `ms:kod`, `ms:teryt`, `ms:simc`, `ms:ulic`, `ms:status`, `ms:data_od`, `ms:wazny_od`
+ * and `ms:adres`. The adapter reads `ulica` as the street, `numer` as the house number, `kod` as the
+ * postcode and `miejscowosc` as the locality, with `czescmiejscowosci` as the part of the locality
+ * where the publisher populates it. Every column is written on every member, and three hold a value
+ * on every one of the 500: `numer`, `kod` and `miejscowosc`. Two are written and populated on none:
+ * `status` and `czescmiejscowosci`.
+ *
+ * A row with no street is the ordinary rural case rather than a defect. Over those 500 members 299
+ * hold a street and 201 leave `ms:ulica` written and empty, and the publisher's own `ms:adres` writes
+ * the streetless form as locality and number: `Księżyki 5`, `Jawor 19`. The adapter emits those rows
+ * with a locality and a number and no street. Poland delivers to that form.
+ *
+ * `ms:jednostkaadmnistracyjna` holds the administrative path as a PostgreSQL array literal,
+ * `{Polska,łódzkie,"Piotrków Trybunalski","Piotrków Trybunalski"}`, whose second element is the
+ * voivodeship. It is left unread: a Polish postal address does not include the voivodeship, the codex
+ * layout for `PL` leaves a `region` component unplaced, and parsing a quoted array literal to reach a
+ * value the layout leaves unplaced would cost a parse and reach no rendered component.
+ *
+ * The service states no count of its own. `resultType=hits` answers `numberMatched="1000"`, which is
+ * MapServer's default feature cap rather than a count — it stays 1000 when asked with `count=2000` and
+ * when asked at `startIndex=1` — and a feature page answers `numberMatched="unknown"`. The size is a
+ * property of the service on the day it is asked, so this file states no count. The acquisition
+ * step reads it by paging, with `countWFSFeaturesByPaging` from `@mailwoman/core/api`.
+ *
+ * GUGiK's dataset record states one condition of use, in `otherConstraints`: `Brak warunków dostępu i
+ * użytkowania`, no conditions for access and use. The address-source register elected that and names
+ * no license instrument, so no SPDX identifier exists and the publisher's own words are the label the
+ * adapter stamps on every row. GUGiK credit remains good practice and the statement requires it
+ * nowhere.
+ *
+ * The adapter honors `opts.limit` and `opts.signal`. `opts.country` is optional and accepts only `PL`.
+ */
+import { formatAddressRow } from "@mailwoman/codex/address/format";
+import { openReadStream } from "@mailwoman/core/fs/streams";
+import { streamMarkupElements, textAtPath } from "@mailwoman/core/html/elements";
+import { UnsupportedCountryError } from "#adapters/errors";
+import { stableSourceID } from "#adapters/source-id";
+import { SourceRegister } from "#registers";
+import { AddressRole, SurfaceOrigin } from "#types";
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ * The name is the municipal register the feature ids point at: a `ms:gml_id`
+ * reads `PL.ZIPIN.3742.EMUiA_9945001100000172050`.
+ */
+export const EMUIA_ADAPTER_ID = "emuia";
+/**
+ * The one jurisdiction this adapter emits.
+ */
+export const EMUIA_COUNTRIES = ["PL"];
+/**
+ * The terms the address-source register elected for this publisher.
+ *
+ * `unchecked-access-free-pl-g-wny-urz-d-geodezji-i-kartografii` is elected with no `spdx`
+ * and no `termsVersion`, so `electedTerms` is the whole of what the publisher stated
+ * and `electedLicenseLabel` returns exactly this string.
+ * Czechia's row is recorded the same way.
+ */
+export const EMUIA_DEFAULT_LICENSE = "Brak warunków dostępu i użytkowania";
+/**
+ * The feature element MapServer writes, spelled as the document spells it.
+ *
+ * The prefix is the service's own, `ms:`, rather than `ad:`.
+ * A reader comparing the part after the colon against `Address` finds no address
+ * type here, because the local name is `AD.Address`.
+ */
+const ADDRESS_ELEMENT = "ms:AD.Address";
+/**
+ * A member's column value, trimmed, or an empty string.
+ *
+ * `textAtPath` answers an empty string for an element the publisher wrote
+ * and left blank and `undefined` for one it omitted.
+ * Both are an absent value for this adapter's purposes, and all 500 members measured held
+ * every one of the nineteen columns, so the distinction has no effect on this service.
+ */
+function column(feature, name) {
+    return (textAtPath(feature, `ms:${name}`) ?? "").trim();
+}
+export function createEMUiAAdapter() {
+    return {
+        id: EMUIA_ADAPTER_ID,
+        defaultLicense: EMUIA_DEFAULT_LICENSE,
+        addressRole: AddressRole.Premise,
+        register: SourceRegister.PolandEMUiA,
+        surface: SurfaceOrigin.Rendered,
+        description: "Poland's INSPIRE Addresses theme (GUGiK): house-number-level addresses from the municipal EMUiA registers.",
+        async *rows(opts) {
+            if (opts.country && !EMUIA_COUNTRIES.includes(opts.country)) {
+                throw new UnsupportedCountryError(EMUIA_ADAPTER_ID, EMUIA_COUNTRIES, opts.country);
+            }
+            const stream = openReadStream(opts.inputPath);
+            let emitted = 0;
+            try {
+                for await (const feature of streamMarkupElements(stream, ADDRESS_ELEMENT, {
+                    xml: true,
+                    signal: opts.signal,
+                })) {
+                    if (opts.signal?.aborted)
+                        break;
+                    if (opts.limit !== undefined && emitted >= opts.limit)
+                        break;
+                    const street = column(feature, "ulica");
+                    const house = column(feature, "numer");
+                    const postcode = column(feature, "kod");
+                    const locality = column(feature, "miejscowosc");
+                    const partOfLocality = column(feature, "czescmiejscowosci");
+                    if (!locality)
+                        continue;
+                    if (!house && !street)
+                        continue;
+                    const components = {};
+                    if (house) {
+                        components.house_number = house;
+                    }
+                    if (street) {
+                        components.street = street;
+                    }
+                    if (partOfLocality && partOfLocality !== locality) {
+                        components.dependent_locality = partOfLocality;
+                    }
+                    if (postcode) {
+                        components.postcode = postcode;
+                    }
+                    components.locality = locality;
+                    const rendered = formatAddressRow(components, "PL", { singleLine: true });
+                    if (!rendered)
+                        continue;
+                    const { raw, components: aligned } = rendered;
+                    // Keep `ms:gml_id`: it gives the upstream municipal register and its record.
+                    // `ms:id` is this service's own row number.
+                    const seed = column(feature, "gml_id") || column(feature, "id");
+                    const sourceID = seed ? `${EMUIA_ADAPTER_ID}-${seed}` : stableSourceID(EMUIA_ADAPTER_ID, aligned);
+                    yield {
+                        raw,
+                        components: aligned,
+                        country: "PL",
+                        locale: "pl-PL",
+                        source: EMUIA_ADAPTER_ID,
+                        source_id: sourceID,
+                        corpus_version: "",
+                        license: EMUIA_DEFAULT_LICENSE,
+                    };
+                    emitted++;
+                }
+            }
+            finally {
+                stream.destroy();
+            }
+        },
+    };
+}
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const emuiaAdapter = createEMUiAAdapter();
+//# sourceMappingURL=adapter.js.map

package/out/pl/adapters/emuia/adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../lib/pl/adapters/emuia/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAA;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,EAAE,oBAAoB,EAAE,UAAU,EAAsB,MAAM,+BAA+B,CAAA;AAEpG,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,WAAW,EAA8D,aAAa,EAAE,MAAM,QAAQ,CAAA;AAE/G;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,OAAO,CAAA;AAEvC;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAsB,CAAC,IAAI,CAAC,CAAA;AAExD;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,qCAAqC,CAAA;AAE1E;;;;;;GAMG;AACH,MAAM,eAAe,GAAG,eAAe,CAAA;AAEvC;;;;;;;GAOG;AACH,SAAS,MAAM,CAAC,OAAsB,EAAE,IAAY;IACnD,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;AACxD,CAAC;AAED,MAAM,UAAU,kBAAkB;IACjC,OAAO;QACN,EAAE,EAAE,gBAAgB;QACpB,cAAc,EAAE,qBAAqB;QACrC,WAAW,EAAE,WAAW,CAAC,OAAO;QAChC,QAAQ,EAAE,cAAc,CAAC,WAAW;QACpC,OAAO,EAAE,aAAa,CAAC,QAAQ;QAC/B,WAAW,EACV,4GAA4G;QAE7G,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC7D,MAAM,IAAI,uBAAuB,CAAC,gBAAgB,EAAE,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YACnF,CAAC;YAED,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAE7C,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,IAAI,CAAC;gBACJ,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,oBAAoB,CAAC,MAAM,EAAE,eAAe,EAAE;oBACzE,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI,CAAC,MAAM;iBACnB,CAAC,EAAE,CAAC;oBACJ,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;wBAAE,MAAK;oBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;wBAAE,MAAK;oBAE5D,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;oBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;oBACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;oBACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,CAAA;oBAC/C,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAA;oBAE3D,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAEvB,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM;wBAAE,SAAQ;oBAE/B,MAAM,UAAU,GAA+B,EAAE,CAAA;oBAEjD,IAAI,KAAK,EAAE,CAAC;wBACX,UAAU,CAAC,YAAY,GAAG,KAAK,CAAA;oBAChC,CAAC;oBAED,IAAI,MAAM,EAAE,CAAC;wBACZ,UAAU,CAAC,MAAM,GAAG,MAAM,CAAA;oBAC3B,CAAC;oBAED,IAAI,cAAc,IAAI,cAAc,KAAK,QAAQ,EAAE,CAAC;wBACnD,UAAU,CAAC,kBAAkB,GAAG,cAAc,CAAA;oBAC/C,CAAC;oBAED,IAAI,QAAQ,EAAE,CAAC;wBACd,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;oBAC/B,CAAC;oBAED,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;oBAE9B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;oBAEzE,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAEvB,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAA;oBAE7C,6EAA6E;oBAC7E,4CAA4C;oBAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;oBAE/D,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,gBAAgB,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAA;oBAEjG,MAAM;wBACL,GAAG;wBACH,UAAU,EAAE,OAAO;wBACnB,OAAO,EAAE,IAAI;wBACb,MAAM,EAAE,OAAO;wBACf,MAAM,EAAE,gBAAgB;wBACxB,SAAS,EAAE,QAAQ;wBACnB,cAAc,EAAE,EAAE;wBAClB,OAAO,EAAE,qBAAqB;qBAC9B,CAAA;oBAED,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAA;YACjB,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,kBAAkB,EAAE,CAAA"}