@adonis-agora/telescope 0.21.1 → 0.23.0

package/dist/src/stores/parquet.js

@@ -0,0 +1,461 @@
+import { randomUUID } from 'node:crypto';
+import { hostname } from 'node:os';
+import { compareNewestFirst, HOUR_MS, hourStart, PARQUET_COLUMNS, parsePartition, partitionPrefix, resolvePage, toDate, toParquetRow, typePrefix, } from './parquet_format.js';
+import { importOptionalPeer, isNotFound } from './parquet_location.js';
+/**
+ * Entries as hourly-partitioned Parquet files in a {@link ColdObjectStore}:
+ *
+ * ```
+ * type=<type>/dt=<yyyy-mm-dd>/hour=<hh>/<instanceId>-<uuid>.parquet
+ * ```
+ *
+ * Files are append-only and immutable: writing adds files, {@link compact} replaces a
+ * closed hour's files with one merged file (written before the inputs are deleted),
+ * and retention deletes whole files. Rows inside a file are sorted by `created_at, id`
+ * with ~10k-row groups and statistics on, so time-bounded reads skip row groups.
+ *
+ * This is the cold half of `storage.tiered()`; it is not a {@link TelescopeStore} on its
+ * own because recording one entry at a time as Parquet would make one file per entry.
+ */
+export class ParquetColdStore {
+    location;
+    engine;
+    instanceId;
+    libs;
+    rowGroupSize;
+    maxRowsPerFile;
+    listingTtlMs;
+    metadataCacheSize;
+    compactPartitionsPerRun;
+    listings = new Map();
+    /** Small, per-file stats for every file seen (dropped when the file disappears). */
+    stats = new Map();
+    /** Parsed footers, LRU-bounded (Map iteration order is insertion order). */
+    footers = new Map();
+    /** Build a store and its engine (the engine may need async setup, e.g. DuckDB). */
+    static async create(options) {
+        const store = new ParquetColdStore(options);
+        store.engine = await options.engine({
+            location: store.location,
+            libs: store.libs,
+            metadata: (file) => store.metadata(file),
+            buffer: (file) => store.buffer(file),
+        });
+        return store;
+    }
+    constructor(options) {
+        this.location = options.location;
+        this.libs = options.libs;
+        this.instanceId = sanitizeInstanceId(options.instanceId ?? `${hostname()}-${process.pid}`);
+        this.rowGroupSize = Math.max(1, options.rowGroupSize ?? 10_000);
+        this.maxRowsPerFile = Math.max(this.rowGroupSize, options.maxRowsPerFile ?? 200_000);
+        this.listingTtlMs = Math.max(0, options.listingTtlMs ?? 2_000);
+        this.metadataCacheSize = Math.max(16, options.metadataCacheSize ?? 2_000);
+        this.compactPartitionsPerRun = Math.max(0, options.compactPartitionsPerRun ?? 4);
+    }
+    // ── Writes ────────────────────────────────────────────────────────────────
+    /**
+     * Write entries as Parquet, one or more files per `(type, hour)`. Resolves to the keys
+     * written, so a caller can delete them again if a later step of its own fails.
+     */
+    async write(entries) {
+        const groups = new Map();
+        for (const entry of entries) {
+            const hs = hourStart(entry.createdAt.getTime());
+            const groupKey = `${hs}\u0000${entry.type}`;
+            let group = groups.get(groupKey);
+            if (!group) {
+                group = { type: entry.type, hourStartMs: hs, rows: [] };
+                groups.set(groupKey, group);
+            }
+            group.rows.push(toParquetRow(entry, this.instanceId));
+        }
+        const written = [];
+        try {
+            for (const group of groups.values()) {
+                group.rows.sort(compareRowsAscending);
+                for (let i = 0; i < group.rows.length; i += this.maxRowsPerFile) {
+                    const key = `${partitionPrefix(group.type, group.hourStartMs)}${this.instanceId}-${randomUUID()}.parquet`;
+                    await this.location.write(key, this.encode(group.rows.slice(i, i + this.maxRowsPerFile)));
+                    written.push(key);
+                }
+            }
+        }
+        catch (error) {
+            await this.location.delete(written).catch(() => undefined);
+            throw error;
+        }
+        finally {
+            this.listings.clear();
+        }
+        return written;
+    }
+    encode(rows) {
+        const columnData = PARQUET_COLUMNS.map((column) => ({
+            name: column.name,
+            type: column.type,
+            nullable: column.nullable,
+            data: rows.map((row) => row[column.name]),
+            ...('bloomFilter' in column ? { bloomFilter: column.bloomFilter } : {}),
+        }));
+        const buffer = this.libs.writer.parquetWriteBuffer({
+            // hyparquet-writer types `data` as its DecodedArray union; rows are plain arrays.
+            columnData: columnData,
+            rowGroupSize: this.rowGroupSize,
+            statistics: true,
+            kvMetadata: [{ key: 'telescope.layout', value: '1' }],
+            createdBy: '@adonis-agora/telescope',
+        });
+        return new Uint8Array(buffer);
+    }
+    // ── Reads ─────────────────────────────────────────────────────────────────
+    /** Entries matching `query`, newest-first, honouring `{ page, size }`. */
+    async list(query = {}) {
+        if (query.traceIds !== undefined && query.traceIds.length === 0)
+            return [];
+        const { offset, size } = resolvePage(query);
+        const files = await this.candidates(query);
+        const rows = await this.engine.list(files, query, size === undefined ? undefined : offset + size);
+        return size === undefined ? rows : rows.slice(offset, offset + size);
+    }
+    /** The newest `limit` matches with no page offset — what the tiered merge needs. */
+    async top(query, limit) {
+        if (query.traceIds !== undefined && query.traceIds.length === 0)
+            return [];
+        return this.engine.list(await this.candidates(query), query, limit);
+    }
+    async get(id) {
+        return this.engine.get(await this.candidates({}), id);
+    }
+    async listTraceIds(query) {
+        const size = Math.max(0, Math.floor(query.size));
+        const offset = (Math.max(1, Math.floor(query.page ?? 1)) - 1) * size;
+        const rows = await this.topTraceIds(query, offset + size);
+        return rows.slice(offset, offset + size);
+    }
+    /** The `limit` most-recently-seen traces, no offset (for the tiered merge). */
+    async topTraceIds(query, limit) {
+        if (limit <= 0)
+            return [];
+        return this.engine.traceIds(await this.candidates(query), query, limit);
+    }
+    async countByBucket(query) {
+        // `after` is exclusive and `before` inclusive here; widen `before` by 1 ms so the
+        // exclusive candidate pruning keeps the file holding the boundary row.
+        const files = await this.candidates({
+            ...(query.type !== undefined ? { type: query.type } : {}),
+            after: query.after,
+            before: new Date(query.before.getTime() + 1),
+        });
+        return this.engine.countByBucket(files, query);
+    }
+    /** Rows in the cold tier, from file footers (no rows are read). */
+    async count() {
+        const files = await this.candidates({});
+        return files.reduce((sum, file) => sum + file.numRows, 0);
+    }
+    /**
+     * The newest `created_at` the cold tier could hold for `query`, or `null` when it holds
+     * nothing for it. Lets the tiered store skip the cold tier when a page is already
+     * filled by hot entries that are all newer.
+     */
+    async newestCreatedAt(query) {
+        const files = await this.candidates(query);
+        return files.length === 0 ? null : new Date(files[0].maxMs);
+    }
+    // ── Retention + compaction ───────────────────────────────────────────────
+    /**
+     * Delete every file whose whole hour is older than `olderThan`, keeping the newest
+     * files of that doomed set until they hold at least `keepLast` rows. Retention is
+     * file-granular: an hour stays until all of it has expired. Resolves to the rows
+     * deleted.
+     */
+    async expire(olderThan, keepLast) {
+        const cutoff = olderThan.getTime();
+        const all = await this.listAll(true);
+        const doomed = all
+            .filter((file) => file.hourStartMs + HOUR_MS <= cutoff)
+            .sort((a, b) => b.hourStartMs - a.hourStartMs || b.key.localeCompare(a.key));
+        if (doomed.length === 0)
+            return 0;
+        let kept = 0;
+        const toDelete = [];
+        for (const file of doomed) {
+            if (keepLast !== undefined && kept < keepLast) {
+                kept += (await this.withStats(file))?.numRows ?? 0;
+                continue;
+            }
+            toDelete.push(file);
+        }
+        let rows = 0;
+        for (const file of toDelete)
+            rows += (await this.withStats(file))?.numRows ?? 0;
+        await this.location.delete(toDelete.map((file) => file.key));
+        this.forget(toDelete.map((file) => file.key));
+        return rows;
+    }
+    /**
+     * Merge the files of closed hours (hours that ended before `closedBefore`) so each
+     * holds one file per {@link ParquetColdStoreOptions.maxRowsPerFile} rows instead of
+     * one per prune cycle. The merged file is written before its inputs are deleted, so
+     * a reader never loses rows; one that lists in between sees both and the merge on
+     * read drops the duplicate ids. Resolves to the number of input files replaced.
+     */
+    async compact(closedBefore) {
+        if (this.compactPartitionsPerRun === 0)
+            return 0;
+        const cutoff = closedBefore.getTime();
+        const byPartition = new Map();
+        for (const file of await this.listAll(true)) {
+            if (file.hourStartMs + HOUR_MS > cutoff)
+                continue;
+            const key = partitionPrefix(file.type, file.hourStartMs);
+            const group = byPartition.get(key) ?? [];
+            group.push(file);
+            byPartition.set(key, group);
+        }
+        let replaced = 0;
+        let partitions = 0;
+        for (const files of byPartition.values()) {
+            if (partitions >= this.compactPartitionsPerRun)
+                break;
+            const withStats = (await Promise.all(files.map((file) => this.withStats(file)))).filter((file) => file !== null);
+            // Only merge files that are small enough to be worth it, packed up to the file cap.
+            const small = withStats
+                .filter((file) => file.numRows < this.maxRowsPerFile)
+                .sort((a, b) => a.minMs - b.minMs);
+            const batches = [];
+            let current = [];
+            let rows = 0;
+            for (const file of small) {
+                if (current.length > 0 && rows + file.numRows > this.maxRowsPerFile) {
+                    batches.push(current);
+                    current = [];
+                    rows = 0;
+                }
+                current.push(file);
+                rows += file.numRows;
+            }
+            if (current.length > 0)
+                batches.push(current);
+            const toMerge = batches.filter((batch) => batch.length > 1);
+            if (toMerge.length === 0)
+                continue;
+            partitions++;
+            for (const batch of toMerge) {
+                const entries = await this.engine.list(batch, {}, undefined);
+                const unique = dedupeById(entries);
+                await this.write(unique);
+                await this.location.delete(batch.map((file) => file.key));
+                this.forget(batch.map((file) => file.key));
+                replaced += batch.length;
+            }
+        }
+        return replaced;
+    }
+    /** Delete every file. */
+    async clear() {
+        const all = await this.listAll(true);
+        await this.location.delete(all.map((file) => file.key));
+        this.forget(all.map((file) => file.key));
+    }
+    async close() {
+        await this.engine.close();
+    }
+    // ── Listing + footers ────────────────────────────────────────────────────
+    /**
+     * Files that can hold rows for `query`: its type's prefix only, hours outside the
+     * time bounds dropped by partition, then files outside them dropped by footer stats.
+     * Sorted newest-first by each file's newest row.
+     */
+    async candidates(query) {
+        const before = query.before?.getTime();
+        const after = query.after?.getTime();
+        const listed = await this.list_(query.type);
+        const byHour = listed.filter((file) => {
+            if (query.type !== undefined && file.type !== query.type)
+                return false;
+            if (before !== undefined && file.hourStartMs >= before)
+                return false;
+            if (after !== undefined && file.hourStartMs + HOUR_MS - 1 <= after)
+                return false;
+            return true;
+        });
+        const resolved = await Promise.all(byHour.map((file) => this.withStats(file)));
+        return resolved
+            .filter((file) => {
+            if (file === null || file.numRows === 0)
+                return false;
+            if (before !== undefined && file.minMs >= before)
+                return false;
+            if (after !== undefined && file.maxMs <= after)
+                return false;
+            return true;
+        })
+            .sort((a, b) => b.maxMs - a.maxMs || b.key.localeCompare(a.key));
+    }
+    list_(type) {
+        const prefix = type === undefined ? '' : typePrefix(type);
+        const now = Date.now();
+        const cached = this.listings.get(prefix);
+        if (cached && now - cached.at < this.listingTtlMs)
+            return cached.files;
+        const files = this.location.list(prefix).then((objects) => {
+            const out = [];
+            for (const object of objects) {
+                const partition = parsePartition(object.key);
+                if (!partition)
+                    continue;
+                out.push(this.stats.get(object.key) ?? {
+                    key: object.key,
+                    size: object.size,
+                    type: partition.type,
+                    hourStartMs: partition.hourStartMs,
+                    numRows: -1,
+                    minMs: partition.hourStartMs,
+                    maxMs: partition.hourStartMs + HOUR_MS - 1,
+                    rowGroups: [],
+                });
+            }
+            return out;
+        });
+        files.catch(() => this.listings.delete(prefix));
+        this.listings.set(prefix, { at: now, files });
+        return files;
+    }
+    async listAll(fresh) {
+        if (fresh)
+            this.listings.clear();
+        return this.list_(undefined);
+    }
+    /** The file with footer stats filled in, or `null` when it no longer exists. */
+    async withStats(file) {
+        if (file.numRows >= 0)
+            return file;
+        const known = this.stats.get(file.key);
+        if (known)
+            return known;
+        let meta;
+        try {
+            meta = await this.metadata(file);
+        }
+        catch (error) {
+            if (isNotFound(error))
+                return null;
+            throw error;
+        }
+        const full = { ...file, ...statsFromFooter(meta, file) };
+        this.stats.set(file.key, full);
+        return full;
+    }
+    /** Parsed footer for a file, through a bounded LRU. */
+    metadata(file) {
+        const hit = this.footers.get(file.key);
+        if (hit) {
+            this.footers.delete(file.key);
+            this.footers.set(file.key, hit);
+            return hit;
+        }
+        const pending = this.libs.hyparquet.parquetMetadataAsync(this.buffer(file));
+        pending.catch(() => this.footers.delete(file.key));
+        this.footers.set(file.key, pending);
+        while (this.footers.size > this.metadataCacheSize) {
+            const oldest = this.footers.keys().next().value;
+            if (oldest === undefined)
+                break;
+            this.footers.delete(oldest);
+        }
+        return pending;
+    }
+    buffer(file) {
+        const location = this.location;
+        return {
+            byteLength: file.size,
+            slice(start, end) {
+                const read = location
+                    .read(file.key, start, end)
+                    .then((bytes) => bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength));
+                // hyparquet starts sibling reads it abandons when one fails, e.g. on a file
+                // deleted mid-read. Marking each read handled keeps those from crashing the
+                // process; whoever awaits it still gets the error.
+                read.catch(() => { });
+                return read;
+            },
+        };
+    }
+    forget(keys) {
+        for (const key of keys) {
+            this.stats.delete(key);
+            this.footers.delete(key);
+        }
+        this.listings.clear();
+    }
+}
+/**
+ * Load the hyparquet trio. They are optional peers (only `storage.tiered()` needs them),
+ * so a missing one fails boot with a message naming it.
+ */
+export async function loadParquetLibs() {
+    const neededBy = 'storage.parquet()';
+    const [hyparquet, writer, compressorsModule] = await Promise.all([
+        importOptionalPeer('hyparquet', neededBy, () => import('hyparquet')),
+        importOptionalPeer('hyparquet-writer', neededBy, () => import('hyparquet-writer')),
+        importOptionalPeer('hyparquet-compressors', neededBy, () => import('hyparquet-compressors')),
+    ]);
+    return { hyparquet, writer, compressors: compressorsModule.compressors };
+}
+function statsFromFooter(meta, file) {
+    const createdAtIndex = meta.schema.slice(1).findIndex((element) => element.name === 'created_at');
+    const rowGroups = [];
+    let rowStart = 0;
+    let minMs = Number.POSITIVE_INFINITY;
+    let maxMs = Number.NEGATIVE_INFINITY;
+    for (const group of meta.row_groups) {
+        const rows = Number(group.num_rows);
+        const statistics = group.columns[createdAtIndex]?.meta_data?.statistics;
+        const lo = statValue(statistics?.min_value ?? statistics?.min, file.hourStartMs);
+        const hi = statValue(statistics?.max_value ?? statistics?.max, file.hourStartMs + HOUR_MS - 1);
+        rowGroups.push({ rowStart, rowEnd: rowStart + rows, minMs: lo, maxMs: hi });
+        minMs = Math.min(minMs, lo);
+        maxMs = Math.max(maxMs, hi);
+        rowStart += rows;
+    }
+    if (rowGroups.length === 0) {
+        minMs = file.hourStartMs;
+        maxMs = file.hourStartMs;
+    }
+    return { numRows: Number(meta.num_rows), minMs, maxMs, rowGroups };
+}
+function statValue(value, fallback) {
+    if (value === undefined || value === null)
+        return fallback;
+    const ms = toDate(value).getTime();
+    return Number.isFinite(ms) ? ms : fallback;
+}
+function compareRowsAscending(a, b) {
+    const t = a.created_at.getTime() - b.created_at.getTime();
+    if (t !== 0)
+        return t;
+    return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
+}
+/** Keep the first occurrence of each id (callers pass newest-first lists). */
+export function dedupeById(entries) {
+    const seen = new Set();
+    const out = [];
+    for (const entry of entries) {
+        if (seen.has(entry.id))
+            continue;
+        seen.add(entry.id);
+        out.push(entry);
+    }
+    return out;
+}
+/** Merge newest-first lists into one newest-first list without duplicate ids. */
+export function mergeNewestFirst(...lists) {
+    return dedupeById(lists.flat().sort(compareNewestFirst));
+}
+function sanitizeInstanceId(value) {
+    const clean = value.replace(/[^A-Za-z0-9_.-]/g, '_');
+    return clean === '' ? 'telescope' : clean;
+}
+//# sourceMappingURL=parquet.js.map

package/dist/src/stores/parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../../../src/stores/parquet.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AASnC,OAAO,EACL,kBAAkB,EAClB,OAAO,EACP,SAAS,EACT,eAAe,EAEf,cAAc,EACd,eAAe,EACf,WAAW,EACX,MAAM,EACN,YAAY,EACZ,UAAU,GACX,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAwB,kBAAkB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAiG7F;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAgB;IAClB,QAAQ,CAAkB;IACnC,MAAM,CAAkB;IACf,UAAU,CAAS;IACX,IAAI,CAAc;IAClB,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,YAAY,CAAS;IACrB,iBAAiB,CAAS;IAC1B,uBAAuB,CAAS;IAChC,QAAQ,GAAG,IAAI,GAAG,EAAsD,CAAC;IAC1F,oFAAoF;IACnE,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IACrD,4EAA4E;IAC3D,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;IAEpE,mFAAmF;IACnF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAgC;QAClD,MAAM,KAAK,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAC5C,KAAK,CAAC,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC;YAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YACxC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;SACrC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IAED,YAAoB,OAAgC;QAClD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC,OAAO,CAAC,UAAU,IAAI,GAAG,QAAQ,EAAE,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QAC3F,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,CAAC,CAAC;QAChE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,CAAC;QACrF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC,CAAC;QAC/D,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,iBAAiB,IAAI,KAAK,CAAC,CAAC;QAC1E,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,uBAAuB,IAAI,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,6EAA6E;IAE7E;;;OAGG;IACH,KAAK,CAAC,KAAK,CAAC,OAAgB;QAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqE,CAAC;QAC5F,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,EAAE,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,GAAG,EAAE,SAAS,KAAK,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,KAAK,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;gBACxD,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC9B,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;gBACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAChE,MAAM,GAAG,GAAG,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,UAAU,IAAI,UAAU,EAAE,UAAU,CAAC;oBAC1G,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;oBAC1F,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC3D,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,MAAM,CAAC,IAAkB;QAC/B,MAAM,UAAU,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAClD,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzC,GAAG,CAAC,aAAa,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE,CAAC,CAAC,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;YACjD,kFAAkF;YAClF,UAAU,EAAE,UAEM;YAClB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,CAAC,EAAE,GAAG,EAAE,kBAAkB,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACrD,SAAS,EAAE,yBAAyB;SACrC,CAAC,CAAC;QACH,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;IAED,6EAA6E;IAE7E,0EAA0E;IAC1E,KAAK,CAAC,IAAI,CAAC,KAAK,GAAe,EAAE;QAC/B,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC3E,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACjC,KAAK,EACL,KAAK,EACL,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAC/C,CAAC;QACF,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;IACvE,CAAC;IAED,oFAAoF;IACpF,KAAK,CAAC,GAAG,CAAC,KAAiB,EAAE,KAAyB;QACpD,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC3E,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACtE,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAmB;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;QACrE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;QAC1D,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;IAC3C,CAAC;IAED,+EAA+E;IAC/E,KAAK,CAAC,WAAW,CACf,KAA6C,EAC7C,KAAa;QAEb,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAuB;QACzC,kFAAkF;QAClF,uEAAuE;QACvE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC;YAClC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;SAC7C,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED,mEAAmE;IACnE,KAAK,CAAC,KAAK;QACT,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,eAAe,CAAC,KAAiB;QACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC3C,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,CAAC;IAED,4EAA4E;IAE5E;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CAAC,SAAe,EAAE,QAAiB;QAC7C,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,GAAG;aACf,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,GAAG,OAAO,IAAI,MAAM,CAAC;aACtD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAElC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,QAAQ,GAAe,EAAE,CAAC;QAChC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;gBAC9C,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC;gBACnD,SAAS;YACX,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,MAAM,IAAI,IAAI,QAAQ;YAAE,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,YAAkB;QAC9B,IAAI,IAAI,CAAC,uBAAuB,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5C,IAAI,IAAI,CAAC,WAAW,GAAG,OAAO,GAAG,MAAM;gBAAE,SAAS;YAClD,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACzD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YACzC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC;QAED,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YACzC,IAAI,UAAU,IAAI,IAAI,CAAC,uBAAuB;gBAAE,MAAM;YACtD,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CACrF,CAAC,IAAI,EAAoB,EAAE,CAAC,IAAI,KAAK,IAAI,CAC1C,CAAC;YACF,oFAAoF;YACpF,MAAM,KAAK,GAAG,SAAS;iBACpB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC;iBACpD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,OAAO,GAAiB,EAAE,CAAC;YACjC,IAAI,OAAO,GAAe,EAAE,CAAC;YAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACtB,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,GAAG,CAAC,CAAC;gBACX,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnB,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC;YACvB,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC5D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YACnC,UAAU,EAAE,CAAC;YACb,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC;gBAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;gBACnC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACzB,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC1D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC3C,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,yBAAyB;IACzB,KAAK,CAAC,KAAK;QACT,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,4EAA4E;IAE5E;;;;OAIG;IACK,KAAK,CAAC,UAAU,CACtB,KAAoD;QAEpD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YACpC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI;gBAAE,OAAO,KAAK,CAAC;YACvE,IAAI,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,IAAI,MAAM;gBAAE,OAAO,KAAK,CAAC;YACrE,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,GAAG,OAAO,GAAG,CAAC,IAAI,KAAK;gBAAE,OAAO,KAAK,CAAC;YACjF,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/E,OAAO,QAAQ;aACZ,MAAM,CAAC,CAAC,IAAI,EAAoB,EAAE;YACjC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YACtD,IAAI,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,MAAM;gBAAE,OAAO,KAAK,CAAC;YAC/D,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,KAAK;gBAAE,OAAO,KAAK,CAAC;YAC7D,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;aACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACrE,CAAC;IAEO,KAAK,CAAC,IAAwB;QACpC,MAAM,MAAM,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC,YAAY;YAAE,OAAO,MAAM,CAAC,KAAK,CAAC;QACvE,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;YACxD,MAAM,GAAG,GAAe,EAAE,CAAC;YAC3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7C,IAAI,CAAC,SAAS;oBAAE,SAAS;gBACzB,GAAG,CAAC,IAAI,CACN,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI;oBAC5B,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,OAAO,EAAE,CAAC,CAAC;oBACX,KAAK,EAAE,SAAS,CAAC,WAAW;oBAC5B,KAAK,EAAE,SAAS,CAAC,WAAW,GAAG,OAAO,GAAG,CAAC;oBAC1C,SAAS,EAAE,EAAE;iBACd,CACF,CAAC;YACJ,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAc;QAClC,IAAI,KAAK;YAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC;IAED,gFAAgF;IACxE,KAAK,CAAC,SAAS,CAAC,IAAc;QACpC,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;QACxB,IAAI,IAAkB,CAAC;QACvB,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YACnC,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,IAAI,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;QACzD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uDAAuD;IACvD,QAAQ,CAAC,IAAc;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChC,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5E,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAChD,IAAI,MAAM,KAAK,SAAS;gBAAE,MAAM;YAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,CAAC,IAAc;QACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,IAAI;YACrB,KAAK,CAAC,KAAa,EAAE,GAAY;gBAC/B,MAAM,IAAI,GAAG,QAAQ;qBAClB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC;qBAC1B,IAAI,CACH,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,MAAM,CAAC,KAAK,CAChB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CACrB,CACnB,CAAC;gBACJ,4EAA4E;gBAC5E,4EAA4E;gBAC5E,mDAAmD;gBACnD,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACrB,OAAO,IAAI,CAAC;YACd,CAAC;SACF,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,IAAc;QAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,MAAM,QAAQ,GAAG,mBAAmB,CAAC;IACrC,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,iBAAiB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC/D,kBAAkB,CAAC,WAAW,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACpE,kBAAkB,CAAC,kBAAkB,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;QAClF,kBAAkB,CAAC,uBAAuB,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC;KAC7F,CAAC,CAAC;IACH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,CAAC,WAAW,EAAE,CAAC;AAC3E,CAAC;AAED,SAAS,eAAe,CACtB,IAAkB,EAClB,IAAc;IAEd,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;IAClG,MAAM,SAAS,GAAoB,EAAE,CAAC;IACtC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,KAAK,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACrC,IAAI,KAAK,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACxE,MAAM,EAAE,GAAG,SAAS,CAAC,UAAU,EAAE,SAAS,IAAI,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACjF,MAAM,EAAE,GAAG,SAAS,CAAC,UAAU,EAAE,SAAS,IAAI,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC;QAC/F,SAAS,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,GAAG,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5E,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,QAAQ,IAAI,IAAI,CAAC;IACnB,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC;QACzB,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC;IAC3B,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,SAAS,CAAC,KAAc,EAAE,QAAgB;IACjD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,QAAQ,CAAC;IAC3D,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACnC,OAAO,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC7C,CAAC;AAED,SAAS,oBAAoB,CAAC,CAAa,EAAE,CAAa;IACxD,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;IAC1D,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,UAAU,CAAC,OAAgB;IACzC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAY,EAAE,CAAC;IACxB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,SAAS;QACjC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,gBAAgB,CAAC,GAAG,KAAgB;IAClD,OAAO,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa;IACvC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACrD,OAAO,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC;AAC5C,CAAC"}

package/dist/src/stores/parquet_duckdb.d.ts

@@ -0,0 +1,52 @@
+import type { Entry } from '../entry.js';
+import type { BucketCountQuery, BucketCountRow, EntryQuery, TraceIdRow } from '../store.js';
+import type { ColdEngineContext, ColdFile, ColdReadEngine } from './parquet.js';
+/** Options for the DuckDB cold engine (`engine: { duckdb: { ... } }`). */
+export interface DuckDbEngineOptions {
+    /** DuckDB `memory_limit`, e.g. `'512MB'`. Default: DuckDB's own (80% of RAM). */
+    memoryLimit?: string;
+    /** DuckDB worker threads. Default 2, so a wide scan cannot take every core. */
+    threads?: number;
+    /**
+     * Where remote (S3) files are copied before DuckDB reads them, since DuckDB only reads
+     * them by path here. Default `<os tmpdir>/adonis-telescope-parquet`.
+     */
+    cacheDir?: string;
+    /** Bytes of copied files kept in {@link cacheDir}. Default 1 GiB. */
+    cacheBytes?: number;
+}
+/**
+ * The optional DuckDB cold engine (`@duckdb/node-api`, an optional peer).
+ *
+ * It only reads files by local path. A directory location's files are read in place;
+ * remote files are fetched through the location (the host's own S3 client) into a
+ * bounded local cache first. The instance is locked down at start: extension
+ * auto-install/auto-load off, external access off except the data and cache
+ * directories, and the configuration locked, so DuckDB can never download or load
+ * `httpfs` / `aws` (they are not bundled; DuckDB would fetch them at runtime).
+ */
+export declare class DuckDbEngine implements ColdReadEngine {
+    private readonly ctx;
+    private readonly instance;
+    readonly name: 'duckdb';
+    private readonly cache;
+    private constructor();
+    /** Build the engine. Fails with a clear error when `@duckdb/node-api` is missing. */
+    static create(ctx: ColdEngineContext, options?: DuckDbEngineOptions): Promise<DuckDbEngine>;
+    list(files: ColdFile[], query: EntryQuery, limit: number | undefined): Promise<Entry[]>;
+    get(files: ColdFile[], id: string): Promise<Entry | null>;
+    traceIds(files: ColdFile[], query: {
+        before?: Date;
+        after?: Date;
+    }, limit: number): Promise<TraceIdRow[]>;
+    countByBucket(files: ColdFile[], query: BucketCountQuery): Promise<BucketCountRow[]>;
+    close(): Promise<void>;
+    /**
+     * Run one statement over `files`. A file deleted between listing and reading
+     * (retention, compaction) makes DuckDB fail the whole scan, so on an IO error the
+     * statement is retried once over the files that still exist.
+     */
+    private query;
+    private paths;
+}
+//# sourceMappingURL=parquet_duckdb.d.ts.map

package/dist/src/stores/parquet_duckdb.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet_duckdb.d.ts","sourceRoot":"","sources":["../../../src/stores/parquet_duckdb.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5F,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAOhF,0EAA0E;AAC1E,MAAM,WAAW,mBAAmB;IAClC,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;GASG;AACH,qBAAa,YAAa,YAAW,cAAc;IAK/C,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAL3B,QAAQ,CAAC,IAAI,EAAG,QAAQ,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAwB;IAE9C,OAAO,eAMN;IAED,qFAAqF;IACrF,OAAa,MAAM,CACjB,GAAG,EAAE,iBAAiB,EACtB,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,YAAY,CAAC,CAuCvB;IAEK,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CA4B5F;IAEK,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAQ9D;IAEK,QAAQ,CACZ,KAAK,EAAE,QAAQ,EAAE,EACjB,KAAK,EAAE;QAAE,MAAM,CAAC,EAAE,IAAI,CAAC;QAAC,KAAK,CAAC,EAAE,IAAI,CAAA;KAAE,EACtC,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,UAAU,EAAE,CAAC,CAkBvB;IAEK,aAAa,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAmBzF;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAE3B;IAED;;;;OAIG;YACW,KAAK;YAsBL,KAAK;CAYpB"}