@adonis-agora/telescope 0.21.1 → 0.23.0

package/dist/src/stores/parquet_hyparquet.js

@@ -0,0 +1,212 @@
+import { compareNewestFirst, fromParquetRow, toDate } from './parquet_format.js';
+import { isNotFound } from './parquet_location.js';
+/**
+ * The default cold engine: pure JavaScript (hyparquet), nothing native and nothing
+ * fetched at runtime.
+ *
+ * Reads walk files newest-first and, inside each file, row groups newest-first,
+ * skipping any group whose `created_at` statistics fall outside the query, and stop
+ * as soon as the page is full and nothing older can still outrank it. Equality
+ * filters on `id` / `trace_id` also consult the bloom filters the writer emits.
+ */
+export class HyparquetEngine {
+    ctx;
+    name = 'hyparquet';
+    constructor(ctx) {
+        this.ctx = ctx;
+    }
+    async list(files, query, limit) {
+        let results = [];
+        const filter = pushdownFilter(query);
+        // Bloom filters exist for `trace_id` (and `id`); they pay off on point lookups only.
+        const useBloom = query.traceId !== undefined || query.traceIds !== undefined;
+        const before = query.before?.getTime();
+        const after = query.after?.getTime();
+        const full = () => limit !== undefined && results.length >= limit;
+        const floor = () => results[results.length - 1].createdAt.getTime();
+        for (const file of files) {
+            if (full() && file.maxMs < floor())
+                break;
+            for (const group of newestGroups(file, before, after)) {
+                if (full() && group.maxMs < floor())
+                    break;
+                const rows = await this.readGroup(file, group, undefined, filter, useBloom);
+                for (const row of rows) {
+                    if (!rowMatches(row, query))
+                        continue;
+                    results.push(fromParquetRow(row));
+                }
+                results.sort(compareNewestFirst);
+                if (limit !== undefined && results.length > limit)
+                    results = results.slice(0, limit);
+            }
+        }
+        return results;
+    }
+    async get(files, id) {
+        const filter = { id: { $eq: id } };
+        for (const file of files) {
+            for (const group of newestGroups(file, undefined, undefined)) {
+                const rows = await this.readGroup(file, group, undefined, filter, true);
+                const hit = rows.find((row) => row.id === id);
+                if (hit)
+                    return fromParquetRow(hit);
+            }
+        }
+        return null;
+    }
+    async traceIds(files, query, limit) {
+        const lastAt = new Map();
+        const before = query.before?.getTime();
+        const after = query.after?.getTime();
+        // Once `limit` traces are newer than everything left to read, nothing older can enter.
+        const kth = () => {
+            if (lastAt.size < limit)
+                return Number.NEGATIVE_INFINITY;
+            return [...lastAt.values()].sort((a, b) => b - a)[limit - 1];
+        };
+        for (const file of files) {
+            if (file.maxMs < kth())
+                break;
+            for (const group of newestGroups(file, before, after)) {
+                if (group.maxMs < kth())
+                    break;
+                const rows = await this.readGroup(file, group, ['trace_id', 'created_at']);
+                for (const row of rows) {
+                    if (row.trace_id == null)
+                        continue;
+                    const at = toDate(row.created_at).getTime();
+                    if (before !== undefined && !(at < before))
+                        continue;
+                    if (after !== undefined && !(at > after))
+                        continue;
+                    const traceId = String(row.trace_id);
+                    const seen = lastAt.get(traceId);
+                    if (seen === undefined || at > seen)
+                        lastAt.set(traceId, at);
+                }
+            }
+        }
+        return [...lastAt.entries()]
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, limit)
+            .map(([traceId, at]) => ({ traceId, lastAt: new Date(at) }));
+    }
+    async countByBucket(files, query) {
+        const startMs = query.after.getTime();
+        const endMs = query.before.getTime();
+        const bucketMs = Math.max(1, Math.floor(query.bucketMs));
+        const tally = new Map();
+        for (const file of files) {
+            for (const group of newestGroups(file, endMs + 1, startMs)) {
+                const rows = await this.readGroup(file, group, ['type', 'created_at']);
+                for (const row of rows) {
+                    const type = String(row.type);
+                    if (query.type !== undefined && type !== query.type)
+                        continue;
+                    const at = toDate(row.created_at).getTime();
+                    if (at <= startMs || at > endMs)
+                        continue;
+                    const index = Math.floor((at - startMs) / bucketMs);
+                    const key = `${index}\u0000${type}`;
+                    const seen = tally.get(key);
+                    if (seen === undefined)
+                        tally.set(key, { index, type, count: 1 });
+                    else
+                        seen.count += 1;
+                }
+            }
+        }
+        return [...tally.values()];
+    }
+    async close() { }
+    /** One row group as plain objects; a file that vanished since listing reads as empty. */
+    async readGroup(file, group, columns, filter, useBloomFilters = false) {
+        const { hyparquet, compressors } = this.ctx.libs;
+        try {
+            const metadata = await this.ctx.metadata(file);
+            return (await hyparquet.parquetReadObjects({
+                file: hyparquet.cachedAsyncBuffer(this.ctx.buffer(file)),
+                metadata,
+                compressors,
+                rowStart: group.rowStart,
+                rowEnd: group.rowEnd,
+                ...(columns !== undefined ? { columns } : {}),
+                ...(filter !== undefined ? { filter, useBloomFilters } : {}),
+            }));
+        }
+        catch (error) {
+            if (isNotFound(error))
+                return [];
+            throw error;
+        }
+    }
+}
+/** A file's row groups that overlap `(after, before)`, newest first. */
+function newestGroups(file, before, after) {
+    return file.rowGroups
+        .filter((group) => {
+        if (before !== undefined && group.minMs >= before)
+            return false;
+        if (after !== undefined && group.maxMs <= after)
+            return false;
+        return true;
+    })
+        .sort((a, b) => b.maxMs - a.maxMs);
+}
+/**
+ * The equality predicates hyparquet can evaluate itself (and use to skip row groups).
+ * Everything else is checked in {@link rowMatches}.
+ */
+function pushdownFilter(query) {
+    const clauses = [];
+    if (query.type !== undefined)
+        clauses.push({ type: { $eq: query.type } });
+    if (query.familyHash !== undefined)
+        clauses.push({ family_hash: { $eq: query.familyHash } });
+    if (query.traceId !== undefined)
+        clauses.push({ trace_id: { $eq: query.traceId } });
+    if (query.traceIds !== undefined)
+        clauses.push({ trace_id: { $in: query.traceIds } });
+    if (clauses.length === 0)
+        return undefined;
+    return clauses.length === 1 ? clauses[0] : { $and: clauses };
+}
+/**
+ * Every {@link EntryQuery} predicate, evaluated on the stored row before it is parsed.
+ * `tag` matches the quoted token inside the JSON text and `search` is a lower-cased
+ * substring over `content` and `tags` — the same semantics as the Lucid store.
+ */
+export function rowMatches(row, query) {
+    if (query.type !== undefined && row.type !== query.type)
+        return false;
+    if (query.familyHash !== undefined && row.family_hash !== query.familyHash)
+        return false;
+    if (query.traceId !== undefined && row.trace_id !== query.traceId)
+        return false;
+    if (query.traceIds !== undefined) {
+        if (typeof row.trace_id !== 'string' || !query.traceIds.includes(row.trace_id))
+            return false;
+    }
+    if (query.before !== undefined || query.after !== undefined) {
+        const at = toDate(row.created_at).getTime();
+        if (query.before !== undefined && !(at < query.before.getTime()))
+            return false;
+        if (query.after !== undefined && !(at > query.after.getTime()))
+            return false;
+    }
+    const tags = String(row.tags ?? '');
+    if (query.tag !== undefined && !tags.includes(JSON.stringify(query.tag)))
+        return false;
+    if (query.search !== undefined) {
+        const needle = query.search.toLowerCase();
+        if (!String(row.content ?? '')
+            .toLowerCase()
+            .includes(needle) &&
+            !tags.toLowerCase().includes(needle)) {
+            return false;
+        }
+    }
+    return true;
+}
+//# sourceMappingURL=parquet_hyparquet.js.map

package/dist/src/stores/parquet_hyparquet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet_hyparquet.js","sourceRoot":"","sources":["../../../src/stores/parquet_hyparquet.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AACjF,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAInD;;;;;;;;GAQG;AACH,MAAM,OAAO,eAAe;IAGG,GAAG;IAFvB,IAAI,GAAG,WAAoB,CAAC;IAErC,YAA6B,GAAsB;mBAAtB,GAAG;IAAsB,CAAC;IAEvD,KAAK,CAAC,IAAI,CAAC,KAAiB,EAAE,KAAiB,EAAE,KAAyB;QACxE,IAAI,OAAO,GAAY,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACrC,qFAAqF;QACrF,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC;QAC7E,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;QAClE,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAErE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,GAAG,KAAK,EAAE;gBAAE,MAAM;YAC1C,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;gBACtD,IAAI,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,GAAG,KAAK,EAAE;oBAAE,MAAM;gBAC3C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAC5E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC;wBAAE,SAAS;oBACtC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBACjC,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,GAAG,KAAK;oBAAE,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAiB,EAAE,EAAU;QACrC,MAAM,MAAM,GAAuB,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;QACvD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC;gBAC7D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;gBACxE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC9C,IAAI,GAAG;oBAAE,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,KAAiB,EACjB,KAAsC,EACtC,KAAa;QAEb,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;QACrC,uFAAuF;QACvF,MAAM,GAAG,GAAG,GAAW,EAAE;YACvB,IAAI,MAAM,CAAC,IAAI,GAAG,KAAK;gBAAE,OAAO,MAAM,CAAC,iBAAiB,CAAC;YACzD,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC;QAChE,CAAC,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,KAAK,GAAG,GAAG,EAAE;gBAAE,MAAM;YAC9B,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;gBACtD,IAAI,KAAK,CAAC,KAAK,GAAG,GAAG,EAAE;oBAAE,MAAM;gBAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC,CAAC;gBAC3E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,IAAI,GAAG,CAAC,QAAQ,IAAI,IAAI;wBAAE,SAAS;oBACnC,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC;oBAC5C,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC;wBAAE,SAAS;oBACrD,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC;wBAAE,SAAS;oBACnD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBACrC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACjC,IAAI,IAAI,KAAK,SAAS,IAAI,EAAE,GAAG,IAAI;wBAAE,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC/D,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;aACzB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAiB,EAAE,KAAuB;QAC5D,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC;gBAC3D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;gBACvE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC9B,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,KAAK,CAAC,IAAI;wBAAE,SAAS;oBAC9D,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC;oBAC5C,IAAI,EAAE,IAAI,OAAO,IAAI,EAAE,GAAG,KAAK;wBAAE,SAAS;oBAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC;oBACpD,MAAM,GAAG,GAAG,GAAG,KAAK,SAAS,IAAI,EAAE,CAAC;oBACpC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5B,IAAI,IAAI,KAAK,SAAS;wBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;;wBAC7D,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,KAAK,KAAmB,CAAC;IAE/B,yFAAyF;IACjF,KAAK,CAAC,SAAS,CACrB,IAAc,EACd,KAAoB,EACpB,OAA6B,EAC7B,MAA2B,EAC3B,eAAe,GAAG,KAAK;QAEvB,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;QACjD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC/C,OAAO,CAAC,MAAM,SAAS,CAAC,kBAAkB,CAAC;gBACzC,IAAI,EAAE,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACxD,QAAQ;gBACR,WAAW;gBACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7C,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7D,CAAC,CAA8B,CAAC;QACnC,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACjC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF;AAED,wEAAwE;AACxE,SAAS,YAAY,CACnB,IAAc,EACd,MAA0B,EAC1B,KAAyB;IAEzB,OAAO,IAAI,CAAC,SAAS;SAClB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAChB,IAAI,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,IAAI,MAAM;YAAE,OAAO,KAAK,CAAC;QAChE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;QAC9D,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,KAAiB;IACvC,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC7F,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACpF,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAC/D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,GAA4B,EAAE,KAAiB;IACxE,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACtE,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,CAAC,WAAW,KAAK,KAAK,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC;IACzF,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAChF,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,OAAO,KAAK,CAAC;IAC/F,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC5D,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/E,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;IAC/E,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACpC,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IACvF,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC1C,IACE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;aACvB,WAAW,EAAE;aACb,QAAQ,CAAC,MAAM,CAAC;YACnB,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EACpC,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/stores/parquet_location.d.ts

@@ -0,0 +1,88 @@
+/** One immutable object (a Parquet file) in a cold location. */
+export interface ColdObject {
+    /** Location-relative key, always `/`-separated. */
+    key: string;
+    /** Size in bytes. */
+    size: number;
+}
+/**
+ * The byte-level surface the Parquet cold store needs from where its files live.
+ * Two ship ({@link DirectoryLocation}, {@link S3Location}); anything else (GCS,
+ * Azure, a test double) implements these five calls.
+ *
+ * Files are written once and never modified, so implementations need no locking:
+ * a key either exists with its final bytes or does not exist.
+ */
+export interface ColdObjectStore {
+    /** Every object whose key starts with `prefix` (location-relative). */
+    list(prefix: string): Promise<ColdObject[]>;
+    /** Bytes `[start, end)` of an object. `end` omitted means "to the end". */
+    read(key: string, start: number, end?: number): Promise<Uint8Array>;
+    /** Write a whole object. Must be atomic: readers never see a partial file. */
+    write(key: string, bytes: Uint8Array): Promise<void>;
+    /** Delete objects. Missing keys are not an error. */
+    delete(keys: string[]): Promise<void>;
+    /**
+     * The object's path on the local filesystem, when it has one. Engines that read
+     * files by path (DuckDB) use it directly; for remote locations they copy the
+     * object into a local cache first.
+     */
+    localPath?(key: string): string;
+    /** The directory every {@link localPath} lives under, when the location is local. */
+    readonly localRoot?: string;
+}
+/** A directory on the local filesystem (or a mounted volume). */
+export declare class DirectoryLocation implements ColdObjectStore {
+    private readonly root;
+    readonly localRoot: string;
+    constructor(root: string);
+    list(prefix: string): Promise<ColdObject[]>;
+    private walk;
+    read(key: string, start: number, end?: number): Promise<Uint8Array>;
+    write(key: string, bytes: Uint8Array): Promise<void>;
+    delete(keys: string[]): Promise<void>;
+    localPath(key: string): string;
+    /** Whole-file read, used by tests and tooling. */
+    readAll(key: string): Promise<Uint8Array>;
+}
+/**
+ * The slice of `@aws-sdk/client-s3`'s `S3Client` the S3 location uses. The host
+ * passes its own configured client, so credentials, region, endpoint (GovCloud, a
+ * VPC endpoint, MinIO) and retry policy are the host's.
+ */
+export interface S3ClientLike {
+    send(command: unknown): Promise<unknown>;
+}
+/** Options for {@link S3Location}. */
+export interface S3LocationOptions {
+    /** The host's `S3Client`. */
+    client: S3ClientLike;
+    bucket: string;
+    /** Key prefix every file goes under, e.g. `telescope/v1`. A trailing `/` is added. */
+    prefix?: string;
+}
+/**
+ * An S3 bucket (or any S3-compatible store) through the host's own client. Reads are
+ * ranged `GetObject` calls, so a footer or a row group is fetched without the file.
+ */
+export declare class S3Location implements ColdObjectStore {
+    private readonly options;
+    private readonly prefix;
+    private commands;
+    constructor(options: S3LocationOptions);
+    private sdk;
+    list(prefix: string): Promise<ColdObject[]>;
+    read(key: string, start: number, end?: number): Promise<Uint8Array>;
+    write(key: string, bytes: Uint8Array): Promise<void>;
+    delete(keys: string[]): Promise<void>;
+}
+/**
+ * Import an optional peer dependency, turning "module not found" into an error
+ * that names the package and the config that needs it. Anything else (a broken
+ * native binding, a syntax error) is rethrown as-is.
+ */
+export declare function importOptionalPeer<T>(name: string, neededBy: string, importer: () => Promise<T>): Promise<T>;
+export declare function isNotFound(error: unknown): boolean;
+/** Make sure a directory exists (used for the DuckDB local file cache). */
+export declare function ensureDir(path: string): Promise<void>;
+//# sourceMappingURL=parquet_location.d.ts.map

package/dist/src/stores/parquet_location.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet_location.d.ts","sourceRoot":"","sources":["../../../src/stores/parquet_location.ts"],"names":[],"mappings":"AAGA,gEAAgE;AAChE,MAAM,WAAW,UAAU;IACzB,mDAAmD;IACnD,GAAG,EAAE,MAAM,CAAC;IACZ,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,eAAe;IAC9B,uEAAuE;IACvE,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAC5C,2EAA2E;IAC3E,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpE,8EAA8E;IAC9E,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrD,qDAAqD;IACrD,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC;;;;OAIG;IACH,SAAS,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;IAChC,qFAAqF;IACrF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,iEAAiE;AACjE,qBAAa,iBAAkB,YAAW,eAAe;IAG3C,OAAO,CAAC,QAAQ,CAAC,IAAI;IAFjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B,YAA6B,IAAI,EAAE,MAAM,EAExC;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAOhD;YAEa,IAAI;IA0BZ,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAiBxE;IAEK,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAQzD;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAI1C;IAED,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAE7B;IAED,kDAAkD;IAClD,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAExC;CACF;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC1C;AAED,sCAAsC;AACtC,MAAM,WAAW,iBAAiB;IAChC,6BAA6B;IAC7B,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,sFAAsF;IACtF,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAID;;;GAGG;AACH,qBAAa,UAAW,YAAW,eAAe;IAIpC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAHpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAA2B;IAE3C,YAA6B,OAAO,EAAE,iBAAiB,EAGtD;YAEa,GAAG;IAWX,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAuBhD;IAEK,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CASxE;IAEK,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAUzD;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAW1C;CACF;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,CAAC,EACxC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACzB,OAAO,CAAC,CAAC,CAAC,CAYZ;AASD,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAIlD;AAED,2EAA2E;AAC3E,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3D"}

package/dist/src/stores/parquet_location.js

@@ -0,0 +1,194 @@
+import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
+import { dirname, join, relative, sep } from 'node:path';
+/** A directory on the local filesystem (or a mounted volume). */
+export class DirectoryLocation {
+    root;
+    localRoot;
+    constructor(root) {
+        this.root = root;
+        this.localRoot = root;
+    }
+    async list(prefix) {
+        const out = [];
+        // Walk only the deepest existing directory the prefix names, not the whole root.
+        const slash = prefix.lastIndexOf('/');
+        const start = slash >= 0 ? prefix.slice(0, slash) : '';
+        await this.walk(join(this.root, start), prefix, out);
+        return out;
+    }
+    async walk(dir, prefix, out) {
+        let names;
+        try {
+            names = await readdir(dir, { withFileTypes: true });
+        }
+        catch (error) {
+            if (isNotFound(error))
+                return;
+            throw error;
+        }
+        for (const dirent of names) {
+            const full = join(dir, dirent.name);
+            if (dirent.isDirectory()) {
+                await this.walk(full, prefix, out);
+                continue;
+            }
+            if (!dirent.isFile() || !dirent.name.endsWith('.parquet'))
+                continue;
+            const key = relative(this.root, full).split(sep).join('/');
+            if (!key.startsWith(prefix))
+                continue;
+            try {
+                out.push({ key, size: (await stat(full)).size });
+            }
+            catch (error) {
+                // Deleted between readdir and stat (retention or compaction on another tick).
+                if (!isNotFound(error))
+                    throw error;
+            }
+        }
+    }
+    async read(key, start, end) {
+        const { open } = await import('node:fs/promises');
+        const handle = await open(this.localPath(key), 'r');
+        try {
+            const last = end ?? (await handle.stat()).size;
+            const length = Math.max(0, last - start);
+            const buffer = new Uint8Array(length);
+            let offset = 0;
+            while (offset < length) {
+                const { bytesRead } = await handle.read(buffer, offset, length - offset, start + offset);
+                if (bytesRead === 0)
+                    break;
+                offset += bytesRead;
+            }
+            return offset === length ? buffer : buffer.subarray(0, offset);
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    async write(key, bytes) {
+        const target = this.localPath(key);
+        await mkdir(dirname(target), { recursive: true });
+        // Write-then-rename so a concurrent reader's directory walk never sees a partial file
+        // (the temp name does not end in `.parquet`, so `list` skips it).
+        const temp = `${target}.${process.pid}.tmp`;
+        await writeFile(temp, bytes);
+        await rename(temp, target);
+    }
+    async delete(keys) {
+        for (const key of keys) {
+            await rm(this.localPath(key), { force: true });
+        }
+    }
+    localPath(key) {
+        return join(this.root, ...key.split('/'));
+    }
+    /** Whole-file read, used by tests and tooling. */
+    readAll(key) {
+        return readFile(this.localPath(key));
+    }
+}
+/**
+ * An S3 bucket (or any S3-compatible store) through the host's own client. Reads are
+ * ranged `GetObject` calls, so a footer or a row group is fetched without the file.
+ */
+export class S3Location {
+    options;
+    prefix;
+    commands = null;
+    constructor(options) {
+        this.options = options;
+        const raw = options.prefix ?? '';
+        this.prefix = raw === '' || raw.endsWith('/') ? raw : `${raw}/`;
+    }
+    async sdk() {
+        if (this.commands === null) {
+            this.commands = await importOptionalPeer('@aws-sdk/client-s3', 'the S3 location of storage.parquet()', () => import('@aws-sdk/client-s3'));
+        }
+        return this.commands;
+    }
+    async list(prefix) {
+        const { ListObjectsV2Command } = await this.sdk();
+        const out = [];
+        let token;
+        do {
+            const page = (await this.options.client.send(new ListObjectsV2Command({
+                Bucket: this.options.bucket,
+                Prefix: this.prefix + prefix,
+                ...(token !== undefined ? { ContinuationToken: token } : {}),
+            })));
+            for (const object of page.Contents ?? []) {
+                if (object.Key === undefined || !object.Key.endsWith('.parquet'))
+                    continue;
+                out.push({ key: object.Key.slice(this.prefix.length), size: object.Size ?? 0 });
+            }
+            token = page.IsTruncated ? page.NextContinuationToken : undefined;
+        } while (token !== undefined);
+        return out;
+    }
+    async read(key, start, end) {
+        const { GetObjectCommand } = await this.sdk();
+        if (end !== undefined && end <= start)
+            return new Uint8Array(0);
+        const range = end === undefined ? `bytes=${start}-` : `bytes=${start}-${end - 1}`;
+        const response = (await this.options.client.send(new GetObjectCommand({ Bucket: this.options.bucket, Key: this.prefix + key, Range: range })));
+        if (!response.Body)
+            return new Uint8Array(0);
+        return response.Body.transformToByteArray();
+    }
+    async write(key, bytes) {
+        const { PutObjectCommand } = await this.sdk();
+        await this.options.client.send(new PutObjectCommand({
+            Bucket: this.options.bucket,
+            Key: this.prefix + key,
+            Body: bytes,
+            ContentType: 'application/vnd.apache.parquet',
+        }));
+    }
+    async delete(keys) {
+        if (keys.length === 0)
+            return;
+        const { DeleteObjectsCommand } = await this.sdk();
+        for (let i = 0; i < keys.length; i += 1000) {
+            await this.options.client.send(new DeleteObjectsCommand({
+                Bucket: this.options.bucket,
+                Delete: { Objects: keys.slice(i, i + 1000).map((key) => ({ Key: this.prefix + key })) },
+            }));
+        }
+    }
+}
+/**
+ * Import an optional peer dependency, turning "module not found" into an error
+ * that names the package and the config that needs it. Anything else (a broken
+ * native binding, a syntax error) is rethrown as-is.
+ */
+export async function importOptionalPeer(name, neededBy, importer) {
+    try {
+        return await importer();
+    }
+    catch (error) {
+        if (isModuleNotFound(error, name)) {
+            throw new Error(`@adonis-agora/telescope: ${neededBy} needs the optional peer dependency "${name}", which is not installed. Install it (e.g. \`npm install ${name}\`) or choose a different option.`, { cause: error });
+        }
+        throw error;
+    }
+}
+function isModuleNotFound(error, name) {
+    if (typeof error !== 'object' || error === null)
+        return false;
+    const code = error.code;
+    const message = String(error.message ?? '');
+    return (code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND') && message.includes(name);
+}
+export function isNotFound(error) {
+    if (typeof error !== 'object' || error === null)
+        return false;
+    const { code, name } = error;
+    return code === 'ENOENT' || name === 'NoSuchKey' || name === 'NotFound';
+}
+/** Make sure a directory exists (used for the DuckDB local file cache). */
+export async function ensureDir(path) {
+    await mkdir(path, { recursive: true });
+}
+//# sourceMappingURL=parquet_location.js.map

package/dist/src/stores/parquet_location.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet_location.js","sourceRoot":"","sources":["../../../src/stores/parquet_location.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACzF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAqCzD,iEAAiE;AACjE,MAAM,OAAO,iBAAiB;IAGC,IAAI;IAFxB,SAAS,CAAS;IAE3B,YAA6B,IAAY;oBAAZ,IAAI;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,GAAG,GAAiB,EAAE,CAAC;QAC7B,iFAAiF;QACjF,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QACrD,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,MAAc,EAAE,GAAiB;QAC/D,IAAI,KAAiC,CAAC;QACtC,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO;YAC9B,MAAM,KAAK,CAAC;QACd,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;gBACnC,SAAS;YACX,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,SAAS;YACpE,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3D,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC;gBAAE,SAAS;YACtC,IAAI,CAAC;gBACH,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,8EAA8E;gBAC9E,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;oBAAE,MAAM,KAAK,CAAC;YACtC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,KAAa,EAAE,GAAY;QACjD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC;YACzC,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;YACtC,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,OAAO,MAAM,GAAG,MAAM,EAAE,CAAC;gBACvB,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC;gBACzF,IAAI,SAAS,KAAK,CAAC;oBAAE,MAAM;gBAC3B,MAAM,IAAI,SAAS,CAAC;YACtB,CAAC;YACD,OAAO,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACjE,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,KAAiB;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,sFAAsF;QACtF,kEAAkE;QAClE,MAAM,IAAI,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;QAC5C,MAAM,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC7B,MAAM,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAc;QACzB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,kDAAkD;IAClD,OAAO,CAAC,GAAW;QACjB,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACvC,CAAC;CACF;AAsBD;;;GAGG;AACH,MAAM,OAAO,UAAU;IAIQ,OAAO;IAHnB,MAAM,CAAS;IACxB,QAAQ,GAAsB,IAAI,CAAC;IAE3C,YAA6B,OAA0B;uBAA1B,OAAO;QAClC,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,GAAG,KAAK,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,GAAG;QACf,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,QAAQ,GAAG,MAAM,kBAAkB,CACtC,oBAAoB,EACpB,sCAAsC,EACtC,GAAG,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CACnC,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;QAClD,MAAM,GAAG,GAAiB,EAAE,CAAC;QAC7B,IAAI,KAAyB,CAAC;QAC9B,GAAG,CAAC;YACF,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAC1C,IAAI,oBAAoB,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC3B,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,MAAM;gBAC5B,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7D,CAAC,CACH,CAIA,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;gBACzC,IAAI,MAAM,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC;oBAAE,SAAS;gBAC3E,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;YAClF,CAAC;YACD,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC;QACpE,CAAC,QAAQ,KAAK,KAAK,SAAS,EAAE;QAC9B,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,KAAa,EAAE,GAAY;QACjD,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9C,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,IAAI,KAAK;YAAE,OAAO,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;QAChE,MAAM,KAAK,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;QAClF,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAC9C,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAC5F,CAA+D,CAAC;QACjE,IAAI,CAAC,QAAQ,CAAC,IAAI;YAAE,OAAO,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,QAAQ,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,KAAiB;QACxC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9C,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAC5B,IAAI,gBAAgB,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;YAC3B,GAAG,EAAE,IAAI,CAAC,MAAM,GAAG,GAAG;YACtB,IAAI,EAAE,KAAK;YACX,WAAW,EAAE,gCAAgC;SAC9C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAc;QACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC9B,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;QAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;YAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAC5B,IAAI,oBAAoB,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC3B,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE;aACxF,CAAC,CACH,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,QAAgB,EAChB,QAA0B;IAE1B,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,IAAI,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CACb,4BAA4B,QAAQ,wCAAwC,IAAI,6DAA6D,IAAI,mCAAmC,EACpL,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,IAAY;IACpD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,IAAI,GAAI,KAA4B,CAAC,IAAI,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,CAAE,KAA+B,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,IAAI,KAAK,sBAAsB,IAAI,IAAI,KAAK,kBAAkB,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACpG,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAc;IACvC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAA2C,CAAC;IACnE,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,UAAU,CAAC;AAC1E,CAAC;AAED,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACzC,CAAC"}

package/dist/src/stores/tiered.d.ts

@@ -0,0 +1,91 @@
+import type { Entry, RecordInput } from '../entry.js';
+import type { BucketCountQuery, BucketCountRow, EntryQuery, TelescopeStore, TraceIdQuery, TraceIdRow } from '../store.js';
+import { type ParquetColdStore } from './parquet.js';
+/** Where the tiered store reports cold-tier trouble. Defaults to `console`. */
+export interface TieredStoreLogger {
+    warn(message: string): void;
+}
+/** Options for {@link TieredTelescopeStore}. */
+export interface TieredStoreOptions {
+    /** The live store every entry is recorded into, e.g. the app's database via Lucid. */
+    hot: TelescopeStore;
+    /** Where entries go once they are older than {@link hotForMs}. */
+    cold: ParquetColdStore;
+    /** How long an entry stays in the hot store, in ms. */
+    hotForMs: number;
+    /** Hot entries read per page while moving them to cold. Default 5,000. */
+    moveBatchSize?: number;
+    /** Entries buffered before a move writes them out as files. Default 50,000. */
+    moveFlushSize?: number;
+    /** Merge each closed hour's small files during prune. Default `true`. */
+    compact?: boolean;
+    logger?: TieredStoreLogger;
+}
+/** What the most recent {@link TieredTelescopeStore.prune} did. */
+export interface TieredPruneReport {
+    /** Hot entries written to the cold tier. */
+    moved: number;
+    /** Entries deleted from the hot tier (moved ones included). */
+    hotDeleted: number;
+    /** Cold rows deleted by retention. */
+    coldExpired: number;
+    /** Cold files replaced by compaction. */
+    compactedFiles: number;
+    /** A cold-tier failure during the cycle, when there was one. */
+    error?: string;
+}
+/**
+ * A two-tier {@link TelescopeStore}: a hot store kept minutes deep and a Parquet cold
+ * tier for everything older.
+ *
+ * - **Writes** only reach the hot store.
+ * - **`prune(olderThan, keepLast)`** first moves hot entries older than
+ *   `now - hotFor` (and newer than `olderThan`) to cold, deleting them from hot only
+ *   after the files are written; then applies the retention cutoff to the cold tier
+ *   (whole hourly files); then compacts closed hours. A failed write deletes the files
+ *   it wrote and leaves the hot store untouched, so the next cycle retries.
+ * - **Reads** go to hot first and to cold for whatever the answer still needs, merged
+ *   newest-first without duplicate ids, so `{ page, size }` pages straight across the
+ *   tier boundary. If the cold tier cannot be read, hot results are returned and a
+ *   warning is logged; the dashboard never fails because of the cold tier.
+ */
+export declare class TieredTelescopeStore implements TelescopeStore {
+    readonly hot: TelescopeStore;
+    readonly cold: ParquetColdStore;
+    private readonly hotForMs;
+    private readonly moveBatchSize;
+    private readonly moveFlushSize;
+    private readonly compactEnabled;
+    private readonly logger;
+    private lastWarnAt;
+    private report;
+    constructor(options: TieredStoreOptions);
+    record<TContent>(input: RecordInput<TContent>): Promise<Entry<TContent>>;
+    get(id: string): Promise<Entry | null>;
+    list(query?: EntryQuery): Promise<Entry[]>;
+    count(): Promise<number>;
+    /**
+     * The union's top traces: each tier's top `offset + size` by last-seen, merged by the
+     * newer of the two timestamps. A trace in the union's top N is necessarily in the top
+     * N of the tier where it was last seen, so nothing is missed.
+     */
+    listTraceIds(query: TraceIdQuery): Promise<TraceIdRow[]>;
+    countByBucket(query: BucketCountQuery): Promise<BucketCountRow[]>;
+    prune(olderThan: Date, keepLast?: number): Promise<number>;
+    /** What the last prune cycle moved, deleted and compacted (null before the first). */
+    lastPrune(): TieredPruneReport | null;
+    clear(): Promise<void>;
+    /** Release the cold engine (DuckDB holds a native instance). */
+    close(): Promise<void>;
+    /**
+     * Copy hot entries in `[after, before)` to cold. Pages are offset-based over a fixed
+     * window that nothing is deleted from until the whole move succeeded, so no entry is
+     * skipped; entries reach the window only by ageing past `before`, after this cycle's
+     * snapshot. Resolves to the number of entries written.
+     */
+    private moveToCold;
+    private bucketsFromHotList;
+    /** Run a cold read; on failure log (at most once a minute) and use `fallback`. */
+    private coldRead;
+}
+//# sourceMappingURL=tiered.d.ts.map

package/dist/src/stores/tiered.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tiered.d.ts","sourceRoot":"","sources":["../../../src/stores/tiered.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,KAAK,EACV,gBAAgB,EAChB,cAAc,EACd,UAAU,EACV,cAAc,EACd,YAAY,EACZ,UAAU,EACX,MAAM,aAAa,CAAC;AACrB,OAAO,EAAoB,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGvE,+EAA+E;AAC/E,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IACjC,sFAAsF;IACtF,GAAG,EAAE,cAAc,CAAC;IACpB,kEAAkE;IAClE,IAAI,EAAE,gBAAgB,CAAC;IACvB,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,0EAA0E;IAC1E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,+EAA+E;IAC/E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,yEAAyE;IACzE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,iBAAiB,CAAC;CAC5B;AAED,mEAAmE;AACnE,MAAM,WAAW,iBAAiB;IAChC,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,+DAA+D;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,oBAAqB,YAAW,cAAc;IACzD,QAAQ,CAAC,GAAG,EAAE,cAAc,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,MAAM,CAAkC;IAEhD,YAAY,OAAO,EAAE,kBAAkB,EAWtC;IAED,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAEvE;IAEK,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAI3C;IAEK,IAAI,CAAC,KAAK,GAAE,UAAe,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CA0BnD;IAEK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAM7B;IAED;;;;OAIG;IACG,YAAY,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAyB7D;IAEK,aAAa,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAatE;IAEK,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA8B/D;IAED,sFAAsF;IACtF,SAAS,IAAI,iBAAiB,GAAG,IAAI,CAEpC;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAE3B;IAED,gEAAgE;IAC1D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAE3B;IAED;;;;;OAKG;YACW,UAAU;YA0BV,kBAAkB;IAmBhC,kFAAkF;YACpE,QAAQ;CAcvB"}