@adonis-agora/telescope 0.22.0 → 0.24.0

package/dist/src/stores/parquet_duckdb.js

@@ -0,0 +1,266 @@
+import { createHash } from 'node:crypto';
+import { access, rename, rm, stat, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join, resolve, sep } from 'node:path';
+import { fromParquetRow, toNumber } from './parquet_format.js';
+import { ensureDir, importOptionalPeer, isNotFound } from './parquet_location.js';
+/**
+ * The optional DuckDB cold engine (`@duckdb/node-api`, an optional peer).
+ *
+ * It only reads files by local path. A directory location's files are read in place;
+ * remote files are fetched through the location (the host's own S3 client) into a
+ * bounded local cache first. The instance is locked down at start: extension
+ * auto-install/auto-load off, external access off except the data and cache
+ * directories, and the configuration locked, so DuckDB can never download or load
+ * `httpfs` / `aws` (they are not bundled; DuckDB would fetch them at runtime).
+ */
+export class DuckDbEngine {
+    ctx;
+    instance;
+    name = 'duckdb';
+    cache;
+    constructor(ctx, instance, cache) {
+        this.ctx = ctx;
+        this.instance = instance;
+        this.cache = cache;
+    }
+    /** Build the engine. Fails with a clear error when `@duckdb/node-api` is missing. */
+    static async create(ctx, options = {}) {
+        const duckdb = await loadDuckDb();
+        const localRoot = ctx.location.localRoot;
+        const cache = localRoot === undefined
+            ? new LocalFileCache(ctx, resolve(options.cacheDir ?? join(tmpdir(), 'adonis-telescope-parquet')), options.cacheBytes ?? 1024 ** 3)
+            : null;
+        if (cache)
+            await cache.init();
+        const dataDir = localRoot ?? cache.dir;
+        const instance = await duckdb.DuckDBInstance.create(':memory:', {
+            autoinstall_known_extensions: 'false',
+            autoload_known_extensions: 'false',
+            allow_community_extensions: 'false',
+            threads: String(Math.max(1, Math.floor(options.threads ?? 2))),
+            ...(options.memoryLimit !== undefined ? { memory_limit: options.memoryLimit } : {}),
+        });
+        const connection = await instance.connect();
+        try {
+            // Checked before the lockdown: listing extensions reads DuckDB's extension directory.
+            const loaded = await connection.runAndReadAll("SELECT extension_name FROM duckdb_extensions() WHERE loaded AND extension_name IN ('httpfs', 'aws')");
+            if (loaded.getRowObjectsJson().length > 0) {
+                throw new Error('@adonis-agora/telescope: DuckDB started with httpfs/aws loaded; refusing to use it.');
+            }
+            await connection.run(`SET allowed_directories = [${literal(withSlash(resolve(dataDir)))}]`);
+            await connection.run('SET enable_external_access = false');
+            await connection.run('SET lock_configuration = true');
+        }
+        finally {
+            connection.closeSync();
+        }
+        return new DuckDbEngine(ctx, instance, cache);
+    }
+    async list(files, query, limit) {
+        if (files.length === 0)
+            return [];
+        const where = new Where();
+        if (query.type !== undefined)
+            where.add('type = ?', query.type);
+        if (query.familyHash !== undefined)
+            where.add('family_hash = ?', query.familyHash);
+        if (query.traceId !== undefined)
+            where.add('trace_id = ?', query.traceId);
+        if (query.traceIds !== undefined) {
+            if (query.traceIds.length === 0)
+                return [];
+            where.add(`trace_id IN (${query.traceIds.map(() => '?').join(', ')})`, ...query.traceIds);
+        }
+        if (query.before !== undefined)
+            where.add('created_at < epoch_ms(?::BIGINT)', query.before.getTime());
+        if (query.after !== undefined)
+            where.add('created_at > epoch_ms(?::BIGINT)', query.after.getTime());
+        if (query.tag !== undefined)
+            where.add('contains(tags, ?)', JSON.stringify(query.tag));
+        if (query.search !== undefined) {
+            const needle = query.search.toLowerCase();
+            where.add('(contains(lower(content), ?) OR contains(lower(tags), ?))', needle, needle);
+        }
+        const rows = await this.query(files, (source) => `SELECT ${ENTRY_COLUMNS} FROM ${source}${where.sql()} ORDER BY created_at_ms DESC, sequence DESC, id DESC${limit !== undefined ? ` LIMIT ${Math.max(0, Math.floor(limit))}` : ''}`, where.values);
+        return rows.map(toEntry);
+    }
+    async get(files, id) {
+        if (files.length === 0)
+            return null;
+        const rows = await this.query(files, (source) => `SELECT ${ENTRY_COLUMNS} FROM ${source} WHERE id = ? LIMIT 1`, [id]);
+        return rows.length === 0 ? null : toEntry(rows[0]);
+    }
+    async traceIds(files, query, limit) {
+        if (files.length === 0)
+            return [];
+        const where = new Where();
+        where.add('trace_id IS NOT NULL');
+        if (query.before !== undefined)
+            where.add('created_at < epoch_ms(?::BIGINT)', query.before.getTime());
+        if (query.after !== undefined)
+            where.add('created_at > epoch_ms(?::BIGINT)', query.after.getTime());
+        const rows = await this.query(files, (source) => `SELECT trace_id, max(epoch_ms(created_at)) AS last_at FROM ${source}${where.sql()} GROUP BY trace_id ORDER BY last_at DESC LIMIT ${Math.max(0, Math.floor(limit))}`, where.values);
+        return rows.map((row) => ({
+            traceId: String(row.trace_id),
+            lastAt: new Date(toNumber(row.last_at)),
+        }));
+    }
+    async countByBucket(files, query) {
+        if (files.length === 0)
+            return [];
+        const startMs = query.after.getTime();
+        const bucketMs = Math.max(1, Math.floor(query.bucketMs));
+        const where = new Where();
+        where.add('created_at > epoch_ms(?::BIGINT)', startMs);
+        where.add('created_at <= epoch_ms(?::BIGINT)', query.before.getTime());
+        if (query.type !== undefined)
+            where.add('type = ?', query.type);
+        const rows = await this.query(files, (source) => `SELECT CAST(floor((epoch_ms(created_at) - ${startMs}) / ${bucketMs}) AS BIGINT) AS idx, type, count(*) AS n FROM ${source}${where.sql()} GROUP BY 1, 2`, where.values);
+        return rows.map((row) => ({
+            index: toNumber(row.idx),
+            type: String(row.type),
+            count: toNumber(row.n),
+        }));
+    }
+    async close() {
+        this.instance.closeSync();
+    }
+    /**
+     * Run one statement over `files`. A file deleted between listing and reading
+     * (retention, compaction) makes DuckDB fail the whole scan, so on an IO error the
+     * statement is retried once over the files that still exist.
+     */
+    async query(files, sql, values) {
+        let paths = await this.paths(files);
+        for (let attempt = 0;; attempt++) {
+            if (paths.length === 0)
+                return [];
+            const connection = await this.instance.connect();
+            try {
+                const reader = await connection.runAndReadAll(sql(source(paths)), values);
+                return reader.getRowObjectsJS();
+            }
+            catch (error) {
+                if (attempt > 0 || !/IO Error|No files found/i.test(String(error?.message)))
+                    throw error;
+                paths = await existing(paths);
+            }
+            finally {
+                connection.closeSync();
+            }
+        }
+    }
+    async paths(files) {
+        if (this.cache === null) {
+            const local = this.ctx.location.localPath;
+            return files.map((file) => local.call(this.ctx.location, file.key));
+        }
+        const out = [];
+        for (const file of files) {
+            const path = await this.cache.ensure(file);
+            if (path !== null)
+                out.push(path);
+        }
+        return out;
+    }
+}
+const ENTRY_COLUMNS = 'id, type, family_hash, content, tags, sequence, duration_ms, origin, trace_id, epoch_ms(created_at) AS created_at_ms';
+function toEntry(row) {
+    return fromParquetRow({ ...row, created_at: toNumber(row.created_at_ms) });
+}
+function source(paths) {
+    return `read_parquet([${paths.map(literal).join(', ')}])`;
+}
+class Where {
+    clauses = [];
+    values = [];
+    add(clause, ...values) {
+        this.clauses.push(clause);
+        this.values.push(...values);
+    }
+    sql() {
+        return this.clauses.length === 0 ? '' : ` WHERE ${this.clauses.join(' AND ')}`;
+    }
+}
+function literal(value) {
+    return `'${value.replace(/'/g, "''")}'`;
+}
+function withSlash(path) {
+    return path.endsWith(sep) ? path : `${path}${sep}`;
+}
+async function existing(paths) {
+    const out = [];
+    for (const path of paths) {
+        try {
+            await access(path);
+            out.push(path);
+        }
+        catch {
+            // gone
+        }
+    }
+    return out;
+}
+async function loadDuckDb() {
+    return importOptionalPeer('@duckdb/node-api', "storage.parquet({ engine: 'duckdb' })", () => import('@duckdb/node-api'));
+}
+/**
+ * Remote files copied to local disk for DuckDB, keyed by a hash of their (immutable)
+ * key and evicted least-recently-used past a byte budget. Copies go through the
+ * location, i.e. the host's own S3 client.
+ */
+class LocalFileCache {
+    ctx;
+    dir;
+    budget;
+    entries = new Map();
+    bytes = 0;
+    constructor(ctx, dir, budget) {
+        this.ctx = ctx;
+        this.dir = dir;
+        this.budget = budget;
+    }
+    async init() {
+        await ensureDir(this.dir);
+    }
+    async ensure(file) {
+        const hit = this.entries.get(file.key);
+        if (hit) {
+            this.entries.delete(file.key);
+            this.entries.set(file.key, hit);
+            return hit.path;
+        }
+        const path = join(this.dir, `${createHash('sha256').update(file.key).digest('hex')}.parquet`);
+        try {
+            await stat(path);
+        }
+        catch {
+            let bytes;
+            try {
+                bytes = await this.ctx.location.read(file.key, 0);
+            }
+            catch (error) {
+                if (isNotFound(error))
+                    return null;
+                throw error;
+            }
+            const temp = `${path}.${process.pid}.tmp`;
+            await writeFile(temp, bytes);
+            await rename(temp, path);
+        }
+        this.entries.set(file.key, { path, size: file.size });
+        this.bytes += file.size;
+        await this.evict();
+        return path;
+    }
+    async evict() {
+        while (this.bytes > this.budget && this.entries.size > 1) {
+            const [key, oldest] = this.entries.entries().next().value;
+            this.entries.delete(key);
+            this.bytes -= oldest.size;
+            await rm(oldest.path, { force: true });
+        }
+    }
+}
+//# sourceMappingURL=parquet_duckdb.js.map

package/dist/src/stores/parquet_duckdb.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet_duckdb.js","sourceRoot":"","sources":["../../../src/stores/parquet_duckdb.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAI/C,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAoBlF;;;;;;;;;GASG;AACH,MAAM,OAAO,YAAY;IAKJ,GAAG;IACH,QAAQ;IALlB,IAAI,GAAG,QAAiB,CAAC;IACjB,KAAK,CAAwB;IAE9C,YACmB,GAAsB,EACtB,QAAwB,EACzC,KAA4B;mBAFX,GAAG;wBACH,QAAQ;QAGzB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,qFAAqF;IACrF,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,GAAsB,EACtB,OAAO,GAAwB,EAAE;QAEjC,MAAM,MAAM,GAAG,MAAM,UAAU,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;QACzC,MAAM,KAAK,GACT,SAAS,KAAK,SAAS;YACrB,CAAC,CAAC,IAAI,cAAc,CAChB,GAAG,EACH,OAAO,CAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,0BAA0B,CAAC,CAAC,EACvE,OAAO,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,CAChC;YACH,CAAC,CAAC,IAAI,CAAC;QACX,IAAI,KAAK;YAAE,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,SAAS,IAAI,KAAM,CAAC,GAAG,CAAC;QAExC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,UAAU,EAAE;YAC9D,4BAA4B,EAAE,OAAO;YACrC,yBAAyB,EAAE,OAAO;YAClC,0BAA0B,EAAE,OAAO;YACnC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;YAC9D,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpF,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,sFAAsF;YACtF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,aAAa,CAC3C,qGAAqG,CACtG,CAAC;YACF,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,MAAM,IAAI,KAAK,CACb,qFAAqF,CACtF,CAAC;YACJ,CAAC;YACD,MAAM,UAAU,CAAC,GAAG,CAAC,8BAA8B,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC5F,MAAM,UAAU,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;YAC3D,MAAM,UAAU,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QACxD,CAAC;gBAAS,CAAC;YACT,UAAU,CAAC,SAAS,EAAE,CAAC;QACzB,CAAC;QACD,OAAO,IAAI,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAiB,EAAE,KAAiB,EAAE,KAAyB;QACxE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS;YAAE,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS;YAAE,KAAK,CAAC,GAAG,CAAC,iBAAiB,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QACnF,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAAE,KAAK,CAAC,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1E,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YAC3C,KAAK,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5F,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS;YAC5B,KAAK,CAAC,GAAG,CAAC,kCAAkC,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;YAC3B,KAAK,CAAC,GAAG,CAAC,kCAAkC,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS;YAAE,KAAK,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvF,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC1C,KAAK,CAAC,GAAG,CAAC,2DAA2D,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACzF,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAC3B,KAAK,EACL,CAAC,MAAM,EAAE,EAAE,CACT,UAAU,aAAa,SAAS,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,uDAClD,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EACrE,EAAE,EACJ,KAAK,CAAC,MAAM,CACb,CAAC;QACF,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAiB,EAAE,EAAU;QACrC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACpC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAC3B,KAAK,EACL,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,aAAa,SAAS,MAAM,uBAAuB,EACzE,CAAC,EAAE,CAAC,CACL,CAAC;QACF,OAAO,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,KAAiB,EACjB,KAAsC,EACtC,KAAa;QAEb,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QAC1B,KAAK,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QAClC,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS;YAC5B,KAAK,CAAC,GAAG,CAAC,kCAAkC,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;YAC3B,KAAK,CAAC,GAAG,CAAC,kCAAkC,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAC3B,KAAK,EACL,CAAC,MAAM,EAAE,EAAE,CACT,8DAA8D,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,kDAAkD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EACtK,KAAK,CAAC,MAAM,CACb,CAAC;QACF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC7B,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;SACxC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAiB,EAAE,KAAuB;QAC5D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QAC1B,KAAK,CAAC,GAAG,CAAC,kCAAkC,EAAE,OAAO,CAAC,CAAC;QACvD,KAAK,CAAC,GAAG,CAAC,mCAAmC,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS;YAAE,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAC3B,KAAK,EACL,CAAC,MAAM,EAAE,EAAE,CACT,6CAA6C,OAAO,OAAO,QAAQ,iDAAiD,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,gBAAgB,EAC1J,KAAK,CAAC,MAAM,CACb,CAAC;QACF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;YACxB,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;YACtB,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;SACvB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,KAAK,CACjB,KAAiB,EACjB,GAA+B,EAC/B,MAA8B;QAE9B,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACpC,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;YAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YAClC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gBAC1E,OAAO,MAAM,CAAC,eAAe,EAA+B,CAAC;YAC/D,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAE,KAAe,EAAE,OAAO,CAAC,CAAC;oBACpF,MAAM,KAAK,CAAC;gBACd,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;oBAAS,CAAC;gBACT,UAAU,CAAC,SAAS,EAAE,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,KAAiB;QACnC,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAU,CAAC;YAC3C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,IAAI,IAAI,KAAK,IAAI;gBAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAED,MAAM,aAAa,GACjB,sHAAsH,CAAC;AAEzH,SAAS,OAAO,CAAC,GAA4B;IAC3C,OAAO,cAAc,CAAC,EAAE,GAAG,GAAG,EAAE,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,MAAM,CAAC,KAAe;IAC7B,OAAO,iBAAiB,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAC5D,CAAC;AAED,MAAM,KAAK;IACQ,OAAO,GAAa,EAAE,CAAC;IAC/B,MAAM,GAA2B,EAAE,CAAC;IAE7C,GAAG,CAAC,MAAc,EAAE,GAAG,MAA8B;QACnD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,GAAG;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACjF,CAAC;CACF;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;AAC1C,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,GAAG,EAAE,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,KAAe;IACrC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,OAAO,kBAAkB,CACvB,kBAAkB,EAClB,uCAAuC,EACvC,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,CACjC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,cAAc;IAKC,GAAG;IACX,GAAG;IACK,MAAM;IANR,OAAO,GAAG,IAAI,GAAG,EAA0C,CAAC;IACrE,KAAK,GAAG,CAAC,CAAC;IAElB,YACmB,GAAsB,EAC9B,GAAW,EACH,MAAc;mBAFd,GAAG;mBACX,GAAG;sBACK,MAAM;IACtB,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAc;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChC,OAAO,GAAG,CAAC,IAAI,CAAC;QAClB,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC9F,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,KAAiB,CAAC;YACtB,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,IAAI,UAAU,CAAC,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC;gBACnC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,IAAI,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;YAC1C,MAAM,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC7B,MAAM,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,OAAO,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzD,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,KAGnD,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;CACF"}

package/dist/src/stores/parquet_format.d.ts

@@ -0,0 +1,108 @@
+import { type Entry } from '../entry.js';
+import type { EntryQuery } from '../store.js';
+/** One hour, in ms. Partitions are hourly. */
+export declare const HOUR_MS = 3600000;
+/**
+ * The on-disk row. `content` and `tags` are JSON text, exactly as the Lucid store keeps
+ * them, so a tag match is the same quoted-token substring test in every engine and no
+ * engine needs nested-type support. `created_at` is a millisecond TIMESTAMP so both the
+ * row-group statistics and DuckDB see a real time column.
+ */
+export interface ParquetRow {
+    id: string;
+    type: string;
+    family_hash: string | null;
+    content: string;
+    tags: string;
+    sequence: bigint;
+    duration_ms: bigint | null;
+    origin: string;
+    trace_id: string | null;
+    instance_id: string;
+    created_at: Date;
+}
+/** Column order and types, shared by the writer and the DuckDB projection. */
+export declare const PARQUET_COLUMNS: readonly [{
+    readonly name: 'id';
+    readonly type: 'STRING';
+    readonly nullable: false;
+    readonly bloomFilter: true;
+}, {
+    readonly name: 'type';
+    readonly type: 'STRING';
+    readonly nullable: false;
+}, {
+    readonly name: 'family_hash';
+    readonly type: 'STRING';
+    readonly nullable: true;
+}, {
+    readonly name: 'content';
+    readonly type: 'STRING';
+    readonly nullable: false;
+}, {
+    readonly name: 'tags';
+    readonly type: 'STRING';
+    readonly nullable: false;
+}, {
+    readonly name: 'sequence';
+    readonly type: 'INT64';
+    readonly nullable: false;
+}, {
+    readonly name: 'duration_ms';
+    readonly type: 'INT64';
+    readonly nullable: true;
+}, {
+    readonly name: 'origin';
+    readonly type: 'STRING';
+    readonly nullable: false;
+}, {
+    readonly name: 'trace_id';
+    readonly type: 'STRING';
+    readonly nullable: true;
+    readonly bloomFilter: true;
+}, {
+    readonly name: 'instance_id';
+    readonly type: 'STRING';
+    readonly nullable: false;
+}, {
+    readonly name: 'created_at';
+    readonly type: 'TIMESTAMP';
+    readonly nullable: false;
+}];
+export declare function toParquetRow(entry: Entry, instanceId: string): ParquetRow;
+/** Rebuild an {@link Entry} from a row as either engine returns it. */
+export declare function fromParquetRow(row: Record<string, unknown>): Entry;
+/** A file's partition, parsed from its key. */
+export interface PartitionKey {
+    type: string;
+    /** Start of the file's UTC hour, epoch ms. */
+    hourStartMs: number;
+}
+/**
+ * `type=<type>/dt=<yyyy-mm-dd>/hour=<hh>/` — type first, so a query on one type (and a
+ * bucket lifecycle rule on one type) only touches that type's prefix.
+ */
+export declare function partitionPrefix(type: string, hourStartMs: number): string;
+export declare function typePrefix(type: string): string;
+/** Parse a location-relative key, or `null` for anything that is not ours. */
+export declare function parsePartition(key: string): PartitionKey | null;
+export declare function hourStart(ms: number): number;
+/**
+ * Newest-first total order used by every tier: `createdAt` desc, then `sequence` desc
+ * (the Lucid store's tiebreaker), then `id` desc so the order is total across pods.
+ */
+export declare function compareNewestFirst(a: Entry, b: Entry): number;
+/** The `{ page, size }` of a query, resolved: how many to skip and how many to keep. */
+export declare function resolvePage(query: EntryQuery): {
+    offset: number;
+    size: number | undefined;
+};
+/**
+ * The predicates of an {@link EntryQuery} that need the row itself (everything except
+ * the time bounds and type, which partitions and statistics already prune). Applied
+ * in JS by the hyparquet engine and as a final guard on merged results.
+ */
+export declare function matchesQuery(entry: Entry, query: EntryQuery): boolean;
+export declare function toNumber(value: unknown): number;
+export declare function toDate(value: unknown): Date;
+//# sourceMappingURL=parquet_format.d.ts.map

package/dist/src/stores/parquet_format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet_format.d.ts","sourceRoot":"","sources":["../../../src/stores/parquet_format.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,KAAK,EAAiB,MAAM,aAAa,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,8CAA8C;AAC9C,eAAO,MAAM,OAAO,UAAY,CAAC;AAEjC;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,IAAI,CAAC;CAClB;AAED,8EAA8E;AAC9E,eAAO,MAAM,eAAe;aACxB,IAAI,EAAE,IAAI;aAAE,IAAI,EAAE,QAAQ;aAAE,QAAQ;aAAS,WAAW;;aACxD,IAAI,EAAE,MAAM;aAAE,IAAI,EAAE,QAAQ;aAAE,QAAQ;;aACtC,IAAI,EAAE,aAAa;aAAE,IAAI,EAAE,QAAQ;aAAE,QAAQ;;aAC7C,IAAI,EAAE,SAAS;aAAE,IAAI,EAAE,QAAQ;aAAE,QAAQ;;aACzC,IAAI,EAAE,MAAM;aAAE,IAAI,EAAE,QAAQ;aAAE,QAAQ;;aACtC,IAAI,EAAE,UAAU;aAAE,IAAI,EAAE,OAAO;aAAE,QAAQ;;aACzC,IAAI,EAAE,aAAa;aAAE,IAAI,EAAE,OAAO;aAAE,QAAQ;;aAC5C,IAAI,EAAE,QAAQ;aAAE,IAAI,EAAE,QAAQ;aAAE,QAAQ;;aACxC,IAAI,EAAE,UAAU;aAAE,IAAI,EAAE,QAAQ;aAAE,QAAQ;aAAQ,WAAW;;aAC7D,IAAI,EAAE,aAAa;aAAE,IAAI,EAAE,QAAQ;aAAE,QAAQ;;aAC7C,IAAI,EAAE,YAAY;aAAE,IAAI,EAAE,WAAW;aAAE,QAAQ;EACzC,CAAC;AAEX,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,GAAG,UAAU,CAczE;AAED,uEAAuE;AACvE,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,KAAK,CAclE;AAED,+CAA+C;AAC/C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAIzE;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE/C;AAID,8EAA8E;AAC9E,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAc/D;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAE5C;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,GAAG,MAAM,CAK7D;AAED,wFAAwF;AACxF,wBAAgB,WAAW,CAAC,KAAK,EAAE,UAAU,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAK3F;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAiBrE;AA2BD,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAK/C;AAED,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAK3C"}

package/dist/src/stores/parquet_format.js

@@ -0,0 +1,175 @@
+import { isBatchOrigin } from '../entry.js';
+/** One hour, in ms. Partitions are hourly. */
+export const HOUR_MS = 3_600_000;
+/** Column order and types, shared by the writer and the DuckDB projection. */
+export const PARQUET_COLUMNS = [
+    { name: 'id', type: 'STRING', nullable: false, bloomFilter: true },
+    { name: 'type', type: 'STRING', nullable: false },
+    { name: 'family_hash', type: 'STRING', nullable: true },
+    { name: 'content', type: 'STRING', nullable: false },
+    { name: 'tags', type: 'STRING', nullable: false },
+    { name: 'sequence', type: 'INT64', nullable: false },
+    { name: 'duration_ms', type: 'INT64', nullable: true },
+    { name: 'origin', type: 'STRING', nullable: false },
+    { name: 'trace_id', type: 'STRING', nullable: true, bloomFilter: true },
+    { name: 'instance_id', type: 'STRING', nullable: false },
+    { name: 'created_at', type: 'TIMESTAMP', nullable: false },
+];
+export function toParquetRow(entry, instanceId) {
+    return {
+        id: entry.id,
+        type: entry.type,
+        family_hash: entry.familyHash,
+        content: safeJson(entry.content),
+        tags: JSON.stringify(entry.tags),
+        sequence: BigInt(Math.trunc(entry.sequence)),
+        duration_ms: entry.durationMs == null ? null : BigInt(Math.round(entry.durationMs)),
+        origin: entry.origin,
+        trace_id: entry.traceId,
+        instance_id: instanceId,
+        created_at: entry.createdAt,
+    };
+}
+/** Rebuild an {@link Entry} from a row as either engine returns it. */
+export function fromParquetRow(row) {
+    const duration = row.duration_ms;
+    return {
+        id: String(row.id),
+        type: String(row.type),
+        familyHash: row.family_hash == null ? null : String(row.family_hash),
+        content: parseJson(String(row.content ?? 'null')),
+        tags: parseTags(String(row.tags ?? '[]')),
+        sequence: toNumber(row.sequence),
+        durationMs: duration == null ? null : toNumber(duration),
+        origin: isBatchOrigin(row.origin) ? row.origin : 'manual',
+        traceId: row.trace_id == null ? null : String(row.trace_id),
+        createdAt: toDate(row.created_at),
+    };
+}
+/**
+ * `type=<type>/dt=<yyyy-mm-dd>/hour=<hh>/` — type first, so a query on one type (and a
+ * bucket lifecycle rule on one type) only touches that type's prefix.
+ */
+export function partitionPrefix(type, hourStartMs) {
+    const d = new Date(hourStartMs);
+    const dt = `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())}`;
+    return `${typePrefix(type)}dt=${dt}/hour=${pad(d.getUTCHours())}/`;
+}
+export function typePrefix(type) {
+    return `type=${encodeURIComponent(type)}/`;
+}
+const KEY_PATTERN = /^type=([^/]+)\/dt=(\d{4})-(\d{2})-(\d{2})\/hour=(\d{2})\/[^/]+\.parquet$/;
+/** Parse a location-relative key, or `null` for anything that is not ours. */
+export function parsePartition(key) {
+    const match = KEY_PATTERN.exec(key);
+    if (!match)
+        return null;
+    const [, type, y, m, d, h] = match;
+    let decoded;
+    try {
+        decoded = decodeURIComponent(type);
+    }
+    catch {
+        return null;
+    }
+    return {
+        type: decoded,
+        hourStartMs: Date.UTC(Number(y), Number(m) - 1, Number(d), Number(h)),
+    };
+}
+export function hourStart(ms) {
+    return Math.floor(ms / HOUR_MS) * HOUR_MS;
+}
+/**
+ * Newest-first total order used by every tier: `createdAt` desc, then `sequence` desc
+ * (the Lucid store's tiebreaker), then `id` desc so the order is total across pods.
+ */
+export function compareNewestFirst(a, b) {
+    const t = b.createdAt.getTime() - a.createdAt.getTime();
+    if (t !== 0)
+        return t;
+    if (a.sequence !== b.sequence)
+        return b.sequence - a.sequence;
+    return a.id < b.id ? 1 : a.id > b.id ? -1 : 0;
+}
+/** The `{ page, size }` of a query, resolved: how many to skip and how many to keep. */
+export function resolvePage(query) {
+    if (query.size === undefined || query.size <= 0)
+        return { offset: 0, size: undefined };
+    const size = Math.floor(query.size);
+    const page = Math.max(1, Math.floor(query.page ?? 1));
+    return { offset: (page - 1) * size, size };
+}
+/**
+ * The predicates of an {@link EntryQuery} that need the row itself (everything except
+ * the time bounds and type, which partitions and statistics already prune). Applied
+ * in JS by the hyparquet engine and as a final guard on merged results.
+ */
+export function matchesQuery(entry, query) {
+    if (query.type !== undefined && entry.type !== query.type)
+        return false;
+    if (query.familyHash !== undefined && entry.familyHash !== query.familyHash)
+        return false;
+    if (query.traceId !== undefined && entry.traceId !== query.traceId)
+        return false;
+    if (query.traceIds !== undefined) {
+        if (entry.traceId === null || !query.traceIds.includes(entry.traceId))
+            return false;
+    }
+    if (query.before !== undefined && !(entry.createdAt < query.before))
+        return false;
+    if (query.after !== undefined && !(entry.createdAt > query.after))
+        return false;
+    if (query.tag !== undefined && !entry.tags.includes(query.tag))
+        return false;
+    if (query.search !== undefined) {
+        const needle = query.search.toLowerCase();
+        if (!entry.tags.some((tag) => tag.toLowerCase().includes(needle))) {
+            if (!safeJson(entry.content).toLowerCase().includes(needle))
+                return false;
+        }
+    }
+    return true;
+}
+function pad(n) {
+    return n < 10 ? `0${n}` : String(n);
+}
+function safeJson(value) {
+    try {
+        return JSON.stringify(value) ?? 'null';
+    }
+    catch {
+        return 'null';
+    }
+}
+function parseJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return null;
+    }
+}
+function parseTags(text) {
+    const parsed = parseJson(text);
+    return Array.isArray(parsed) ? parsed.filter((t) => typeof t === 'string') : [];
+}
+export function toNumber(value) {
+    if (typeof value === 'number')
+        return value;
+    if (typeof value === 'bigint')
+        return Number(value);
+    if (typeof value === 'string')
+        return Number(value) || 0;
+    return 0;
+}
+export function toDate(value) {
+    if (value instanceof Date)
+        return value;
+    if (typeof value === 'number' || typeof value === 'bigint')
+        return new Date(Number(value));
+    if (typeof value === 'string')
+        return new Date(value);
+    return new Date(0);
+}
+//# sourceMappingURL=parquet_format.js.map

package/dist/src/stores/parquet_format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet_format.js","sourceRoot":"","sources":["../../../src/stores/parquet_format.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,aAAa,EAAE,MAAM,aAAa,CAAC;AAGxD,8CAA8C;AAC9C,MAAM,CAAC,MAAM,OAAO,GAAG,SAAS,CAAC;AAsBjC,8EAA8E;AAC9E,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;IAClE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE;IACjD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;IACvD,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE;IACpD,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE;IACjD,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE;IACpD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE;IACtD,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE;IACnD,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE;IACvE,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE;IACxD,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,KAAK,EAAE;CAClD,CAAC;AAEX,MAAM,UAAU,YAAY,CAAC,KAAY,EAAE,UAAkB;IAC3D,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,UAAU;QAC7B,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC;QAChC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;QAChC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5C,WAAW,EAAE,KAAK,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACnF,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,KAAK,CAAC,OAAO;QACvB,WAAW,EAAE,UAAU;QACvB,UAAU,EAAE,KAAK,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,cAAc,CAAC,GAA4B;IACzD,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,CAAC;IACjC,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;QACtB,UAAU,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;QACpE,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC;QACjD,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;QACzC,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC;QAChC,UAAU,EAAE,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACxD,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;QACzD,OAAO,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC3D,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;KAClC,CAAC;AACJ,CAAC;AASD;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,WAAmB;IAC/D,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC;IAChC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,cAAc,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;IACtF,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,QAAQ,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC;AAC7C,CAAC;AAED,MAAM,WAAW,GAAG,0EAA0E,CAAC;AAE/F,8EAA8E;AAC9E,MAAM,UAAU,cAAc,CAAC,GAAW;IACxC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC;IACnC,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,kBAAkB,CAAC,IAAK,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO;QACL,IAAI,EAAE,OAAO;QACb,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;KACtE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAU;IAClC,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,OAAO,CAAC;AAC5C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,CAAQ,EAAE,CAAQ;IACnD,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;IACxD,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ;QAAE,OAAO,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;IAC9D,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,WAAW,CAAC,KAAiB;IAC3C,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IACvF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IACtD,OAAO,EAAE,MAAM,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,IAAI,EAAE,CAAC;AAC7C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,KAAY,EAAE,KAAiB;IAC1D,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxE,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC;IAC1F,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IACjF,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC;IACtF,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IAClF,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAChF,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7E,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YAClE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC5E,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC;IAChB,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/F,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACpD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,KAAc;IACnC,IAAI,KAAK,YAAY,IAAI;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3F,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;AACrB,CAAC"}

package/dist/src/stores/parquet_hyparquet.d.ts

@@ -0,0 +1,34 @@
+import type { Entry } from '../entry.js';
+import type { BucketCountQuery, BucketCountRow, EntryQuery, TraceIdRow } from '../store.js';
+import type { ColdEngineContext, ColdFile, ColdReadEngine } from './parquet.js';
+/**
+ * The default cold engine: pure JavaScript (hyparquet), nothing native and nothing
+ * fetched at runtime.
+ *
+ * Reads walk files newest-first and, inside each file, row groups newest-first,
+ * skipping any group whose `created_at` statistics fall outside the query, and stop
+ * as soon as the page is full and nothing older can still outrank it. Equality
+ * filters on `id` / `trace_id` also consult the bloom filters the writer emits.
+ */
+export declare class HyparquetEngine implements ColdReadEngine {
+    private readonly ctx;
+    readonly name: 'hyparquet';
+    constructor(ctx: ColdEngineContext);
+    list(files: ColdFile[], query: EntryQuery, limit: number | undefined): Promise<Entry[]>;
+    get(files: ColdFile[], id: string): Promise<Entry | null>;
+    traceIds(files: ColdFile[], query: {
+        before?: Date;
+        after?: Date;
+    }, limit: number): Promise<TraceIdRow[]>;
+    countByBucket(files: ColdFile[], query: BucketCountQuery): Promise<BucketCountRow[]>;
+    close(): Promise<void>;
+    /** One row group as plain objects; a file that vanished since listing reads as empty. */
+    private readGroup;
+}
+/**
+ * Every {@link EntryQuery} predicate, evaluated on the stored row before it is parsed.
+ * `tag` matches the quoted token inside the JSON text and `search` is a lower-cased
+ * substring over `content` and `tags` — the same semantics as the Lucid store.
+ */
+export declare function rowMatches(row: Record<string, unknown>, query: EntryQuery): boolean;
+//# sourceMappingURL=parquet_hyparquet.d.ts.map

package/dist/src/stores/parquet_hyparquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet_hyparquet.d.ts","sourceRoot":"","sources":["../../../src/stores/parquet_hyparquet.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5F,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAiB,MAAM,cAAc,CAAC;AAM/F;;;;;;;;GAQG;AACH,qBAAa,eAAgB,YAAW,cAAc;IAGxC,OAAO,CAAC,QAAQ,CAAC,GAAG;IAFhC,QAAQ,CAAC,IAAI,EAAG,WAAW,CAAU;IAErC,YAA6B,GAAG,EAAE,iBAAiB,EAAI;IAEjD,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAwB5F;IAEK,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAU9D;IAEK,QAAQ,CACZ,KAAK,EAAE,QAAQ,EAAE,EACjB,KAAK,EAAE;QAAE,MAAM,CAAC,EAAE,IAAI,CAAC;QAAC,KAAK,CAAC,EAAE,IAAI,CAAA;KAAE,EACtC,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,UAAU,EAAE,CAAC,CA8BvB;IAEK,aAAa,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAsBzF;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAG;IAE/B,yFAAyF;YAC3E,SAAS;CAwBxB;AA+BD;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CA0BnF"}