@3minapi/cli 0.0.0-stage → 0.22.1

package/dist/modules/workers/results/query.js

@@ -0,0 +1,175 @@
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { DESCRIBE_MAX_DIFF_PATHS, DESCRIBE_MAX_GROUPS, DESCRIBE_MAX_MERGE, REPORT_ENVELOPE_RESERVE, REPORT_MAX_BYTES } from "../../../core/domain/index.js";
+import { fitShape, ShapeMerger } from "./shape.js";
+import { resultsDbPath } from "./store.js";
+const RESULTS_CHILD = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './results-child.ts' : './results-child.js', import.meta.url));
+export async function runQuery(root, sql, timeoutMs, signal) {
+    const r = await runResultsChild({ op: 'sql', file: resultsDbPath(root), sql }, timeoutMs, signal);
+    if (r === 'timeout') {
+        return {
+            ok: false,
+            kind: 'timeout',
+            message: `the query did not finish within ${timeoutMs / 1000}s`
+        };
+    }
+    if (r === 'cancelled')
+        return { ok: false, kind: 'cancelled', message: 'the query was cancelled' };
+    return r;
+}
+export async function runReaderChild(root, hostId, request, timeoutMs, signal) {
+    const r = await runResultsChild({ op: 'reader', root, hostId, file: resultsDbPath(root), request }, timeoutMs, signal);
+    return r;
+}
+function runResultsChild(message, timeoutMs, signal) {
+    return new Promise((resolve, reject) => {
+        if (signal?.aborted)
+            return resolve('cancelled');
+        const child = spawn(process.execPath, [RESULTS_CHILD], {
+            stdio: ['pipe', 'pipe', 'pipe'],
+            windowsHide: true
+        });
+        const out = [];
+        const err = [];
+        let timedOut = false;
+        const timer = setTimeout(() => {
+            timedOut = true;
+            child.kill('SIGKILL');
+        }, timeoutMs);
+        let cancelled = false;
+        const onAbort = () => {
+            cancelled = true;
+            child.kill('SIGKILL');
+        };
+        signal?.addEventListener('abort', onAbort, { once: true });
+        child.stdout.on('data', (c) => out.push(c));
+        child.stderr.on('data', (c) => err.push(c));
+        child.on('error', (e) => {
+            clearTimeout(timer);
+            reject(e);
+        });
+        child.on('close', (code) => {
+            clearTimeout(timer);
+            signal?.removeEventListener('abort', onAbort);
+            if (cancelled)
+                return resolve('cancelled');
+            if (timedOut)
+                return resolve('timeout');
+            const text = Buffer.concat(out).toString('utf8');
+            try {
+                resolve(JSON.parse(text));
+            }
+            catch {
+                const detail = Buffer.concat(err).toString('utf8').trim();
+                reject(new Error(`the query process ended without a result (exit code ${code})${detail ? `: ${detail}` : ''}`));
+            }
+        });
+        child.stdin.end(JSON.stringify(message));
+    });
+}
+export const DESCRIBE_MAX_BYTES = REPORT_MAX_BYTES - REPORT_ENVELOPE_RESERVE;
+export function replacer(_key, value) {
+    if (typeof value === 'bigint')
+        return value.toString();
+    if (value instanceof Uint8Array)
+        return `<blob ${value.byteLength} bytes>`;
+    return value;
+}
+export const jsonBytes = (v) => Buffer.byteLength(JSON.stringify(v, replacer));
+export function describeTask(r, maxBytes = DESCRIBE_MAX_BYTES) {
+    const build = (shape) => ({
+        task_id: r.task_id,
+        worker: r.worker,
+        created_at: r.created_at,
+        shape_hash: r.shape_hash ?? '',
+        shape
+    });
+    return build(fitShape(r.shape, (s) => jsonBytes(build(s)) <= maxBytes));
+}
+export function describeWorker(store, filter, maxBytes = DESCRIBE_MAX_BYTES, maxMerge = DESCRIBE_MAX_MERGE) {
+    return store.snapshot(() => describeIn(store, filter, maxBytes, maxMerge));
+}
+function describeIn(store, filter, maxBytes, maxMerge) {
+    const counted = store.shapeGroups(filter);
+    const total = counted.reduce((n, g) => n + g.count, 0);
+    const baseline = store.latestShapeHash(filter);
+    const baseGroup = counted.find((g) => g.shape_hash === baseline);
+    const ordered = baseGroup ? [baseGroup, ...counted.filter((g) => g !== baseGroup)] : counted;
+    let merged = null;
+    if (baseline !== null) {
+        const merger = new ShapeMerger();
+        let results = 0;
+        let since = '';
+        for (const r of store.recentShapes(filter, baseline, maxMerge)) {
+            if (results === 0)
+                since = r.created_at;
+            merger.add(JSON.parse(r.shape));
+            results++;
+        }
+        const shape = merger.result();
+        merged = { shape, paths: new Set(Object.keys(shape.paths)), results, since };
+    }
+    const cap = (list) => list.length > DESCRIBE_MAX_DIFF_PATHS
+        ? {
+            list: list.slice(0, DESCRIBE_MAX_DIFF_PATHS),
+            more: list.length - DESCRIBE_MAX_DIFF_PATHS
+        }
+        : { list, more: 0 };
+    const pathsOf = (hash) => {
+        if (hash === baseline && merged)
+            return merged.paths;
+        for (const r of store.recentShapes(filter, hash, 1)) {
+            return new Set(Object.keys(JSON.parse(r.shape).paths));
+        }
+        return new Set();
+    };
+    const groups = ordered.slice(0, DESCRIBE_MAX_GROUPS).map((g) => {
+        const paths = pathsOf(g.shape_hash);
+        const base = merged?.paths;
+        const added = cap(base ? [...paths].filter((p) => !base.has(p)) : []);
+        const removed = cap(base ? [...base].filter((p) => !paths.has(p)) : []);
+        return {
+            shape_hash: g.shape_hash,
+            count: g.count,
+            first: g.first,
+            last: g.last,
+            task_ids: store.recentTaskIds(filter, g.shape_hash, 3),
+            added: added.list,
+            removed: removed.list,
+            ...(added.more > 0 ? { added_more: added.more } : {}),
+            ...(removed.more > 0 ? { removed_more: removed.more } : {})
+        };
+    });
+    const partial = merged !== null && baseGroup !== undefined && merged.results < baseGroup.count;
+    const build = (kept, shape) => {
+        const rest = ordered.slice(kept);
+        return {
+            worker: filter.worker ?? '',
+            total,
+            baseline,
+            groups: groups.slice(0, kept),
+            others: rest.length > 0
+                ? { groups: rest.length, count: rest.reduce((n, g) => n + g.count, 0) }
+                : null,
+            shape,
+            shape_results: merged?.results ?? 0,
+            shape_since: merged?.since ?? null,
+            ...(partial
+                ? {
+                    note: `values and ranges in shape cover the latest ${merged.results} of ${baseGroup.count} results ` +
+                        `with this shape (since ${merged.since}). For all of them, query with json_extract — min/max or DISTINCT`
+                }
+                : {})
+        };
+    };
+    let kept = groups.length;
+    for (;;) {
+        const fitted = merged
+            ? fitShape(merged.shape, (s) => jsonBytes(build(kept, s)) <= maxBytes)
+            : null;
+        const out = build(kept, fitted);
+        if (kept === 0 || jsonBytes(out) <= maxBytes)
+            return out;
+        kept = Math.floor(kept / 2);
+    }
+}

package/dist/modules/workers/results/reader-input.js

@@ -0,0 +1,134 @@
+import { BUILTIN_TASK_IDS_MAX, READ_LIMIT_MAX } from "../../../core/domain/index.js";
+export const READER_USAGE = 'input is one of: {"op":"describe","task_id":"…","run_no?":1} | {"op":"describe","worker":"…","since?":"ISO date"} | ' +
+    '{"op":"query","sql":"SELECT … FROM results …","worker?":"…","task_id?":"…","task_ids?":["…"],"since?":"ISO date","expect_shape?":"…"} ' +
+    '(at least one of worker, task_id, task_ids, since) | {"op":"read","task_id":"…","run_no?":1,"path?":"$.items","offset?":0,"limit?":100} ' +
+    '(run_no picks a run of a repeating task; the latest run when left out)';
+const TASK_ID = /^[0-9A-Za-z-]{1,64}$/;
+const SHAPE_HASH = /^[0-9a-f]{16}$/;
+const WORKER_MAX_CHARS = 50;
+const SQL_MAX_CHARS = 20_000;
+const PATH_MAX_CHARS = 1_000;
+export function parseReaderInput(input) {
+    try {
+        return { ok: true, req: parse(input) };
+    }
+    catch (err) {
+        return { ok: false, message: `${err.message}. ${READER_USAGE}` };
+    }
+}
+function parse(input) {
+    if (input === null || typeof input !== 'object' || Array.isArray(input)) {
+        throw new Error('input must be a JSON object');
+    }
+    const o = input;
+    switch (o.op) {
+        case 'describe':
+            if (o.task_id !== undefined) {
+                only(o, ['op', 'task_id', 'run_no']);
+                return {
+                    op: 'describe',
+                    task_id: taskId(o.task_id, 'task_id'),
+                    ...(o.run_no !== undefined ? { run_no: runNo(o.run_no) } : {})
+                };
+            }
+            if (o.worker !== undefined) {
+                only(o, ['op', 'worker', 'since']);
+                return {
+                    op: 'describe',
+                    worker: worker(o.worker),
+                    ...(o.since !== undefined ? { since: since(o.since) } : {})
+                };
+            }
+            throw new Error('describe needs task_id or worker');
+        case 'query': {
+            only(o, ['op', 'sql', 'worker', 'task_id', 'task_ids', 'since', 'expect_shape']);
+            if (typeof o.sql !== 'string' || o.sql.trim() === '')
+                throw new Error('sql must be a string');
+            if (o.sql.length > SQL_MAX_CHARS)
+                throw new Error(`sql is longer than ${SQL_MAX_CHARS} characters`);
+            const ids = [];
+            if (o.task_id !== undefined)
+                ids.push(taskId(o.task_id, 'task_id'));
+            if (o.task_ids !== undefined) {
+                if (!Array.isArray(o.task_ids) || o.task_ids.length === 0) {
+                    throw new Error('task_ids must be a non-empty array');
+                }
+                for (const id of o.task_ids)
+                    ids.push(taskId(id, 'task_ids[]'));
+            }
+            if (ids.length > BUILTIN_TASK_IDS_MAX) {
+                throw new Error(`at most ${BUILTIN_TASK_IDS_MAX} task ids — use worker and since instead`);
+            }
+            const scope = {
+                ...(o.worker !== undefined ? { worker: worker(o.worker) } : {}),
+                ...(ids.length > 0 ? { task_ids: [...new Set(ids)] } : {}),
+                ...(o.since !== undefined ? { since: since(o.since) } : {})
+            };
+            if (Object.keys(scope).length === 0) {
+                throw new Error('query needs a scope: worker, task_id, task_ids or since');
+            }
+            if (o.expect_shape !== undefined) {
+                if (typeof o.expect_shape !== 'string' || !SHAPE_HASH.test(o.expect_shape)) {
+                    throw new Error('expect_shape must be a shape_hash (16 hex characters)');
+                }
+            }
+            return {
+                op: 'query',
+                sql: o.sql,
+                scope,
+                ...(o.expect_shape !== undefined ? { expect_shape: o.expect_shape } : {})
+            };
+        }
+        case 'read': {
+            only(o, ['op', 'task_id', 'run_no', 'path', 'offset', 'limit']);
+            const path = o.path === undefined ? '$' : o.path;
+            if (typeof path !== 'string' || !path.startsWith('$') || path.length > PATH_MAX_CHARS) {
+                throw new Error('path must be a JSON path starting with $ (for example $.items)');
+            }
+            return {
+                op: 'read',
+                task_id: taskId(o.task_id, 'task_id'),
+                ...(o.run_no !== undefined ? { run_no: runNo(o.run_no) } : {}),
+                path,
+                offset: o.offset === undefined ? 0 : count(o.offset, 'offset', 0, Number.MAX_SAFE_INTEGER),
+                limit: o.limit === undefined ? null : count(o.limit, 'limit', 1, READ_LIMIT_MAX)
+            };
+        }
+        default:
+            throw new Error('op must be "describe", "query" or "read"');
+    }
+}
+function runNo(v) {
+    return count(v, 'run_no', 1, Number.MAX_SAFE_INTEGER);
+}
+function only(o, keys) {
+    const extra = Object.keys(o).filter((k) => !keys.includes(k));
+    if (extra.length > 0)
+        throw new Error(`unknown field ${extra.map((k) => `"${k}"`).join(', ')}`);
+}
+function taskId(v, field) {
+    if (typeof v !== 'string' || !TASK_ID.test(v))
+        throw new Error(`${field} must be a task id`);
+    return v;
+}
+function worker(v) {
+    if (typeof v !== 'string' ||
+        v.length === 0 ||
+        Array.from(v).length > WORKER_MAX_CHARS ||
+        v.includes('\0')) {
+        throw new Error('worker must be a worker name');
+    }
+    return v;
+}
+function since(v) {
+    const t = typeof v === 'string' ? Date.parse(v) : Number.NaN;
+    if (Number.isNaN(t))
+        throw new Error('since must be an ISO date or time (for example 2026-10-01)');
+    return new Date(t).toISOString();
+}
+function count(v, field, min, max) {
+    if (typeof v !== 'number' || !Number.isInteger(v) || v < min || v > max) {
+        throw new Error(`${field} must be an integer from ${min} to ${max}`);
+    }
+    return v;
+}