@3minapi/cli 0.0.0-stage → 0.22.1

package/dist/modules/workers/results/reader.js

@@ -0,0 +1,430 @@
+import { randomBytes } from 'node:crypto';
+import { DESCRIBE_MAX_GROUPS, READ_MAX_BYTES, RESULTS_TABLE } from "../../../core/domain/index.js";
+import { DESCRIBE_MAX_BYTES, describeTask, describeWorker, jsonBytes } from "./query.js";
+export { replacer } from "./query.js";
+import { MISSING_PATHS_MAX, missingPaths, missingPathsWarning } from "./missing-paths.js";
+import { childPath } from "./shape.js";
+import { ResultStore } from "./store.js";
+export const REMOTE_SOURCES = ['pull', 'public'];
+class Failure extends Error {
+    reason;
+    constructor(reason, message) {
+        super(message);
+        this.reason = reason;
+    }
+}
+export function runReader(root, db, c, req, hostId) {
+    try {
+        switch (req.op) {
+            case 'describe':
+                return {
+                    ok: true,
+                    result: 'task_id' in req
+                        ? describeOne(db, req.task_id, hostId, req.run_no)
+                        : describeMany(root, req, hostId)
+                };
+            case 'query':
+                return { ok: true, result: query(db, c, req, hostId) };
+            case 'read':
+                return { ok: true, result: read(db, req, hostId) };
+        }
+    }
+    catch (err) {
+        if (err instanceof Failure)
+            return { ok: false, reason: err.reason, message: err.message };
+        throw err;
+    }
+}
+function findRow(db, taskId, hostId, withPayload = false, runNo) {
+    const row = db
+        .prepare(`SELECT task_id, worker, created_at, status, bytes, shape_hash, shape, run_no${withPayload ? ', payload' : ''}
+			 FROM ${RESULTS_TABLE} WHERE task_id = ? AND host_id = ? AND source IN (${REMOTE_SOURCES.map(() => '?').join(', ')})
+			 ${runNo === undefined ? 'ORDER BY run_no DESC LIMIT 1' : 'AND run_no = ?'}`)
+        .get(taskId, hostId, ...REMOTE_SOURCES, ...(runNo === undefined ? [] : [runNo]));
+    if (!row) {
+        const runs = runNo === undefined ? null : runInfo(db, taskId, hostId);
+        throw new Failure('result_not_found', runs
+            ? `no saved run ${runNo} for task ${taskId} — ${runs.count} run(s) are saved, the latest is run ${runs.last_run_no}`
+            : `no saved result for task ${taskId} on this host — it ran on another host, was pruned, or its worker has not finished`);
+    }
+    return row;
+}
+function runInfo(db, taskId, hostId) {
+    const r = db
+        .prepare(`SELECT count(*) AS n, max(run_no) AS last FROM ${RESULTS_TABLE}
+			 WHERE task_id = ? AND host_id = ? AND source IN (${REMOTE_SOURCES.map(() => '?').join(', ')})`)
+        .get(taskId, hostId, ...REMOTE_SOURCES);
+    return Number(r.n) > 0 ? { count: Number(r.n), last_run_no: Number(r.last) } : null;
+}
+function runFields(db, row, hostId) {
+    const runs = runInfo(db, row.task_id, hostId);
+    if (!runs || (runs.last_run_no <= 1 && row.run_no <= 1))
+        return {};
+    return { run_no: row.run_no, runs };
+}
+function describeOne(db, taskId, hostId, runNo) {
+    const row = findRow(db, taskId, hostId, false, runNo);
+    const run = runFields(db, row, hostId);
+    if (row.status === 'failed' || row.shape === null) {
+        const error = JSON.parse(findRow(db, taskId, hostId, true, row.run_no).payload);
+        const out = {
+            task_id: row.task_id,
+            ...run,
+            worker: row.worker,
+            created_at: row.created_at,
+            status: row.status,
+            error
+        };
+        return jsonBytes(out) <= DESCRIBE_MAX_BYTES
+            ? out
+            : {
+                ...out,
+                error: { message: 'the saved error is too large to show; read it with op "read"' }
+            };
+    }
+    return {
+        ...run,
+        status: row.status,
+        bytes: row.bytes,
+        ...describeTask({
+            task_id: row.task_id,
+            host_id: hostId,
+            worker: row.worker,
+            created_at: row.created_at,
+            shape_hash: row.shape_hash,
+            shape: JSON.parse(row.shape),
+            source: 'pull',
+            status: row.status,
+            reply: 'result',
+            bytes: row.bytes,
+            payload: ''
+        })
+    };
+}
+function describeMany(root, req, hostId) {
+    const store = ResultStore.openReadOnly(root);
+    if (!store)
+        throw new Failure('result_not_found', 'no results are saved on this host');
+    try {
+        const out = describeWorker(store, {
+            worker: req.worker,
+            ...(req.since !== undefined ? { since: req.since } : {}),
+            sources: REMOTE_SOURCES,
+            hostId
+        });
+        if (out.total === 0) {
+            throw new Failure('result_not_found', `no completed results of worker "${req.worker}"${req.since ? ` since ${req.since}` : ''} are saved on this host`);
+        }
+        return out;
+    }
+    finally {
+        store.close();
+    }
+}
+export const RESULTS_VIEW_COLUMNS = [
+    'task_id',
+    'run_no',
+    'worker',
+    'created_at',
+    'status',
+    'bytes',
+    'shape_hash',
+    'payload'
+];
+function query(db, c, req, hostId) {
+    const inner = `_scope_${randomBytes(8).toString('hex')}`;
+    db.exec(`CREATE TEMP VIEW ${inner} AS SELECT ${RESULTS_VIEW_COLUMNS.join(', ')} FROM main.${RESULTS_TABLE} WHERE ${scopeSql(req.scope, hostId)};
+		 CREATE TEMP VIEW results AS SELECT * FROM ${inner};`);
+    const groups = db
+        .prepare(`SELECT status, shape_hash, count(*) AS n FROM ${inner} GROUP BY status, shape_hash ORDER BY n DESC`)
+        .all();
+    if (groups.length === 0) {
+        throw new Failure('result_not_found', 'no saved results match this scope on this host (worker, task_id(s) and since are combined with AND)');
+    }
+    const shapes = groups
+        .filter((g) => g.status === 'completed' && g.shape_hash !== null)
+        .map((g) => ({ shape_hash: g.shape_hash, tasks: g.n }));
+    const failed = groups.filter((g) => g.status === 'failed').reduce((n, g) => n + g.n, 0);
+    const shown = shapes.slice(0, DESCRIBE_MAX_GROUPS);
+    const rest = shapes.slice(DESCRIBE_MAX_GROUPS);
+    const shapesMore = rest.length > 0 ? { groups: rest.length, tasks: rest.reduce((n, s) => n + s.tasks, 0) } : null;
+    if (req.expect_shape !== undefined) {
+        const other = shapes.filter((s) => s.shape_hash !== req.expect_shape);
+        if (other.length > 0 || shapes.length === 0) {
+            const listed = shown.map((s) => `${s.shape_hash}: ${s.tasks}`).join(', ') +
+                (shapesMore ? `, and ${shapesMore.groups} more` : '');
+            throw new Failure('shape_mismatch', `the scope has results of other shapes (${listed || 'none completed'}) ` +
+                `than expect_shape ${req.expect_shape} — the query did not run. Narrow the scope (task_ids or since), or describe the worker to see when the shape changed`);
+        }
+    }
+    const missing = scopeMissingPaths(db, req, hostId);
+    db.setAuthorizer((action, p1, _p2, dbName, view) => {
+        if (action === c.SQLITE_SELECT ||
+            action === c.SQLITE_FUNCTION ||
+            action === c.SQLITE_RECURSIVE) {
+            return c.SQLITE_OK;
+        }
+        if (action === c.SQLITE_READ) {
+            if (dbName === 'main' && p1 === RESULTS_TABLE && view === inner)
+                return c.SQLITE_OK;
+            if (dbName === 'temp' && (p1 === 'results' || p1 === inner))
+                return c.SQLITE_OK;
+            if (p1 === 'json_each' || p1 === 'json_tree')
+                return c.SQLITE_OK;
+            if (dbName === null)
+                return c.SQLITE_OK;
+        }
+        return c.SQLITE_DENY;
+    });
+    let stmt;
+    try {
+        stmt = db.prepare(req.sql);
+    }
+    catch (err) {
+        throw sqlFailure(err);
+    }
+    const columns = stmt.columns().map((col) => col.name);
+    stmt.setReturnArrays(true);
+    const tasks = groups.reduce((n, g) => n + g.n, 0);
+    const warnings = [
+        ...(shapes.length > 1
+            ? [
+                `the scope mixes ${shapes.length} result shapes — a path that exists in one shape is NULL in another, ` +
+                    'so sums and counts can be silently wrong. Check with describe, then pass expect_shape or narrow the scope'
+            ]
+            : []),
+        ...(missing.length > 0 ? [missingPathsWarning(missing)] : [])
+    ];
+    const build = (rows, truncated) => ({
+        columns,
+        rows,
+        row_count: rows.length,
+        ...(truncated
+            ? {
+                truncated: true,
+                note: `stopped at ${rows.length} rows to stay within the reply limit — aggregate in SQL (count, sum, GROUP BY) or add LIMIT/OFFSET`
+            }
+            : {}),
+        scope: { tasks, ...(failed > 0 ? { failed } : {}) },
+        shapes: shown,
+        ...(shapesMore ? { shapes_more: shapesMore } : {}),
+        ...(missing.length > 0 ? { missing_paths: missing.slice(0, MISSING_PATHS_MAX) } : {}),
+        ...(warnings.length > 0 ? { warning: warnings.join('. ') } : {})
+    });
+    const budget = DESCRIBE_MAX_BYTES - jsonBytes(build([], true)) - 16;
+    const rows = [];
+    let used = 0;
+    let truncated = false;
+    try {
+        for (const row of stmt.iterate()) {
+            const size = jsonBytes(row) + 1;
+            if (used + size > budget) {
+                truncated = true;
+                break;
+            }
+            used += size;
+            rows.push(row);
+        }
+    }
+    catch (err) {
+        throw sqlFailure(err);
+    }
+    return build(rows, truncated);
+}
+function scopeMissingPaths(db, req, hostId) {
+    const rows = db
+        .prepare(`SELECT shape FROM main.${RESULTS_TABLE} WHERE ${scopeSql(req.scope, hostId)} AND status = 'completed' GROUP BY shape_hash`)
+        .all();
+    const known = new Set();
+    for (const row of rows) {
+        if (row.shape === null)
+            return [];
+        let shape;
+        try {
+            shape = JSON.parse(row.shape);
+        }
+        catch {
+            return [];
+        }
+        if (shape.truncated)
+            return [];
+        for (const path of Object.keys(shape.paths))
+            known.add(path);
+    }
+    if (known.size === 0)
+        return [];
+    return missingPaths(req.sql, known);
+}
+function scopeSql(scope, hostId) {
+    const lit = (v) => `'${v.replace(/'/g, "''")}'`;
+    const conds = [`source IN (${REMOTE_SOURCES.map(lit).join(', ')})`, `host_id = ${lit(hostId)}`];
+    if (scope.worker !== undefined)
+        conds.push(`worker = ${lit(scope.worker)}`);
+    if (scope.task_ids !== undefined)
+        conds.push(`task_id IN (${scope.task_ids.map(lit).join(', ')})`);
+    if (scope.since !== undefined)
+        conds.push(`created_at >= ${lit(scope.since)}`);
+    return conds.join(' AND ');
+}
+function sqlFailure(err) {
+    const e = err;
+    const message = e.message ?? String(err);
+    if (e.errcode === 23) {
+        return new Failure('query_denied', `${message} — queries may only read the view "results" (columns: ${RESULTS_VIEW_COLUMNS.join(', ')}) and use functions such as json_extract and json_each`);
+    }
+    return new Failure('query_error', message);
+}
+function read(db, req, hostId) {
+    const row = findRow(db, req.task_id, hostId, true, req.run_no);
+    return readNode(JSON.parse(row.payload), { task_id: req.task_id, ...runFields(db, row, hostId), status: row.status, path: req.path }, req);
+}
+export function readPayloadSlice(payload, at) {
+    try {
+        const result = readNode(JSON.parse(payload), { path: at.path }, at);
+        return { ok: true, result };
+    }
+    catch (err) {
+        if (err instanceof Failure || err instanceof SyntaxError) {
+            return { ok: false, message: err.message };
+        }
+        throw err;
+    }
+}
+function readNode(root, base, req) {
+    const view = req.summarize ?? ((v) => v);
+    let node = root;
+    for (const step of parsePath(req.path)) {
+        if (typeof step === 'number') {
+            if (!Array.isArray(node) || step >= node.length)
+                throw missing(req.path);
+            node = node[step];
+        }
+        else {
+            if (node === null ||
+                typeof node !== 'object' ||
+                Array.isArray(node) ||
+                !Object.hasOwn(node, step)) {
+                throw missing(req.path);
+            }
+            node = node[step];
+        }
+    }
+    const limit = req.limit ?? Number.MAX_SAFE_INTEGER;
+    if (Array.isArray(node)) {
+        return slice(base, 'array', node.length, req.offset, limit, (i) => view(node[i]), (items) => ({ items }), req.path);
+    }
+    if (node !== null && typeof node === 'object') {
+        const keys = Object.keys(node);
+        const obj = node;
+        return slice(base, 'object', keys.length, req.offset, limit, (i) => [keys[i], view(obj[keys[i]])], (entries) => ({
+            items: Object.fromEntries(entries)
+        }), req.path);
+    }
+    if (typeof node === 'string')
+        return readText(base, node, req.offset, limit);
+    if (req.offset > 0)
+        throw new Failure('read_out_of_range', `${req.path} is a single value — read it with offset 0`);
+    return { ...base, kind: 'value', total: 1, offset: 0, count: 1, next: null, value: node };
+}
+function slice(base, kind, total, offset, limit, at, wrap, path) {
+    if (offset > total || (offset === total && total > 0)) {
+        throw new Failure('read_out_of_range', `offset ${offset} is past the end of ${path} (${total} ${kind === 'array' ? 'elements' : 'keys'})`);
+    }
+    const items = [];
+    let used = 0;
+    for (let i = offset; i < total && items.length < limit; i++) {
+        const v = at(i);
+        const size = jsonBytes(v) + 1;
+        if (used + size > READ_MAX_BYTES) {
+            if (items.length === 0) {
+                const sub = kind === 'array' ? `${path}[${i}]` : childPath(path, v[0]);
+                throw new Failure('read_out_of_range', `${sub} alone is ${size} bytes, more than one read (${READ_MAX_BYTES} bytes) — read a path inside it (for example ${sub}.<field>) or query it with SQL`);
+            }
+            break;
+        }
+        used += size;
+        items.push(v);
+    }
+    const end = offset + items.length;
+    return {
+        ...base,
+        kind,
+        total,
+        offset,
+        count: items.length,
+        next: end < total ? end : null,
+        ...wrap(items)
+    };
+}
+function readText(base, text, offset, limit) {
+    const chars = Array.from(text);
+    const total = chars.length;
+    if (offset > total || (offset === total && total > 0)) {
+        throw new Failure('read_out_of_range', `offset ${offset} is past the end of the text (${total} characters)`);
+    }
+    let used = 2;
+    let end = offset;
+    while (end < total && end - offset < limit) {
+        const size = jsonBytes(chars[end]) - 2;
+        if (used + size > READ_MAX_BYTES)
+            break;
+        used += size;
+        end++;
+    }
+    return {
+        ...base,
+        kind: 'text',
+        total,
+        offset,
+        count: end - offset,
+        next: end < total ? end : null,
+        text: chars.slice(offset, end).join('')
+    };
+}
+function missing(path) {
+    return new Failure('read_out_of_range', `${path} does not exist in this result — check the paths with op "describe"`);
+}
+export function parsePath(path) {
+    const steps = [];
+    let i = 1;
+    const bad = () => new Failure('read_out_of_range', `cannot parse path ${path} — use $.key, $."a key" or $.list[3]`);
+    if (path[0] !== '$')
+        throw bad();
+    while (i < path.length) {
+        if (path[i] === '.') {
+            i++;
+            if (path[i] === '"') {
+                let j = i + 1;
+                while (j < path.length && path[j] !== '"')
+                    j += path[j] === '\\' ? 2 : 1;
+                if (j >= path.length)
+                    throw bad();
+                try {
+                    steps.push(JSON.parse(path.slice(i, j + 1)));
+                }
+                catch {
+                    throw bad();
+                }
+                i = j + 1;
+            }
+            else {
+                const m = /^[A-Za-z_][A-Za-z0-9_]*/.exec(path.slice(i));
+                if (!m)
+                    throw bad();
+                steps.push(m[0]);
+                i += m[0].length;
+            }
+        }
+        else if (path[i] === '[') {
+            const m = /^\[(\d+)\]/.exec(path.slice(i));
+            if (!m)
+                throw bad();
+            steps.push(Number(m[1]));
+            i += m[0].length;
+        }
+        else {
+            throw bad();
+        }
+    }
+    return steps;
+}

package/dist/modules/workers/results/results-child.js

@@ -0,0 +1,64 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { suppressSqliteExperimentalWarning } from "../../../core/warnings.js";
+suppressSqliteExperimentalWarning();
+const { DatabaseSync, constants } = await import('node:sqlite');
+const ALLOWED = new Set([
+    constants.SQLITE_SELECT,
+    constants.SQLITE_READ,
+    constants.SQLITE_FUNCTION,
+    constants.SQLITE_RECURSIVE
+]);
+const chunks = [];
+for await (const chunk of process.stdin)
+    chunks.push(chunk);
+const msg = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+const { replacer, runReader } = await import(__rewriteRelativeImportExtension(import.meta.url.endsWith('.ts') ? './reader.ts' : './reader.js'));
+let result;
+if (msg.op === 'sql') {
+    try {
+        const db = new DatabaseSync(msg.file, { readOnly: true, timeout: 5_000 });
+        try {
+            db.setAuthorizer((action) => ALLOWED.has(action) ? constants.SQLITE_OK : constants.SQLITE_DENY);
+            const stmt = db.prepare(msg.sql);
+            const columns = stmt.columns().map((c) => c.name);
+            stmt.setReturnArrays(true);
+            const rows = stmt.all();
+            result = { ok: true, columns, rows };
+        }
+        finally {
+            db.close();
+        }
+    }
+    catch (err) {
+        const e = err;
+        const message = e.message ?? String(err);
+        const kind = e.errcode === 23
+            ? 'denied'
+            : /syntax error|incomplete input/.test(message)
+                ? 'syntax'
+                : 'error';
+        result = { ok: false, kind, message };
+    }
+}
+else {
+    try {
+        const db = new DatabaseSync(msg.file, { readOnly: true, timeout: 5_000 });
+        try {
+            result = runReader(msg.root, db, constants, msg.request, msg.hostId);
+        }
+        finally {
+            db.close();
+        }
+    }
+    catch (err) {
+        result = { ok: false, reason: 'query_error', message: err.message ?? String(err) };
+    }
+}
+process.stdout.write(JSON.stringify(result, replacer));