@3minapi/cli 0.0.0-stage → 0.22.1

package/dist/modules/workers/runtime/error-chain.js

@@ -0,0 +1,109 @@
+const MAX_DEPTH = 5;
+const MAX_AGGREGATE_ITEMS = 3;
+const NON_ERROR_MAX_CHARS = 200;
+const CHAIN = ' ← ';
+export function errorMessage(err) {
+    if (!(err instanceof Error))
+        return String(err);
+    const head = err.message || err.name;
+    const inner = describeCauses(err, 1, new Set([err]));
+    return join(head, inner);
+}
+export function errorStack(err) {
+    if (!(err instanceof Error))
+        return undefined;
+    const lines = [err.stack ?? errorMessage(err)];
+    const seen = new Set([err]);
+    const walk = (e, depth) => {
+        if (depth > MAX_DEPTH)
+            return;
+        for (const c of causesOf(e)) {
+            if (seen.has(c))
+                continue;
+            if (typeof c === 'object' && c !== null)
+                seen.add(c);
+            lines.push(`Caused by: ${c instanceof Error ? (c.stack ?? c.message) : nonError(c)}`);
+            if (c instanceof Error)
+                walk(c, depth + 1);
+        }
+    };
+    walk(err, 1);
+    return lines.join('\n');
+}
+export function withoutHome(message, home) {
+    if (home.length < 3)
+        return message;
+    const forms = [...new Set([home, home.replaceAll('\\', '/')])];
+    const pattern = forms.map((f) => f.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|');
+    return message.replace(new RegExp(`(?:${pattern})(?=[\\\\/'"\`\\s:),;]|$)`, 'g'), '~');
+}
+function causesOf(e) {
+    const list = [];
+    if (Array.isArray(e.errors))
+        list.push(...e.errors);
+    if ('cause' in e && e.cause !== undefined)
+        list.push(e.cause);
+    return list;
+}
+function describeCauses(e, depth, seen) {
+    if (depth > MAX_DEPTH)
+        return '';
+    const parts = [];
+    if (Array.isArray(e.errors) && e.errors.length > 0) {
+        const items = e.errors
+            .slice(0, MAX_AGGREGATE_ITEMS)
+            .map((x) => describe(x, depth, seen))
+            .filter((s) => s !== '');
+        const more = e.errors.length - MAX_AGGREGATE_ITEMS;
+        if (more > 0)
+            items.push(`and ${more} more`);
+        if (items.length > 0)
+            parts.push(items.join('; '));
+    }
+    if ('cause' in e && e.cause !== undefined) {
+        const c = describe(e.cause, depth, seen);
+        if (c !== '')
+            parts.push(c);
+    }
+    return parts.join(CHAIN);
+}
+function describe(value, depth, seen) {
+    if (typeof value === 'object' && value !== null) {
+        if (seen.has(value))
+            return '';
+        seen.add(value);
+    }
+    if (!(value instanceof Error))
+        return nonError(value);
+    const e = value;
+    const aggregate = Array.isArray(e.errors) && e.errors.length > 0;
+    const message = e.message || (aggregate ? '' : e.name);
+    const code = typeof e.code === 'string' && !message.includes(e.code) ? e.code : null;
+    const head = aggregate && !message ? '' : code ? `${message} (${code})` : message;
+    return join(head, describeCauses(e, depth + 1, seen));
+}
+function join(head, inner) {
+    if (!head)
+        return inner;
+    if (!inner)
+        return head;
+    if (inner === head || inner.startsWith(`${head}${CHAIN}`))
+        return inner;
+    return `${head}${CHAIN}${inner}`;
+}
+function nonError(value) {
+    let text;
+    if (typeof value === 'string')
+        text = value;
+    else if (value === null || typeof value !== 'object')
+        text = String(value);
+    else {
+        try {
+            text = JSON.stringify(value) ?? String(value);
+        }
+        catch {
+            text = Object.prototype.toString.call(value);
+        }
+    }
+    return text.length > NON_ERROR_MAX_CHARS ? `${text.slice(0, NON_ERROR_MAX_CHARS)}…` : text;
+}

package/dist/modules/workers/runtime/executor.js

@@ -0,0 +1,289 @@
+import { getDefaultAutoSelectFamilyAttemptTimeout } from 'node:net';
+import { homedir } from 'node:os';
+import { createInterface } from 'node:readline';
+import { Worker } from 'node:worker_threads';
+import { CANCEL_GRACE_MS, DEFAULT_WORKER_TIMEOUT_MS, ERROR_MESSAGE_MAX_CHARS, STORED_ERROR_MAX_CHARS, TERMINATE_WAIT_MS, WORKER_TIMEOUT_MAX_MS } from "../../../core/domain/index.js";
+import { redactAndTruncate } from "../../../core/redact.js";
+import { errorMessage, errorStack, withoutHome } from "./error-chain.js";
+import { resolveWorkerFile } from "./resolve.js";
+export const THREAD_URL = new URL(import.meta.url.endsWith('.ts') ? './thread.ts' : './thread.js', import.meta.url);
+export class Executor {
+    opts;
+    active = new Map();
+    queue = [];
+    stopping = new Set();
+    halted = null;
+    constructor(opts) {
+        this.opts = opts;
+    }
+    get size() {
+        return this.active.size + this.queue.length;
+    }
+    get concurrency() {
+        return this.opts.concurrency;
+    }
+    get free() {
+        return Math.max(0, this.opts.concurrency - this.size);
+    }
+    has(id) {
+        return this.active.has(id) || this.queue.some((q) => q.job.id === id);
+    }
+    run(job, hooks = {}) {
+        return new Promise((resolve) => {
+            if (this.halted !== null)
+                return resolve({ status: 'cancelled', reason: this.halted });
+            const entry = { job, hooks, resolve };
+            if (this.active.size < this.opts.concurrency)
+                this.start(entry);
+            else
+                this.queue.push(entry);
+        });
+    }
+    cancel(id, reason = 'cancelled') {
+        const i = this.queue.findIndex((q) => q.job.id === id);
+        if (i !== -1) {
+            const [q] = this.queue.splice(i, 1);
+            q.resolve({ status: 'cancelled', reason });
+            return true;
+        }
+        const a = this.active.get(id);
+        if (!a)
+            return false;
+        a.cancel(reason);
+        return true;
+    }
+    async cancelAll(reason) {
+        this.halted ??= reason;
+        this.drainQueue(reason);
+        for (const a of [...this.active.values()])
+            a.cancel(reason);
+        await this.idle();
+    }
+    async haltAll(reason) {
+        this.halted ??= reason;
+        this.drainQueue(reason);
+        for (const a of [...this.active.values()])
+            a.kill(reason);
+        await this.idle();
+    }
+    async idle() {
+        while (this.active.size > 0 || this.queue.length > 0 || this.stopping.size > 0) {
+            await Promise.race([new Promise((r) => setTimeout(r, 20)), ...this.stopping]);
+        }
+    }
+    drainQueue(reason) {
+        for (const q of this.queue.splice(0))
+            q.resolve({ status: 'cancelled', reason });
+    }
+    next() {
+        while (this.halted === null &&
+            this.queue.length > 0 &&
+            this.active.size < this.opts.concurrency) {
+            this.start(this.queue.shift());
+        }
+    }
+    start({ job, hooks, resolve }) {
+        const builtin = this.opts.builtin?.(job.worker) ?? null;
+        if (builtin) {
+            this.started(job, hooks);
+            return this.startBuiltin(job, builtin, resolve);
+        }
+        const found = resolveWorkerFile(this.opts.workersDir, job.worker);
+        if (!found.ok) {
+            resolve({ status: 'failed', reason: found.reason, message: found.message });
+            return;
+        }
+        const log = (message, level) => {
+            hooks.onLog?.(message, level);
+            this.opts.logger?.worker(job.worker, level, message, {
+                task_id: job.id,
+                ...(job.run !== undefined ? { run_no: job.run } : {})
+            });
+        };
+        let env;
+        try {
+            env = typeof this.opts.env === 'function' ? this.opts.env() : this.opts.env;
+        }
+        catch (err) {
+            resolve({ status: 'failed', reason: 'invalid_worker', message: err.message });
+            return;
+        }
+        const workerData = {
+            file: found.file,
+            input: job.input,
+            connectAttemptTimeoutMs: getDefaultAutoSelectFamilyAttemptTimeout()
+        };
+        this.started(job, hooks);
+        const thread = new Worker(THREAD_URL, {
+            workerData,
+            env,
+            execArgv: [],
+            stdout: true,
+            stderr: true,
+            name: `3minapi:${job.worker}`
+        });
+        pipeLines(thread.stdout, (line) => log(line, 'info'));
+        pipeLines(thread.stderr, (line) => log(line, 'warn'));
+        const startedAt = Date.now();
+        let settled = false;
+        let cancelReason = null;
+        let timer;
+        let grace;
+        const settle = (outcome) => {
+            if (settled)
+                return;
+            settled = true;
+            if (outcome.status === 'failed') {
+                outcome = {
+                    ...outcome,
+                    message: redactAndTruncate(outcome.message, ERROR_MESSAGE_MAX_CHARS),
+                    detail: redactAndTruncate(outcome.message, STORED_ERROR_MAX_CHARS)
+                };
+            }
+            clearTimeout(timer);
+            clearTimeout(grace);
+            this.active.delete(job.id);
+            this.stopThread(thread, job);
+            resolve(cancelReason !== null ? { status: 'cancelled', reason: cancelReason } : outcome);
+            this.next();
+        };
+        const arm = (ms) => {
+            clearTimeout(timer);
+            timer = setTimeout(() => settle({
+                status: 'failed',
+                reason: 'timeout',
+                message: `did not finish within ${ms} ms`
+            }), Math.max(0, ms - (Date.now() - startedAt)));
+        };
+        arm(DEFAULT_WORKER_TIMEOUT_MS);
+        thread.on('message', (msg) => {
+            switch (msg.type) {
+                case 'loaded':
+                    if (msg.timeoutMs !== null)
+                        arm(Math.min(msg.timeoutMs, WORKER_TIMEOUT_MAX_MS));
+                    return;
+                case 'log':
+                    log(msg.message, 'info');
+                    return;
+                case 'report': {
+                    const ack = () => {
+                        if (!settled)
+                            post(thread, { type: 'report-ack', id: msg.id });
+                    };
+                    Promise.resolve(hooks.onReport?.(msg.message)).then(ack, ack);
+                    return;
+                }
+                case 'done':
+                    return settle({ status: 'completed', resultJson: msg.json });
+                case 'fail':
+                    if (msg.stack)
+                        log(msg.stack, 'error');
+                    return settle({ status: 'failed', reason: msg.reason, message: msg.message });
+            }
+        });
+        thread.on('error', (err) => {
+            const stack = errorStack(err);
+            if (stack)
+                log(stack, 'error');
+            settle({
+                status: 'failed',
+                reason: 'handler_error',
+                message: `uncaught error: ${withoutHome(errorMessage(err), homedir())}`
+            });
+        });
+        thread.on('exit', (code) => settle({
+            status: 'failed',
+            reason: 'handler_error',
+            message: `the worker exited before returning a result (exit code ${code})`
+        }));
+        this.active.set(job.id, {
+            cancel: (reason) => {
+                if (settled || cancelReason !== null)
+                    return;
+                cancelReason = reason;
+                post(thread, { type: 'cancel' });
+                grace = setTimeout(() => settle({ status: 'cancelled', reason }), this.opts.cancelGraceMs ?? CANCEL_GRACE_MS);
+            },
+            kill: (reason) => {
+                cancelReason ??= reason;
+                settle({ status: 'cancelled', reason });
+            }
+        });
+    }
+    started(job, hooks) {
+        try {
+            hooks.onStart?.();
+        }
+        catch (err) {
+            this.opts.logger?.warn('task start hook failed', {
+                task_id: job.id,
+                message: err?.message ?? String(err)
+            });
+        }
+    }
+    startBuiltin(job, run, resolve) {
+        const controller = new AbortController();
+        let settled = false;
+        let cancelReason = null;
+        const settle = (outcome) => {
+            if (settled)
+                return;
+            settled = true;
+            if (outcome.status === 'failed') {
+                outcome = {
+                    ...outcome,
+                    message: redactAndTruncate(outcome.message, ERROR_MESSAGE_MAX_CHARS),
+                    detail: redactAndTruncate(outcome.message, STORED_ERROR_MAX_CHARS)
+                };
+            }
+            this.active.delete(job.id);
+            resolve(cancelReason !== null ? { status: 'cancelled', reason: cancelReason } : outcome);
+            this.next();
+        };
+        const stop = (reason) => {
+            if (settled)
+                return;
+            cancelReason ??= reason;
+            controller.abort();
+            settle({ status: 'cancelled', reason });
+        };
+        this.active.set(job.id, { cancel: stop, kill: stop });
+        run(job.input, controller.signal).then(settle, (err) => settle({
+            status: 'failed',
+            reason: 'handler_error',
+            message: `the built-in worker failed: ${err?.message ?? String(err)}`
+        }));
+    }
+    stopThread(thread, job) {
+        const wait = this.opts.terminateWaitMs ?? TERMINATE_WAIT_MS;
+        let timeout;
+        const done = Promise.race([
+            thread.terminate().then(() => true),
+            new Promise((r) => {
+                timeout = setTimeout(() => r(false), wait);
+            })
+        ]).then((stopped) => {
+            clearTimeout(timeout);
+            if (!stopped) {
+                this.opts.logger?.warn('worker thread did not stop; it may be blocked in native or synchronous code', {
+                    worker: job.worker,
+                    task_id: job.id
+                });
+            }
+        });
+        this.stopping.add(done);
+        void done.finally(() => this.stopping.delete(done));
+    }
+}
+function post(thread, msg) {
+    try {
+        thread.postMessage(msg);
+    }
+    catch {
+    }
+}
+function pipeLines(stream, onLine) {
+    if (!stream)
+        return;
+    createInterface({ input: stream, crlfDelay: Infinity }).on('line', onLine);
+}

package/dist/modules/workers/runtime/inspect.js

@@ -0,0 +1,53 @@
+import { getDefaultAutoSelectFamilyAttemptTimeout } from 'node:net';
+import { Worker } from 'node:worker_threads';
+import { ERROR_MESSAGE_MAX_CHARS } from "../../../core/domain/index.js";
+import { redactAndTruncate } from "../../../core/redact.js";
+import { THREAD_URL } from "./executor.js";
+export const INSPECT_TIMEOUT_MS = 10_000;
+export function inspectWorker(file, env, timeoutMs = INSPECT_TIMEOUT_MS) {
+    return new Promise((resolve) => {
+        const workerData = {
+            file,
+            input: null,
+            inspect: true,
+            connectAttemptTimeoutMs: getDefaultAutoSelectFamilyAttemptTimeout()
+        };
+        const thread = new Worker(THREAD_URL, {
+            workerData,
+            env,
+            execArgv: [],
+            stdout: true,
+            stderr: true
+        });
+        thread.stdout.resume();
+        thread.stderr.resume();
+        let settled = false;
+        const settle = (result) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            void thread.terminate();
+            resolve(result.ok
+                ? result
+                : { ...result, message: redactAndTruncate(result.message, ERROR_MESSAGE_MAX_CHARS) });
+        };
+        const timer = setTimeout(() => settle({
+            ok: false,
+            reason: 'invalid_worker',
+            message: `loading did not finish within ${timeoutMs} ms`
+        }), timeoutMs);
+        thread.on('message', (msg) => {
+            if (msg.type === 'meta')
+                settle({ ok: true, meta: msg.meta });
+            else if (msg.type === 'fail')
+                settle({ ok: false, reason: msg.reason, message: msg.message });
+        });
+        thread.on('error', (err) => settle({ ok: false, reason: 'invalid_worker', message: `cannot load: ${err.message}` }));
+        thread.on('exit', (code) => settle({
+            ok: false,
+            reason: 'invalid_worker',
+            message: `exited while loading (exit code ${code})`
+        }));
+    });
+}

package/dist/modules/workers/runtime/protocol.js

@@ -0,0 +1 @@
+export {};

package/dist/modules/workers/runtime/resolve.js

@@ -0,0 +1,30 @@
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { WORKER_EXTENSIONS } from "../../../core/domain/index.js";
+import { checkWorkerName, invalidNameMessage } from "../../../core/names.js";
+export function resolveWorkerFile(workersDir, name) {
+    const check = checkWorkerName(name);
+    if (!check.ok) {
+        return {
+            ok: false,
+            reason: 'invalid_worker',
+            message: invalidNameMessage(name, check.reason)
+        };
+    }
+    const found = WORKER_EXTENSIONS.map((ext) => path.join(workersDir, name + ext)).filter((f) => existsSync(f));
+    if (found.length === 0) {
+        return {
+            ok: false,
+            reason: 'worker_not_found',
+            message: `no worker file for "${name}" in the ${path.basename(workersDir)} folder (expected ${name}${WORKER_EXTENSIONS.join(` / ${name}`)})`
+        };
+    }
+    if (found.length > 1) {
+        return {
+            ok: false,
+            reason: 'invalid_worker',
+            message: `more than one file for worker "${name}": ${found.map((f) => path.basename(f)).join(', ')}. Keep only one.`
+        };
+    }
+    return { ok: true, file: found[0] };
+}

package/dist/modules/workers/runtime/schema.js

@@ -0,0 +1,85 @@
+export function validate(schema, value, at = 'input') {
+    if (!isObject(schema))
+        return null;
+    const s = schema;
+    if (s.type !== undefined) {
+        const types = Array.isArray(s.type) ? s.type : [s.type];
+        if (!types.some((t) => matchesType(t, value))) {
+            return `${at}: expected ${types.join(' or ')}, got ${typeOf(value)}`;
+        }
+    }
+    if (Array.isArray(s.enum) && !s.enum.some((option) => deepEqual(option, value))) {
+        return `${at}: must be one of ${s.enum.map((v) => JSON.stringify(v)).join(', ')}`;
+    }
+    if (isObject(value)) {
+        const obj = value;
+        if (Array.isArray(s.required)) {
+            for (const key of s.required) {
+                if (typeof key === 'string' && !(key in obj))
+                    return `${at}.${key}: required`;
+            }
+        }
+        if (isObject(s.properties)) {
+            for (const [key, sub] of Object.entries(s.properties)) {
+                if (!(key in obj))
+                    continue;
+                const reason = validate(sub, obj[key], `${at}.${key}`);
+                if (reason)
+                    return reason;
+            }
+        }
+    }
+    if (Array.isArray(value) && isObject(s.items)) {
+        for (let i = 0; i < value.length; i++) {
+            const reason = validate(s.items, value[i], `${at}[${i}]`);
+            if (reason)
+                return reason;
+        }
+    }
+    return null;
+}
+function matchesType(type, value) {
+    switch (type) {
+        case 'object':
+            return isObject(value);
+        case 'array':
+            return Array.isArray(value);
+        case 'string':
+            return typeof value === 'string';
+        case 'number':
+            return typeof value === 'number' && Number.isFinite(value);
+        case 'integer':
+            return Number.isInteger(value);
+        case 'boolean':
+            return typeof value === 'boolean';
+        case 'null':
+            return value === null;
+        default:
+            return true;
+    }
+}
+function typeOf(value) {
+    if (value === null)
+        return 'null';
+    if (Array.isArray(value))
+        return 'array';
+    if (Number.isInteger(value))
+        return 'integer';
+    return typeof value;
+}
+function isObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function deepEqual(a, b) {
+    if (a === b)
+        return true;
+    if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null)
+        return false;
+    if (Array.isArray(a) !== Array.isArray(b))
+        return false;
+    const ka = Object.keys(a);
+    const kb = Object.keys(b);
+    if (ka.length !== kb.length)
+        return false;
+    return ka.every((k) => deepEqual(a[k], b[k]));
+}

package/dist/modules/workers/runtime/thread.js

@@ -0,0 +1,117 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { realpathSync } from 'node:fs';
+import { setDefaultAutoSelectFamilyAttemptTimeout } from 'node:net';
+import { homedir } from 'node:os';
+import path from 'node:path';
+import { parentPort, workerData } from 'node:worker_threads';
+import { pathToFileURL } from 'node:url';
+import { errorMessage, errorStack, withoutHome } from "./error-chain.js";
+import { validate } from "./schema.js";
+const port = parentPort;
+const { file, input, inspect, connectAttemptTimeoutMs } = workerData;
+if (connectAttemptTimeoutMs !== undefined)
+    setDefaultAutoSelectFamilyAttemptTimeout(connectAttemptTimeoutMs);
+const controller = new AbortController();
+const pendingReports = new Map();
+let nextReportId = 1;
+function post(msg) {
+    port.postMessage(msg);
+}
+port.on('message', (msg) => {
+    if (msg.type === 'cancel')
+        controller.abort(new Error('task cancelled'));
+    else if (msg.type === 'report-ack') {
+        pendingReports.get(msg.id)?.();
+        pendingReports.delete(msg.id);
+    }
+});
+const ctx = Object.freeze({
+    log(message) {
+        post({ type: 'log', message: String(message) });
+    },
+    report(message) {
+        const id = nextReportId++;
+        return new Promise((resolve) => {
+            pendingReports.set(id, resolve);
+            post({ type: 'report', id, message: String(message) });
+        });
+    },
+    signal: controller.signal,
+    env: process.env
+});
+function withoutPath(message, file) {
+    let real = file;
+    try {
+        real = realpathSync(file);
+    }
+    catch {
+    }
+    const forms = [...new Set([real, file].flatMap((p) => [pathToFileURL(p).href, p]))].sort((a, b) => b.length - a.length);
+    let out = message;
+    for (const form of forms)
+        out = out.split(form).join(path.basename(file));
+    return out;
+}
+function fail(reason, message, stack) {
+    post({ type: 'fail', reason, message, stack });
+}
+async function main() {
+    let mod;
+    try {
+        mod = (await import(__rewriteRelativeImportExtension(pathToFileURL(file).href)));
+    }
+    catch (err) {
+        const e = err;
+        if (e?.code === 'ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX' ||
+            e?.code === 'ERR_INVALID_TYPESCRIPT_SYNTAX') {
+            return fail('unsupported_syntax', `${e.message.split('\n')[0]} — this TypeScript syntax cannot run without a build step. ` +
+                'Use plain type annotations: replace enums with const objects and avoid namespaces and constructor parameter properties, or write the worker in JavaScript.');
+        }
+        return fail('invalid_worker', `cannot load ${path.basename(file)}: ${withoutPath(e?.message ?? String(err), file)}`);
+    }
+    const handler = mod.handler;
+    if (typeof handler !== 'function') {
+        return fail('invalid_worker', `${path.basename(file)} must export an async function named "handler"`);
+    }
+    const raw = mod.timeoutMs;
+    const timeoutMs = typeof raw === 'number' && Number.isFinite(raw) && raw > 0 ? raw : null;
+    if (inspect) {
+        const description = mod.description;
+        return post({
+            type: 'meta',
+            meta: {
+                description: typeof description === 'string' ? description : null,
+                timeoutMs,
+                hasInputSchema: typeof mod.inputSchema === 'object' && mod.inputSchema !== null
+            }
+        });
+    }
+    post({ type: 'loaded', timeoutMs });
+    const invalid = validate(mod.inputSchema, input);
+    if (invalid)
+        return fail('invalid_input', `input does not match the worker's inputSchema — ${invalid}`);
+    let result;
+    try {
+        result = await handler(input, ctx);
+    }
+    catch (err) {
+        const message = withoutHome(withoutPath(errorMessage(err), file), homedir());
+        return fail('handler_error', message, errorStack(err));
+    }
+    let json;
+    try {
+        json = JSON.stringify(result ?? null);
+    }
+    catch (err) {
+        return fail('handler_error', `the handler's return value is not JSON-serializable: ${err.message}`);
+    }
+    post({ type: 'done', json: json ?? 'null' });
+}
+await main();