@3minapi/cli 0.0.0-stage → 0.22.1

package/dist/modules/workers/start.js

@@ -0,0 +1,265 @@
+import path from 'node:path';
+import { ApiClient } from "../../core/api.js";
+import { getOrCreateLocalServerToken } from "../../core/credentials.js";
+import { BIN, CliError, LIVE_RETRY_MS, LOCAL_SERVER_TOKEN_FILE, StopError, isReservedWorkerName } from "../../core/domain/index.js";
+import { recordStop, stopHint } from "../../core/host/session.js";
+import { formatLocal } from "../../core/time.js";
+import { builtinResultsSpec, hostChangedWarning, resolveHostIdentity } from "./builtin/host-identity.js";
+import { resultsWorker } from "./builtin/results-worker.js";
+import { startLocalServer } from "./local-server.js";
+import { WorkersModule } from "./module.js";
+import { ResultStore } from "./results/store.js";
+import { publicKeyEnv, workerEnv } from "./runtime/env.js";
+import { Executor } from "./runtime/executor.js";
+import { resyncWorkers, syncWorkers } from "./sync.js";
+import { TaskRunner } from "./task-runner.js";
+import { workerFiles } from "./workers.js";
+export async function startWorkers(opts) {
+    const { ctx, session, api, lock, root, workersDir, serve, isPublic } = opts;
+    const { out } = ctx;
+    const cfg = session.config.values;
+    const results = openResults(ctx, session, root);
+    let host = null;
+    try {
+        if (results && hasSyncableFiles(workersDir)) {
+            host = await lookupHost(ctx, session, api, opts.token);
+        }
+        if (results && host)
+            results.setHost(host.id);
+    }
+    catch (err) {
+        results?.close();
+        throw err;
+    }
+    const builtin = host ? builtinResultsSpec(host) : null;
+    let workers;
+    const isBuiltin = (name) => builtin !== null && name === workers?.builtin;
+    const readBuiltin = host ? resultsWorker(root, host.id) : null;
+    const executor = new Executor({
+        workersDir,
+        concurrency: cfg.concurrency,
+        env: () => workerEnv(root, ctx.env),
+        logger: session.logger,
+        builtin: (name) => (isBuiltin(name) ? readBuiltin : null)
+    });
+    const syncOpts = {
+        root,
+        workersDir,
+        isPublic: serve && isPublic,
+        intervalMs: cfg.receiveMode === 'live' ? LIVE_RETRY_MS : cfg.pollIntervalMs,
+        signal: opts.signal,
+        builtin,
+        onLive: opts.onLive,
+        facts: {
+            receiveMode: cfg.receiveMode,
+            pollIntervalMs: cfg.pollIntervalMs,
+            concurrency: cfg.concurrency,
+            cliVersion: ctx.version,
+            platform: `${ctx.platform}-${process.arch}`
+        }
+    };
+    let synced;
+    try {
+        synced = await syncWorkers(ctx, session, api, syncOpts);
+    }
+    catch (err) {
+        results?.close();
+        throw err;
+    }
+    if (!synced) {
+        results?.close();
+        return null;
+    }
+    workers = synced.workers;
+    const runner = new TaskRunner({
+        executor,
+        db: session.db,
+        logger: session.logger,
+        now: ctx.now,
+        onStop: (err) => opts.onStop(err),
+        onEvent: (line) => out.info(`${out.err.dim(formatLocal(ctx.now()))}  ${line}`),
+        results,
+        isBuiltin,
+        reader: () => (workers.builtinActive ? workers.builtin : null),
+        repeatReporter: api,
+        onSlotFree: () => module.notifySlotFree()
+    });
+    const module = new WorkersModule({
+        api,
+        runner,
+        logger: session.logger,
+        workers,
+        onWorkersUnregistered: (names) => {
+            for (const name of names) {
+                out.warn(`${out.err.dim(formatLocal(ctx.now()))}  ${name} could not load and is not registered on the server — ` +
+                    `fix the file and run '${BIN} reload'.`);
+            }
+        },
+        resync: () => resyncWorkers(ctx, session, api, syncOpts, workers, lock),
+        onWorkersDropped: (names) => {
+            for (const name of names) {
+                out.info(`${out.err.dim(formatLocal(ctx.now()))}  ${name} was deleted with '${BIN} worker delete' — it is off for this run.`);
+            }
+        },
+        onWorkersDeleted: (names) => {
+            for (const name of names) {
+                if (name === workers.builtin) {
+                    out.warn(`${out.err.dim(formatLocal(ctx.now()))}  the built-in results reader ${name} was deleted on the server — ` +
+                        `remote AIs cannot read saved results until '${BIN} reload'.`);
+                    continue;
+                }
+                out.warn(`${out.err.dim(formatLocal(ctx.now()))}  ${name} was deleted on the server — it is off for this run. ` +
+                    `If its file is still there, run '${BIN} reload' to register it again.`);
+            }
+        }
+    });
+    let server = null;
+    return {
+        module,
+        quota: synced.quota,
+        live: synced.live,
+        recover: () => runner.recover(api),
+        async serve() {
+            if (!serve)
+                return;
+            server = await startLocalServer({
+                port: cfg.localServerPort,
+                public: isPublic,
+                token: getOrCreateLocalServerToken(ctx.dirs.config),
+                workersDir,
+                env: () => publicKeyEnv(root, ctx.env),
+                version: ctx.version,
+                inactive: (name) => inactiveMessage(workers, name),
+                onTask: (task, reporter) => void runner.handle(task, reporter, { ack: false }),
+                reporterFor: (key) => pushReporter(api, key, cfg.apiUrl, opts.userAgent),
+                onRejected: rejectionLogger(session.logger),
+                onError: (err) => session.logger.error('local server request failed', {
+                    message: err?.stack ?? String(err)
+                })
+            }).catch((err) => {
+                throw new CliError(`cannot open the local server: ${err.message}`);
+            });
+        },
+        serverPort: () => server?.port ?? null,
+        bannerSubject: () => `workers in ${displayPath(ctx.cwd, workersDir)}`,
+        bannerLines: () => bannerLines(ctx, server, isPublic, workers, cfg.receiveMode),
+        endMessage: (ended) => {
+            const gone = ended.deletedWorkers;
+            return ((gone.length > 0
+                ? `no workers left to run — ${gone.join(', ')} ${gone.length === 1 ? 'was' : 'were'} deleted on the server. Stopped.\n`
+                : `no workers left to run — none of the workers in ${workersDir} is registered (see above). Stopped.\n`) +
+                `Run '${BIN} start' again to register the workers in ${workersDir}.`);
+        },
+        async close() {
+            await server?.close();
+            results?.close();
+        }
+    };
+}
+async function lookupHost(ctx, session, api, token) {
+    try {
+        const { host, error, previousHostId } = await resolveHostIdentity(session.db, api, token, ctx.now());
+        if (host && previousHostId) {
+            ctx.out.warn(hostChangedWarning(previousHostId, host));
+            session.logger.warn('host changed', { previous_host_id: previousHostId, host_id: host.id });
+        }
+        if (host)
+            return host;
+        ctx.out.warn(`could not look up this host (${error}) — the built-in results reader is off this run; remote AIs cannot read saved results.`);
+        session.logger.warn('host lookup failed; built-in reader off', { message: error });
+        return null;
+    }
+    catch (err) {
+        if (err instanceof StopError) {
+            recordStop(session, err, ctx.now());
+            throw new StopError(err.reason, `${err.message}\n${stopHint(err.reason)}`, err.resetAt);
+        }
+        throw err;
+    }
+}
+function hasSyncableFiles(workersDir) {
+    try {
+        return [...workerFiles(workersDir).keys()].some((name) => !isReservedWorkerName(name));
+    }
+    catch {
+        return true;
+    }
+}
+function openResults(ctx, session, root) {
+    try {
+        const { store, warnings } = ResultStore.open(root);
+        for (const w of warnings) {
+            ctx.out.warn(w);
+            session.logger.warn(w);
+        }
+        return store;
+    }
+    catch (err) {
+        const message = err.message;
+        ctx.out.warn(`cannot open data.db to save worker results: ${message}. Results are not saved this run, and results over 100 KiB fail.`);
+        session.logger.warn('cannot open data.db; results are not saved', { message });
+        return null;
+    }
+}
+export function rejectionLogger(logger) {
+    const last = new Map();
+    return (worker, reason) => {
+        const now = Date.now();
+        const key = `${worker}\n${reason}`;
+        if (now - (last.get(key) ?? 0) < 60_000)
+            return;
+        if (last.size >= 1000)
+            last.clear();
+        last.set(key, now);
+        logger.warn('public request rejected', { worker, reason });
+    };
+}
+function bannerLines(ctx, server, isPublic, workers, mode) {
+    const lines = [];
+    if (server) {
+        lines.push(isPublic
+            ? `public worker server on port ${server.port} (all interfaces, signed requests only)`
+            : `HTTP server on ${server.url} (token in ${path.join(ctx.dirs.config, LOCAL_SERVER_TOKEN_FILE)})`);
+    }
+    if (workers.pullActiveCount === 0) {
+        lines.push(ctx.out.err.yellow(mode === 'live'
+            ? 'no local workers are active — nothing will be fetched, but each live connect still uses one call'
+            : 'no local workers are active — polls will find nothing to run but still use one call each'));
+    }
+    return lines;
+}
+function inactiveMessage(workers, name) {
+    switch (workers.inactiveReason(name)) {
+        case null:
+            return null;
+        case 'rejected':
+            return `worker "${name}" was not registered with 3Min API (see the start output). Fix it and run '${BIN} reload'.`;
+        case 'deleted_on_server':
+            return `worker "${name}" was deleted on the server. If its file is still there, run '${BIN} reload' to register it again.`;
+        case 'not_on_server':
+            return `worker "${name}" is not registered for this run. Run '${BIN} reload' to register it.`;
+    }
+}
+function pushReporter(main, key, apiUrl, userAgent) {
+    const client = new ApiClient({ apiUrl, token: key, userAgent });
+    return {
+        async report(url, body) {
+            if (main.halted)
+                throw main.halted;
+            try {
+                return await client.report(url, body);
+            }
+            catch (err) {
+                if (err instanceof StopError)
+                    main.halt(err);
+                throw err;
+            }
+        }
+    };
+}
+function displayPath(cwd, dir) {
+    const rel = path.relative(cwd, dir);
+    if (rel === '')
+        return '.';
+    return rel.startsWith('..') || path.isAbsolute(rel) ? dir : `.${path.sep}${rel}`;
+}

package/dist/modules/workers/sync.js

@@ -0,0 +1,241 @@
+import { ApiError } from "../../core/api.js";
+import { BIN, CliError, StopError, SYNC_WORKERS_MAX, WORKER_KEY_ENV, WORKER_SECRET_ENV, isReservedWorkerName } from "../../core/domain/index.js";
+import { rememberQuota, clearRunnerRecords, recordEnd, recordStop, stopHint } from "../../core/host/session.js";
+import { formatCount } from "../../core/text.js";
+import { describeAt, formatInterval, formatLocal, sleep } from "../../core/time.js";
+import { publicKeyEnv, workerEnv } from "./runtime/env.js";
+import { WorkerSet } from "./worker-set.js";
+import { collectWorkers } from "./workers.js";
+export async function syncWorkers(ctx, session, api, opts) {
+    const { out } = ctx;
+    const collected = await collectForSync(ctx, opts);
+    let res;
+    while (!opts.signal.aborted) {
+        try {
+            res = await api.syncWorkers(syncBody(collected, builtinFor(collected, opts), opts.facts));
+            break;
+        }
+        catch (err) {
+            if (err instanceof StopError) {
+                recordStop(session, err, ctx.now());
+                throw new StopError(err.reason, `${err.message}\n${stopHint(err.reason)}`, err.resetAt);
+            }
+            if (!(err instanceof ApiError))
+                throw err;
+            if (err.code === 'invalid_request') {
+                session.logger.error('the server rejected the worker sync request (a CLI bug)', {
+                    message: err.message
+                });
+                throw new CliError(`the server rejected the list of workers: ${err.message}\n` +
+                    "This is likely a bug in 3minapi. Please report it with the output of '3minapi worker list'.");
+            }
+            else {
+                session.logger.warn('worker sync failed; trying again at the next interval', {
+                    status: err.status,
+                    message: err.message
+                });
+            }
+            out.warn(`could not register the workers: ${err.message}\nTrying again in ${formatInterval(opts.intervalMs)}.`);
+        }
+        await sleep(opts.intervalMs, opts.signal);
+    }
+    if (!res)
+        return null;
+    clearRunnerRecords(session);
+    const workers = new WorkerSet(localNames(collected), builtinFor(collected, opts)?.name ?? null);
+    const rejected = workers.applySync(res, syncContext(ctx, opts), collected.unsent.map((w) => w.name));
+    announceSync(ctx, session, res, workers, rejected, collected, 'workers synced');
+    if (workers.activeCount === 0) {
+        recordEnd(session, 'no_active_workers', [], ctx.now());
+        throw new CliError(collected.local.length === 0
+            ? `no workers in ${opts.workersDir}. Create one with '${BIN} worker new <name>', then run start again.`
+            : `none of the workers in ${opts.workersDir} could be registered (see above). Fix them and run start again.`);
+    }
+    return { workers, quota: res.quota, live: res.live ?? null };
+}
+export async function resyncWorkers(ctx, session, api, opts, workers, lock) {
+    const { out } = ctx;
+    const startedAt = Date.now();
+    lock.writeReload({ state: 'running', startedAt });
+    out.info(`${out.err.dim(formatLocal(ctx.now()))}  reloading worker files (3minapi reload)...`);
+    let result;
+    try {
+        const collected = await collectForSync(ctx, opts);
+        const res = await api.syncWorkers(syncBody(collected, builtinFor(collected, opts), opts.facts));
+        const rejected = workers.resync(localNames(collected), res, syncContext(ctx, opts), collected.unsent.map((w) => w.name), builtinFor(collected, opts)?.name ?? null);
+        announceSync(ctx, session, res, workers, rejected, collected, 'workers resynced');
+        if (res.live)
+            opts.onLive?.(res.live);
+        const names = (r) => res.workers.filter((w) => w.result === r).map((w) => w.name);
+        result = {
+            ok: true,
+            created: names('created'),
+            updated: names('updated'),
+            unchanged: names('unchanged'),
+            rejected: [
+                ...rejected.map((r) => ({ name: r.name, code: r.code, error: r.error })),
+                ...collected.caseClashes.map(({ worker, others }) => ({
+                    name: worker.name,
+                    code: 'name_case_conflict',
+                    error: `differs from ${others.join(', ')} only by letter case`
+                })),
+                ...collected.reserved.map((w) => ({
+                    name: w.name,
+                    code: 'invalid_worker_name',
+                    error: RESERVED_FILE_MESSAGE
+                }))
+            ],
+            notLoaded: collected.unsent.map((w) => ({
+                name: w.name,
+                message: w.message ?? 'could not load'
+            })),
+            active: workers.activeNames(),
+            quota: res.quota
+        };
+    }
+    catch (err) {
+        if (err instanceof StopError) {
+            lock.writeReload({
+                state: 'done',
+                startedAt,
+                finishedAt: Date.now(),
+                result: { ok: false, error: err.message, stopped: err.reason }
+            });
+            throw err;
+        }
+        const message = err instanceof Error ? err.message : String(err);
+        session.logger.warn('worker reload failed; keeping the current workers', { message });
+        out.warn(`reload failed: ${message} — keeping the current workers.`);
+        result = { ok: false, error: message };
+    }
+    lock.writeReload({ state: 'done', startedAt, finishedAt: Date.now(), result });
+}
+const RESERVED_FILE_MESSAGE = 'names starting with "_" are reserved for built-in workers';
+async function collectForSync(ctx, opts) {
+    const local = await collectWorkers(opts.workersDir, workerEnv(opts.root, ctx.env));
+    if (local.length > SYNC_WORKERS_MAX) {
+        throw new CliError(`${opts.workersDir} has ${local.length} workers — at most ${SYNC_WORKERS_MAX} can be registered from one host.`);
+    }
+    const reserved = local.filter((w) => isReservedWorkerName(w.name));
+    const { sent, caseClashes } = withoutCaseClashes(local.filter((w) => (w.problem === null || w.problem === 'invalid_name') && !isReservedWorkerName(w.name)));
+    const unsent = local.filter((w) => w.problem === 'load_failed' || w.problem === 'duplicate');
+    if (opts.builtin && sent.length >= SYNC_WORKERS_MAX) {
+        ctx.out.warn(`${opts.workersDir} has ${sent.length} workers to register — the most one host can register — so the built-in results reader ` +
+            'is left out this run; remote AIs cannot read saved results.');
+    }
+    return { local, sent, unsent, caseClashes, reserved };
+}
+function localNames(collected) {
+    return collected.local.filter((w) => !isReservedWorkerName(w.name)).map((w) => w.name);
+}
+function builtinFor(collected, opts) {
+    return collected.sent.length > 0 && collected.sent.length < SYNC_WORKERS_MAX
+        ? opts.builtin
+        : null;
+}
+function syncBody(collected, builtin, facts) {
+    const workers = collected.sent.map((w) => ({
+        name: w.name,
+        description: w.meta?.description ?? null
+    }));
+    if (builtin)
+        workers.push({ name: builtin.name, description: builtin.description });
+    return {
+        workers,
+        receive_mode: facts.receiveMode,
+        poll_interval_ms: facts.receiveMode === 'live' ? null : facts.pollIntervalMs,
+        concurrency: facts.concurrency,
+        cli_version: facts.cliVersion,
+        platform: facts.platform
+    };
+}
+function syncContext(ctx, opts) {
+    const publicKeys = opts.isPublic ? publicKeyEnv(opts.root, ctx.env) : {};
+    return {
+        public: opts.isPublic,
+        hasPushKeys: (name) => Boolean(publicKeys[WORKER_SECRET_ENV(name)] && publicKeys[WORKER_KEY_ENV(name)])
+    };
+}
+function announceSync(ctx, session, res, workers, rejected, collected, event) {
+    const { out } = ctx;
+    rememberQuota(session, res.quota, ctx.now());
+    session.logger.info(event, {
+        workers: res.workers,
+        rejected: rejected.map((r) => ({ name: r.name, code: r.code })),
+        push_served: workers.pushServedNames(),
+        not_sent: collected.local.filter((w) => !collected.sent.includes(w)).map((w) => w.name),
+        local: collected.local.map((w) => ({
+            name: w.name,
+            description: w.meta?.description ?? null,
+            timeout_ms: w.meta?.timeoutMs ?? null,
+            input_schema: w.meta?.hasInputSchema ?? false,
+            problem: w.problem,
+            message: w.message
+        }))
+    });
+    for (const r of rejected) {
+        if (r.name === workers.builtin) {
+            out.warn(`the built-in results reader ${r.name} was not registered: ${r.error} Remote AIs cannot read saved results this run.`);
+            continue;
+        }
+        out.warn(`${r.error} ${r.name} is off for this run — fix it and run '${BIN} reload'.`);
+    }
+    for (const w of collected.reserved) {
+        out.warn(`${w.name}: ${RESERVED_FILE_MESSAGE}, so it is not registered — rename the file and run '${BIN} reload'.`);
+    }
+    for (const { worker, others } of collected.caseClashes) {
+        out.warn(`${worker.name}: differs from ${others.join(', ')} only by letter case. Worker names ignore case, ` +
+            `so it is not registered — rename the file and run '${BIN} reload'.`);
+    }
+    for (const w of collected.unsent) {
+        out.warn(`${w.name}: ${w.message}. It was left out of the registration; if an earlier run registered it, ` +
+            'its tasks fail until the file loads.');
+    }
+    const served = workers.pushServedNames();
+    if (served.length > 0) {
+        out.info(`public   ${served.join(', ')} — served as public workers on this machine (--public)`);
+    }
+    out.info(workersLine(ctx, res, workers));
+}
+function withoutCaseClashes(workers) {
+    const byKey = new Map();
+    for (const w of workers) {
+        const key = w.name.toLowerCase();
+        byKey.set(key, [...(byKey.get(key) ?? []), w]);
+    }
+    const sent = [];
+    const caseClashes = [];
+    for (const group of byKey.values()) {
+        if (group.length === 1) {
+            sent.push(group[0]);
+            continue;
+        }
+        for (const w of group) {
+            if (w.problem === null)
+                sent.push(w);
+            else
+                caseClashes.push({
+                    worker: w,
+                    others: group.filter((o) => o !== w).map((o) => o.name)
+                });
+        }
+    }
+    return { sent, caseClashes };
+}
+function workersLine(ctx, res, workers) {
+    const active = workers.activeNames();
+    const count = (result) => res.workers.filter((w) => w.result === result).length;
+    const changes = [
+        count('created') > 0 ? `${count('created')} new` : null,
+        count('updated') > 0 ? `${count('updated')} updated` : null
+    ].filter(Boolean);
+    const { remaining, limit, reset_at } = res.quota;
+    return [
+        `workers  ${active.length} active (${active.join(', ') || 'none'})` +
+            (changes.length > 0 ? ctx.out.err.dim(` · registered: ${changes.join(', ')}`) : ''),
+        ...(workers.builtinActive
+            ? [`reader   ${workers.builtin} — remote AIs can query the saved results through it`]
+            : []),
+        `quota    ${formatCount(remaining)} of ${formatCount(limit)} calls left · resets ${describeAt(reset_at, ctx.now())}`
+    ].join('\n');
+}