@method-ai-hq/sdk 0.0.0-stage → 0.14.2

package/dist/packages/sdk/src/worker.js

@@ -0,0 +1,443 @@
+import { spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { closeSync, existsSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { hostname } from "node:os";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+import { gunzipSync, gzipSync } from "node:zlib";
+import { writePrivateJson } from "./files.js";
+import { DEFAULT_SERVER } from "./method-client.js";
+import { methodCache } from "./prepare.js";
+import { open, openBytes, seal, sealBytes } from "./run-data.js";
+const sleep = (ms, signal) => new Promise(done => {
+    const timer = setTimeout(done, ms);
+    signal?.addEventListener("abort", () => { clearTimeout(timer); done(); }, { once: true });
+});
+export class ProductionWorker {
+    api;
+    options;
+    id;
+    done = Promise.resolve();
+    stopping = new AbortController();
+    active = new Map();
+    cancelActive = false;
+    constructor(api, options = {}) {
+        this.api = api;
+        this.options = options;
+        this.id = options.workerId ?? `${hostname()}-${process.pid}-${randomBytes(3).toString("hex")}`;
+    }
+    get running() { return !this.stopping.signal.aborted; }
+    start() {
+        const slots = Math.max(1, Math.min(32, Math.floor(this.options.concurrency ?? 1)));
+        this.done = Promise.all(Array.from({ length: slots }, () => this.loop())).then(() => undefined);
+        return this;
+    }
+    /** Stop claiming. Running runs finish, unless cancel is true: then they stop and go back to the queue. */
+    async stop(options = {}) {
+        this.stopping.abort();
+        if (options.cancel) {
+            this.cancelActive = true;
+            for (const controller of this.active.values())
+                controller.abort();
+        }
+        await this.done;
+    }
+    report(error) {
+        const value = error instanceof Error ? error : new Error(String(error));
+        if (this.options.onError)
+            this.options.onError(value);
+        else
+            process.stderr.write(`Method worker: ${value.message}\n`);
+    }
+    async loop() {
+        let backoff = 1_000;
+        const waitMs = this.options.waitMs ?? 25_000;
+        while (!this.stopping.signal.aborted) {
+            let claim;
+            try {
+                const methods = [...this.options.methods ?? []];
+                const response = await this.api.request("/v1/worker/claim", "POST", { worker_id: this.id, wait_ms: waitMs, ...(methods.length ? { methods } : {}) }, waitMs + 15_000, this.stopping.signal);
+                backoff = 1_000;
+                if (response.status === 204) {
+                    if (!waitMs)
+                        await sleep(3_000, this.stopping.signal);
+                    continue;
+                }
+                claim = response.body;
+            }
+            catch (error) {
+                if (this.stopping.signal.aborted)
+                    break;
+                // A key without the worker scope, or a revoked key, cannot recover by retrying.
+                if (error?.status === 401 || error?.status === 403) {
+                    this.report(error);
+                    this.stopping.abort();
+                    break;
+                }
+                this.report(error);
+                await sleep(backoff, this.stopping.signal);
+                backoff = Math.min(backoff * 2, 30_000);
+                continue;
+            }
+            await this.handle(claim);
+        }
+    }
+    async handle(claim) {
+        const controller = new AbortController(), directory = resolve(this.options.directory ?? join(methodCache(), "worker"));
+        this.active.set(claim.run_id, controller);
+        let lost = false, cancelled = false, runDir = join(directory, claim.run_id);
+        const path = `/v1/worker/runs/${encodeURIComponent(claim.run_id)}`;
+        const heartbeat = setInterval(async () => {
+            try {
+                const response = await this.api.request(`${path}/heartbeat`, "POST", { lease_id: claim.lease_id });
+                if (response.body.cancel_requested && !cancelled) {
+                    cancelled = true;
+                    controller.abort();
+                }
+            }
+            catch (error) {
+                // Another worker holds the run now. Stop this copy and do not complete it.
+                if (error?.status === 409) {
+                    lost = true;
+                    controller.abort();
+                }
+                else
+                    this.report(error);
+            }
+        }, Math.max(1_000, Math.floor(claim.lease_ms / 3)));
+        // The run itself keeps the process alive; the heartbeat alone must not.
+        heartbeat.unref();
+        let outcome;
+        try {
+            const opened = this.open(claim);
+            // A run that waited continues its folder. A worker that does not have the folder gets it from the server.
+            if (claim.resume && !existsSync(join(runDir, "checkpoint.json"))) {
+                if (!claim.snapshot)
+                    throw Object.assign(Error("The run folder of this waiting run is not on this worker and was not saved."), { code: "run_folder_missing" });
+                runDir = await this.restore(claim, runDir);
+            }
+            outcome = await (this.options.execute ?? cliExecutor())(opened, { directory, runDir, signal: controller.signal, api: this.api });
+        }
+        catch (error) {
+            const final = ["wrong_key", "run_folder_missing", "invalid_sealed"].includes(error?.code);
+            outcome = { status: "failed", error: { code: error?.code ?? "worker_failed", message: String(error?.message ?? error) }, retryable: !final };
+        }
+        finally {
+            clearInterval(heartbeat);
+            this.active.delete(claim.run_id);
+        }
+        if (lost)
+            return;
+        if (outcome.status === "waiting" && !cancelled && !this.cancelActive) {
+            await this.wait(claim, runDir, outcome);
+            return;
+        }
+        if (outcome.status === "waiting")
+            outcome = cancelled ? { status: "cancelled", error: { code: "cancelled", message: "The run was cancelled." } }
+                : { status: "failed", error: { code: "worker_stopped", message: "The worker stopped." }, retryable: true };
+        if (cancelled && outcome.status !== "succeeded")
+            outcome = { status: "cancelled", error: { code: "cancelled", message: "The run was cancelled." } };
+        else if (this.cancelActive && outcome.status !== "succeeded")
+            outcome = { status: "failed", error: { code: "worker_stopped", message: "The worker stopped." }, retryable: true };
+        const done = outcome;
+        const finished = { run_id: claim.run_id, method_id: claim.method_id, version_id: claim.version_id, status: done.status,
+            run_data: claim.run_data, run_dir: runDir,
+            ...(done.result !== undefined ? { result: done.result } : {}), ...(done.error ? { error: done.error } : {}) };
+        // A requeued run is not finished; it runs again.
+        if (!(done.status === "failed" && done.retryable && claim.attempt < 3)) {
+            try {
+                await this.options.onResult?.(finished);
+            }
+            catch (error) {
+                this.report(error);
+            }
+        }
+        const body = { lease_id: claim.lease_id, status: done.status, ...(done.error ? { error: done.error } : {}),
+            ...(done.retryable ? { retryable: true } : {}),
+            // A device run's result stays on this computer.
+            ...(claim.run_data === "account" && done.result !== undefined ? { result: done.result } : {}) };
+        await this.send(`${path}/complete`, body);
+    }
+    async send(path, body) {
+        for (let attempt = 0;; attempt++) {
+            try {
+                await this.api.request(path, "POST", body);
+                return true;
+            }
+            catch (error) {
+                if (error?.status && error.status < 500 && error.status !== 429 || attempt >= 4) {
+                    this.report(error);
+                    return false;
+                }
+                await sleep(1_000 * 2 ** attempt);
+            }
+        }
+    }
+    /** Open sealed inputs and answers with this worker's credential. */
+    open(claim) {
+        const key = this.api.apiKey;
+        const inputs = claim.sealed_inputs ? open(key, "inputs", claim.method_id, claim.sealed_inputs) : claim.inputs;
+        const answers = Object.fromEntries(Object.entries(claim.answers ?? {}).map(([step, answer]) => [step, "sealed" in answer ? { outputs: open(key, "answer", claim.method_id, answer.sealed) } : answer]));
+        const { sealed_inputs: _sealed, ...rest } = claim;
+        return { ...rest, inputs, answers };
+    }
+    /** Save the run folder, report the question, and let go of the run. */
+    async wait(claim, runDir, outcome) {
+        const path = `/v1/worker/runs/${encodeURIComponent(claim.run_id)}`;
+        try {
+            const packed = packFolder(runDir);
+            const bytes = claim.run_data === "device" ? sealBytes(this.api.apiKey, claim.method_id, packed) : packed;
+            await this.api.bytes(`${path}/snapshot`, "PUT", bytes, { "method-lease": claim.lease_id });
+        }
+        catch (error) {
+            // Without a saved folder only this worker can continue the run. Report the question anyway.
+            this.report(new Error(`The run folder was not saved, so only this worker can continue run ${claim.run_id}: ${error.message}`));
+        }
+        const asked = claim.run_data === "device"
+            ? { sealed_question: seal(this.api.apiKey, "question", claim.method_id, { question: outcome.question.question, form: outcome.question.form }) }
+            : { question: outcome.question.question, form: outcome.question.form };
+        const sent = await this.send(`${path}/wait`, { lease_id: claim.lease_id, step: outcome.question.step, ...asked,
+            ...(outcome.timeoutMs ? { timeout_ms: outcome.timeoutMs } : {}) });
+        if (sent)
+            try {
+                await this.options.onWaiting?.({ run_id: claim.run_id, method_id: claim.method_id, question: outcome.question });
+            }
+            catch (error) {
+                this.report(error);
+            }
+    }
+    /** Get the saved run folder. It goes back to the path it had, when this computer can write there, so the run can resume. */
+    async restore(claim, fallback) {
+        const response = await this.api.bytes(`/v1/worker/runs/${encodeURIComponent(claim.run_id)}/snapshot`, "GET", undefined, { "method-lease": claim.lease_id });
+        const packed = claim.run_data === "device" ? openBytes(this.api.apiKey, claim.method_id, response) : response;
+        return unpackFolder(packed, fallback);
+    }
+}
+const skipped = (path) => /(^|\/)(source|node_modules|\.venv|__pycache__)(\/|$)/.test(path) || /\.log$/.test(path);
+/** The run folder as gzip JSON: its own path and each file. */
+export function packFolder(dir) {
+    const files = [];
+    const visit = (folder) => {
+        for (const name of readdirSync(folder)) {
+            const full = join(folder, name), rel = relative(dir, full).split(sep).join("/");
+            if (skipped(rel))
+                continue;
+            const stat = lstatSync(full);
+            if (stat.isDirectory())
+                visit(full);
+            else if (stat.isFile()) {
+                files.push({ path: rel, data: readFileSync(full).toString("base64") });
+            }
+        }
+    };
+    visit(dir);
+    return gzipSync(JSON.stringify({ schema: "method-run-folder/1", dir, files }));
+}
+export function unpackFolder(packed, fallback) {
+    const folder = JSON.parse(gunzipSync(packed).toString("utf8"));
+    if (folder.schema !== "method-run-folder/1")
+        throw Error("Unknown run folder format.");
+    let target = fallback;
+    // The run's saved configuration names its folder, so the original path lets the runtime resume it.
+    try {
+        mkdirSync(folder.dir, { recursive: true, mode: 0o700 });
+        target = folder.dir;
+    }
+    catch {
+        mkdirSync(fallback, { recursive: true, mode: 0o700 });
+    }
+    for (const file of folder.files) {
+        const path = resolve(target, file.path);
+        if (!path.startsWith(resolve(target) + sep))
+            throw Error("The saved run folder has a file outside the folder.");
+        mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+        writeFileSync(path, Buffer.from(file.data, "base64"), { mode: 0o600 });
+    }
+    return target;
+}
+/** The last question the runtime asked in a run folder: the human.required event. */
+export function lastQuestion(dir) {
+    let lines;
+    try {
+        lines = readFileSync(join(dir, "events.jsonl"), "utf8").trim().split("\n");
+    }
+    catch {
+        return null;
+    }
+    for (const line of lines.reverse()) {
+        let event;
+        try {
+            event = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (event.event !== "human.required")
+            continue;
+        let form = event.schema;
+        if (!form) {
+            // A runtime that does not record the form: derive it from the step's out types.
+            try {
+                form = outputForm(JSON.parse(readFileSync(join(dir, "saved.method"), "utf8")).steps?.[event.step]?.out ?? {});
+            }
+            catch {
+                form = { type: "object", properties: {} };
+            }
+        }
+        return { step: `${event.step}:${event.iteration ?? 0}`, question: String(event.prompt ?? ""), form };
+    }
+    return null;
+}
+function outputForm(out) {
+    const one = (def) => {
+        const d = typeof def === "string" ? { type: def } : def ?? {};
+        const base = d.type === "number" ? { type: "number" } : d.type === "boolean" ? { type: "boolean" } : d.type === "list" ? { type: "array", items: d.fields ? outputForm(d.fields) : one(d.items) }
+            : d.type === "record" ? outputForm(d.fields ?? {}) : { type: "string" };
+        return d.description ? { ...base, description: d.description } : base;
+    };
+    return { type: "object", properties: Object.fromEntries(Object.entries(out).map(([k, v]) => [k, one(v)])), required: Object.keys(out), additionalProperties: false };
+}
+const finalCodes = new Set(["needs_input", "needs_update", "missing_secret", "invalid_inputs", "model_not_private"]);
+/** Run a claim with `method run METHOD_ID --version VERSION_ID` in a child process, then read its summary. */
+export function cliExecutor(options = {}) {
+    return async (claim, { runDir: dir, signal, api }) => {
+        mkdirSync(dir, { recursive: true, mode: 0o700 });
+        // The run record uses the production run ID, so the dashboard links both.
+        if (!existsSync(join(dir, "request.json")))
+            writePrivateJson(join(dir, "request.json"), { run_id: claim.run_id, workflow_id: claim.method_id, version_id: claim.version_id, server: api.server });
+        const inputs = join(dir, "production-inputs.json");
+        writePrivateJson(inputs, claim.inputs);
+        // Answers of ask steps, in the runtime's --human format.
+        const answers = Object.entries(claim.answers ?? {});
+        const human = join(dir, "production-answers.json");
+        if (answers.length)
+            writePrivateJson(human, { steps: Object.fromEntries(answers.map(([step, answer]) => [step, { outputs: answer.outputs }])) });
+        const command = options.command ?? methodCommand();
+        const out = openSync(join(dir, "worker.stdout.log"), "a", 0o600), err = openSync(join(dir, "worker.stderr.log"), "a", 0o600);
+        // The CLI sign-in of this computer reaches the child through its saved credential, not the environment.
+        const credential = api.apiKey.startsWith("mk_live_") ? { METHOD_API_KEY: api.apiKey } : {};
+        const child = spawn(command[0], [...command.slice(1), "run", claim.method_id, "--version", claim.version_id, "--server", api.server,
+            "--run-dir", dir, "--inputs", inputs, "--resume", ...(answers.length ? ["--human", human] : [])], {
+            detached: true, stdio: ["ignore", out, err],
+            env: { ...process.env, ...credential, METHOD_RUN_WORKER: "1", METHOD_NO_EXPLAIN: "1" },
+        });
+        closeSync(out);
+        closeSync(err);
+        const stop = () => {
+            try {
+                process.kill(-child.pid, "SIGTERM");
+            }
+            catch { /* Already stopped. */ }
+            setTimeout(() => { try {
+                process.kill(-child.pid, "SIGKILL");
+            }
+            catch { /* Stopped. */ } }, 10_000).unref();
+        };
+        signal.addEventListener("abort", stop, { once: true });
+        const code = await new Promise((done, fail) => { child.once("error", fail); child.once("exit", done); })
+            .finally(() => signal.removeEventListener("abort", stop));
+        const read = (name) => { try {
+            return JSON.parse(readFileSync(join(dir, name), "utf8"));
+        }
+        catch {
+            return null;
+        } };
+        const summary = read("summary.json");
+        if (summary?.status === "completed" || summary?.status === "unconfirmed")
+            return { status: "succeeded", result: summary.result };
+        if (signal.aborted)
+            return { status: "cancelled", error: { code: "cancelled", message: "The run was stopped." } };
+        if (summary?.status === "needs_input") {
+            // An ask step: the run waits for an answer, until it is answered or cancelled, or for the step's limits.timeout_ms.
+            const question = lastQuestion(dir);
+            if (question) {
+                let timeoutMs;
+                try {
+                    timeoutMs = JSON.parse(readFileSync(join(dir, "saved.method"), "utf8")).steps?.[question.step.split(":")[0]]?.limits?.timeout_ms;
+                }
+                catch { /* Default. */ }
+                return { status: "waiting", question, ...(typeof timeoutMs === "number" ? { timeoutMs } : {}) };
+            }
+            return { status: "failed", error: { code: "needs_input", message: String(summary.error ?? "The run needs input that a production worker cannot give.") } };
+        }
+        if (summary?.status === "failed")
+            return { status: "failed", error: { code: summary.code ?? "execution_failed", message: String(summary.error ?? "The run failed.") } };
+        const setup = read("setup-error.json");
+        if (setup)
+            return { status: "failed", error: { code: setup.code, message: String(setup.error) }, retryable: !finalCodes.has(setup.code) };
+        return { status: "failed", error: { code: "worker_failed", message: `The run process stopped (exit ${code}). See ${join(dir, "worker.stderr.log")}.` }, retryable: true };
+    };
+}
+function methodCommand() {
+    const source = import.meta.url.endsWith(".ts");
+    return [process.execPath, ...(source ? process.execArgv : []), fileURLToPath(new URL(`./method.${source ? "ts" : "js"}`, import.meta.url))];
+}
+/** `method worker` and `method keys`. */
+export async function productionCommand(args, clientFactory) {
+    const print = (value) => process.stdout.write(JSON.stringify(value, null, 2) + "\n");
+    if (args[0] === "worker") {
+        const { values } = parseArgs({ args: args.slice(1), options: { method: { type: "string", multiple: true }, concurrency: { type: "string" }, server: { type: "string" },
+                // The Python library sends more Method IDs on stdin, one a line, as its app starts runs of them.
+                "methods-from-stdin": { type: "boolean" } } });
+        const concurrency = values.concurrency === undefined ? 1 : Number(values.concurrency);
+        if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 32)
+            throw Error("Use --concurrency 1 to 32.");
+        const { Method } = await import("./production.js");
+        // METHOD_API_KEY, or the CLI sign-in of this computer.
+        const method = new Method({ server: values.server ?? DEFAULT_SERVER, worker: false });
+        const methods = new Set(values.method ?? []);
+        // One JSON line for each finished run, and one for each run that waits for an answer. A device run's result is printed here only.
+        const worker = method.worker.start({ concurrency, ...(methods.size || values["methods-from-stdin"] ? { methods } : {}),
+            onResult: result => { process.stdout.write(JSON.stringify(result) + "\n"); },
+            onWaiting: run => { process.stdout.write(JSON.stringify({ run_id: run.run_id, status: "waiting", question: run.question }) + "\n"); } });
+        if (values["methods-from-stdin"]) {
+            process.stdin.setEncoding("utf8");
+            let pending = "";
+            process.stdin.on("data", chunk => {
+                pending += chunk;
+                const lines = pending.split("\n");
+                pending = lines.pop() ?? "";
+                for (const line of lines)
+                    if (/^wf_[A-Za-z0-9_]+$/.test(line.trim()))
+                        methods.add(line.trim());
+            });
+        }
+        process.stderr.write(`Method worker ${worker.id} is waiting for runs on ${method.api.server}. Press Ctrl+C to stop after the current runs.\n`);
+        let signals = 0;
+        const stop = () => { signals++; void worker.stop({ cancel: signals > 1 }); if (signals === 1)
+            process.stderr.write("Stopping after the current runs. Press Ctrl+C again to stop them now.\n"); };
+        process.on("SIGINT", stop);
+        process.on("SIGTERM", stop);
+        try {
+            await worker.done;
+        }
+        finally {
+            process.removeListener("SIGINT", stop);
+            process.removeListener("SIGTERM", stop);
+            process.stdin.pause();
+        }
+        return;
+    }
+    const { values, positionals } = parseArgs({ args: args.slice(1), allowPositionals: true, options: { name: { type: "string" }, server: { type: "string" } } });
+    const client = clientFactory(values.server ?? DEFAULT_SERVER);
+    if (!client.token())
+        await client.login();
+    const action = positionals[0];
+    if (action === "create") {
+        if (!values.name)
+            throw Error("Use method keys create --name NAME.");
+        const created = await client.request("/api/service-keys", "POST", { name: values.name });
+        print({ ...created, next: "Save key as METHOD_API_KEY and webhook_secret as METHOD_WEBHOOK_SECRET in your app's secret store. They are shown only once. To write the key to an app's .env without showing it, use method connect APP_FOLDER." });
+        return;
+    }
+    if (action === "list" || !action) {
+        print(await client.request("/api/service-keys"));
+        return;
+    }
+    if (action === "revoke" && positionals[1]) {
+        print(await client.request(`/api/service-keys/${encodeURIComponent(positionals[1])}/revoke`, "POST", {}));
+        return;
+    }
+    throw Error("Use method keys create --name NAME, method keys list, or method keys revoke KEY_ID.");
+}

package/dist/packages/workflow-language/src/execution.d.ts

@@ -0,0 +1 @@
+export { stepDependencies } from "./validate.js";

package/dist/packages/workflow-language/src/execution.js

@@ -0,0 +1 @@
+export { stepDependencies } from "./validate.js";